diff --git a/.editorconfig b/.editorconfig index ed3ef24af..a04fb084e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -199,3 +199,22 @@ dotnet_naming_style.begins_with_i.required_prefix = I dotnet_naming_style.begins_with_i.required_suffix = dotnet_naming_style.begins_with_i.word_separator = dotnet_naming_style.begins_with_i.capitalization = pascal_case +csharp_style_namespace_declarations = block_scoped:silent +csharp_style_prefer_method_group_conversion = true:silent +csharp_style_prefer_top_level_statements = true:silent + +[*.{cs,vb}] +dotnet_style_operator_placement_when_wrapping = beginning_of_line +tab_width = 4 +indent_size = 4 +end_of_line = crlf +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_auto_properties = true:silent +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_prefer_simplified_boolean_expressions = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +dotnet_style_explicit_tuple_names = true:suggestion \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 759d4a1dd..59ff30570 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,6 @@ updates: directory: "/" # Location of package manifests schedule: interval: "daily" - target-branch: "develop" + target-branch: "housekeeping" ignore: - dependency-name: "Google.Protobuf.Tools" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2283f69d2..3094c3027 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,12 @@ on: branches: - master - develop + - housekeeping pull_request: branches: - master - develop + - housekeeping workflow_dispatch: jobs: @@ -27,7 +29,7 @@ jobs: - name: dotnet restore solution run: dotnet restore Fusee.sln - name: dotnet format solution - run: dotnet format Fusee.sln + run: dotnet format style Fusee.sln - name: Commiting changes uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/Directory.Build.props b/Directory.Build.props index 7748f48f0..f8220464e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -13,7 +13,7 @@ 0.13.0 0.13.0 0.13.0 - Copyright 2013-2022 + Copyright 2013-2023 MIT https://fusee3d.org/ true @@ -27,12 +27,14 @@ $(FuseeEngineRoot)\art\Deliverables\FuseeLogo.ico false - 10 + latest true false $(FuseeEngineRoot)\bin\Release\nuget + $(FuseeEngineRoot)\Fusee.ruleset + + + + + + \ No newline at end of file diff --git a/Examples/Complete/AdvancedUI/Android/Fusee.Examples.AdvancedUI.Android.csproj b/Examples/Complete/AdvancedUI/Android/Fusee.Examples.AdvancedUI.Android.csproj index 716bc73d3..3f1ea1465 100644 --- a/Examples/Complete/AdvancedUI/Android/Fusee.Examples.AdvancedUI.Android.csproj +++ b/Examples/Complete/AdvancedUI/Android/Fusee.Examples.AdvancedUI.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.AdvancedUI.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,36 +28,33 @@ 4 None None - true + True True False - false + False False - armeabi-v7a;x86_64 Xamarin False True - false - false - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/AdvancedUI/Android/Properties/AndroidManifest.xml b/Examples/Complete/AdvancedUI/Android/Properties/AndroidManifest.xml index 760b68c26..0ca26b228 100644 --- a/Examples/Complete/AdvancedUI/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/AdvancedUI/Android/Properties/AndroidManifest.xml @@ -2,5 +2,5 @@ - + \ No newline at end of file diff --git a/Examples/Complete/AdvancedUI/Core/AdvancedUI.cs b/Examples/Complete/AdvancedUI/Core/AdvancedUI.cs index 89d2a3332..00f31ee63 100644 --- a/Examples/Complete/AdvancedUI/Core/AdvancedUI.cs +++ b/Examples/Complete/AdvancedUI/Core/AdvancedUI.cs @@ -1,4 +1,4 @@ -using Fusee.Base.Common; +using Fusee.Base.Common; using Fusee.Base.Core; using Fusee.Engine.Common; using Fusee.Engine.Core; @@ -156,15 +156,15 @@ public override void Init() _gui = CreateGui(); - // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); - - //Create a scene picker for performing visibility tests - _scenePicker = new ScenePicker(_scene); - // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_scene); _guiRenderer = new SceneRendererForward(_gui); + + //Create a scene picker for performing visibility tests + _scenePicker = new ScenePicker(_scene, _sceneRenderer.PrePassVisitor.CameraPrepassResults); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override void Update() @@ -235,6 +235,11 @@ public override void RenderAFrame() float4x4 mvpMonkey = projection * view * model; float3 clipPos = float4x4.TransformPerspective(mvpMonkey, uiInput.Position); + // go from clip pos to pixel coordinates + var pixelPos = clipPos * 0.5f + 0.5f; // shift from [-1,1] to [0,1] + pixelPos.y = 1f - pixelPos.y; // invert y + pixelPos.x = pixelPos.x * Width; + pixelPos.y = pixelPos.y * Height; float2 canvasPosCircle = new float2(clipPos.x, clipPos.y) * 0.5f + 0.5f; canvasPosCircle.x *= _canvasWidth; @@ -247,7 +252,8 @@ public override void RenderAFrame() circle.GetComponent().Offsets = GuiElementPosition.CalcOffsets(AnchorPos.Middle, pos, _canvasHeight, _canvasWidth, uiInput.Size); //1.1 Check if circle is visible - PickResult newPick = _scenePicker.Pick(RC, new float2(clipPos.x, clipPos.y)).ToList().OrderBy(pr => pr.ClipPos.z).FirstOrDefault(); + MeshPickResult newPick = (MeshPickResult)_scenePicker.Pick(pixelPos.xy, Width, Height).ToList().OrderBy(pr => pr.ClipPos.z).FirstOrDefault(); + if (newPick != null && uiInput.AffectedTriangles[0] == newPick.Triangle) //VISIBLE { @@ -336,11 +342,11 @@ public override void RenderAFrame() // Constantly check for interactive objects. if (!Mouse.Desc.Contains("Android")) - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); if (Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } Present(); diff --git a/Examples/Complete/Camera/Android/Fusee.Examples.Camera.Android.csproj b/Examples/Complete/Camera/Android/Fusee.Examples.Camera.Android.csproj index 0646d5f7f..1c50b8b55 100644 --- a/Examples/Complete/Camera/Android/Fusee.Examples.Camera.Android.csproj +++ b/Examples/Complete/Camera/Android/Fusee.Examples.Camera.Android.csproj @@ -1,14 +1,10 @@ - + Fusee.Examples.Camera.Android Fusee.Examples.Camera.Android Debug AnyCPU - - $(FuseeEngineRoot)\bin\$(Configuration) - - ..\..\..\..\bin\$(Configuration) $(BaseOutputPath)\Examples\Camera\Android 8.0.30703 @@ -20,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -32,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a;x86_64 Xamarin @@ -46,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/Camera/Android/Properties/AndroidManifest.xml b/Examples/Complete/Camera/Android/Properties/AndroidManifest.xml index 87f7791a6..776cdb91c 100644 --- a/Examples/Complete/Camera/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/Camera/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/Camera/Blazor/Fusee.Examples.Camera.Blazor.csproj b/Examples/Complete/Camera/Blazor/Fusee.Examples.Camera.Blazor.csproj index 339c3470f..21ebdddd1 100644 --- a/Examples/Complete/Camera/Blazor/Fusee.Examples.Camera.Blazor.csproj +++ b/Examples/Complete/Camera/Blazor/Fusee.Examples.Camera.Blazor.csproj @@ -14,9 +14,9 @@ - - - + + + diff --git a/Examples/Complete/Camera/Core/Camera.cs b/Examples/Complete/Camera/Core/Camera.cs index abce4009a..65c491e8e 100644 --- a/Examples/Complete/Camera/Core/Camera.cs +++ b/Examples/Complete/Camera/Core/Camera.cs @@ -44,8 +44,6 @@ private async Task Load() { _gui = await FuseeGuiHelper.CreateDefaultGuiAsync(this, CanvasRenderMode.Screen, "FUSEE Camera Example"); - // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); _frustum = new WireframeCube(); SceneNode frustumNode = new() @@ -118,6 +116,9 @@ private async Task Load() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_rocketScene); _guiRenderer = new SceneRendererForward(_gui); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override async Task InitAsync() @@ -194,11 +195,11 @@ public override void RenderAFrame() if (!Mouse.Desc.Contains("Android")) { - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); } if (Touch != null && Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } // Swap buffers: Show the contents of the backbuffer (containing the currently rendered frame) on the front buffer. diff --git a/Examples/Complete/ComputeFractal/Core/ComputeFractal.cs b/Examples/Complete/ComputeFractal/Core/ComputeFractal.cs index 56afe751b..366e5966f 100644 --- a/Examples/Complete/ComputeFractal/Core/ComputeFractal.cs +++ b/Examples/Complete/ComputeFractal/Core/ComputeFractal.cs @@ -91,7 +91,8 @@ public override void Init() RC.SetEffect(_computeShader); _rect.SetData(_rectData); _colors.SetData(_colorData); - _sih = new SceneInteractionHandler(_gui); + + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } // RenderAFrame is called once a frame @@ -115,11 +116,11 @@ public override void RenderAFrame() if (!Mouse.Desc.Contains("Android")) { - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); } if (Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } Present(); } diff --git a/Examples/Complete/Deferred/Android/Fusee.Examples.Deferred.Android.csproj b/Examples/Complete/Deferred/Android/Fusee.Examples.Deferred.Android.csproj index 066bb39a5..aa488835c 100644 --- a/Examples/Complete/Deferred/Android/Fusee.Examples.Deferred.Android.csproj +++ b/Examples/Complete/Deferred/Android/Fusee.Examples.Deferred.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.Deferred.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,44 +28,33 @@ 4 None None - true + True True False - false + False False - armeabi-v7a;x86_64 Xamarin False True - false - false - false - false - ..\..\..\..\bin\Debug\Examples\Deferred\Android\ - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true - false - false - false - false - ..\..\..\..\bin\Release\Examples\Deferred\Android\ + True - + diff --git a/Examples/Complete/Deferred/Android/Properties/AndroidManifest.xml b/Examples/Complete/Deferred/Android/Properties/AndroidManifest.xml index 2344cdcad..5a37c538d 100644 --- a/Examples/Complete/Deferred/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/Deferred/Android/Properties/AndroidManifest.xml @@ -1,11 +1,6 @@  - - - - - - + - + \ No newline at end of file diff --git a/Examples/Complete/Deferred/Blazor/Fusee.Examples.Deferred.Blazor.csproj b/Examples/Complete/Deferred/Blazor/Fusee.Examples.Deferred.Blazor.csproj index 6c52a4787..0dc75b48c 100644 --- a/Examples/Complete/Deferred/Blazor/Fusee.Examples.Deferred.Blazor.csproj +++ b/Examples/Complete/Deferred/Blazor/Fusee.Examples.Deferred.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 service-worker-assets.js @@ -15,9 +15,9 @@ - - - + + + diff --git a/Examples/Complete/GeometryEditing/Android/Fusee.Examples.GeometryEditing.Android.csproj b/Examples/Complete/GeometryEditing/Android/Fusee.Examples.GeometryEditing.Android.csproj index 24943d6ef..600563ba7 100644 --- a/Examples/Complete/GeometryEditing/Android/Fusee.Examples.GeometryEditing.Android.csproj +++ b/Examples/Complete/GeometryEditing/Android/Fusee.Examples.GeometryEditing.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.GeometryEditing.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a,x86 Xamarin @@ -42,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/GeometryEditing/Android/Properties/AndroidManifest.xml b/Examples/Complete/GeometryEditing/Android/Properties/AndroidManifest.xml index 7b2d0a098..d1be16535 100644 --- a/Examples/Complete/GeometryEditing/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/GeometryEditing/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/GeometryEditing/Core/GeometryEditing.cs b/Examples/Complete/GeometryEditing/Core/GeometryEditing.cs index a7368793c..c20810e04 100644 --- a/Examples/Complete/GeometryEditing/Core/GeometryEditing.cs +++ b/Examples/Complete/GeometryEditing/Core/GeometryEditing.cs @@ -120,7 +120,7 @@ public override void Init() _scene = new SceneContainer { Children = new List { _parentNode } }; _renderer = new SceneRendererForward(_scene); - _scenePicker = new ScenePicker(_scene); + _scenePicker = new ScenePicker(_scene, _renderer.PrePassVisitor.CameraPrepassResults); _activeGeometrys = new Dictionary(); } @@ -141,7 +141,7 @@ public override void RenderAFrame() { float2 pickPosClip = _pickPos * new float2(2.0f / Width, -2.0f / Height) + new float2(-1, 1); - PickResult newPick = _scenePicker.Pick(RC, pickPosClip).ToList().OrderBy(pr => pr.ClipPos.z).FirstOrDefault(); + PickResult newPick = _scenePicker.Pick(pickPosClip, Width, Height).ToList().OrderBy(pr => pr.ClipPos.z).FirstOrDefault(); if (newPick?.Node != _currentPick?.Node) { diff --git a/Examples/Complete/Integrations/Core/Main.cs b/Examples/Complete/Integrations/Core/Main.cs index 200794339..bad8773d4 100644 --- a/Examples/Complete/Integrations/Core/Main.cs +++ b/Examples/Complete/Integrations/Core/Main.cs @@ -45,9 +45,6 @@ private async Task Load() _gui = await FuseeGuiHelper.CreateDefaultGuiAsync(this, CanvasRenderMode.Screen, "FUSEE Simple Example"); - // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); - // Load the rocket model _rocketScene = await AssetStorage.GetAsync("RocketFus.fus"); rocketTransform = _rocketScene.Children[0].GetTransform(); @@ -80,6 +77,9 @@ private async Task Load() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_rocketScene); _guiRenderer = new SceneRendererForward(_gui); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override async Task InitAsync() @@ -140,10 +140,10 @@ public override void RenderAFrame() //Constantly check for interactive objects. if (!Mouse.Desc.Contains("Android")) - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); if (Touch != null && Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } Present(); diff --git a/Examples/Complete/Labyrinth/Core/Labyrinth.cs b/Examples/Complete/Labyrinth/Core/Labyrinth.cs index be92b863c..912f596d2 100644 --- a/Examples/Complete/Labyrinth/Core/Labyrinth.cs +++ b/Examples/Complete/Labyrinth/Core/Labyrinth.cs @@ -284,7 +284,6 @@ private SceneContainer CreateScene() public override void Init() { _gui = CreateGui(); - _sih = new SceneInteractionHandler(_gui); // Find the ball and create AABB FindBall(); @@ -311,6 +310,8 @@ public override void Init() _bodyPivotTransform = _scene.Children.FindNodes(node => node.Name == "Bodytrans").FirstOrDefault()?.GetTransform(); _bodyNode = _scene.Children.FindNodes(node => node.Name == "Body")?.FirstOrDefault(); _bodyTransform = _bodyNode?.GetTransform(); + + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override void Update() @@ -369,11 +370,11 @@ public override void RenderAFrame() _guiRenderer.Render(RC); if (!Mouse.Desc.Contains("Android")) - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); if (Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } // Swap buffers: Show the contents of the backbuffer (containing the currently rendered frame) on the front buffer. @@ -843,7 +844,7 @@ private void OnWonGame() if (!_isGuiCreated) { _gui = CreateWinningGui(); - _sih = new SceneInteractionHandler(_gui); + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); _guiRenderer = new SceneRendererForward(_gui); _isGuiCreated = true; } diff --git a/Examples/Complete/Materials/Android/Fusee.Examples.Materials.Android.csproj b/Examples/Complete/Materials/Android/Fusee.Examples.Materials.Android.csproj index a18574c18..95d41cb81 100644 --- a/Examples/Complete/Materials/Android/Fusee.Examples.Materials.Android.csproj +++ b/Examples/Complete/Materials/Android/Fusee.Examples.Materials.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.Materials.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,43 +28,33 @@ 4 None None - true + True True False - false + False False - armeabi-v7a;x86_64 Xamarin False True - false - false - false - ..\..\..\..\bin\Debug\Examples\Materials\Android\ - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true - false - false - false - false - ..\..\..\..\bin\Release\Examples\Materials\Android\ + True - + diff --git a/Examples/Complete/Materials/Android/Properties/AndroidManifest.xml b/Examples/Complete/Materials/Android/Properties/AndroidManifest.xml index f1081c3ee..1a28431ec 100644 --- a/Examples/Complete/Materials/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/Materials/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/Materials/Core/Materials.cs b/Examples/Complete/Materials/Core/Materials.cs index 4861feaf6..b3efceac3 100644 --- a/Examples/Complete/Materials/Core/Materials.cs +++ b/Examples/Complete/Materials/Core/Materials.cs @@ -63,8 +63,6 @@ public override void Init() } }); - // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); // Set the clear color for the backbuffer to white (100% intensity in all color channels R, G, B, A). _campComp.BackgroundColor = new float4(0.8f, 0.9f, 1, 1).LinearColorFromSRgb(); @@ -160,6 +158,10 @@ public override void Init() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererDeferred(_scene); _guiRenderer = new SceneRendererForward(_gui); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); + } public override void Update() @@ -209,12 +211,12 @@ public override void RenderAFrame() if (!Mouse.Desc.Contains("Android")) { - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); } if (Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } Present(); diff --git a/Examples/Complete/MeshingAround/Android/Fusee.Examples.MeshingAround.Android.csproj b/Examples/Complete/MeshingAround/Android/Fusee.Examples.MeshingAround.Android.csproj index 57533037d..75dffe033 100644 --- a/Examples/Complete/MeshingAround/Android/Fusee.Examples.MeshingAround.Android.csproj +++ b/Examples/Complete/MeshingAround/Android/Fusee.Examples.MeshingAround.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.MeshingAround.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a,x86 Xamarin @@ -42,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/MeshingAround/Android/Properties/AndroidManifest.xml b/Examples/Complete/MeshingAround/Android/Properties/AndroidManifest.xml index e6f4950de..34a296837 100644 --- a/Examples/Complete/MeshingAround/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/MeshingAround/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/Picking/Android/Fusee.Examples.Picking.Android.csproj b/Examples/Complete/Picking/Android/Fusee.Examples.Picking.Android.csproj index b42ecd28f..4b6b0e47f 100644 --- a/Examples/Complete/Picking/Android/Fusee.Examples.Picking.Android.csproj +++ b/Examples/Complete/Picking/Android/Fusee.Examples.Picking.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.Picking.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a;x86_64 Xamarin @@ -42,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/Picking/Android/Properties/AndroidManifest.xml b/Examples/Complete/Picking/Android/Properties/AndroidManifest.xml index fe5990574..26a96b96b 100644 --- a/Examples/Complete/Picking/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/Picking/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/Picking/Blazor/Fusee.Examples.Picking.Blazor.csproj b/Examples/Complete/Picking/Blazor/Fusee.Examples.Picking.Blazor.csproj index 154fb67ac..87b377417 100644 --- a/Examples/Complete/Picking/Blazor/Fusee.Examples.Picking.Blazor.csproj +++ b/Examples/Complete/Picking/Blazor/Fusee.Examples.Picking.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 service-worker-assets.js @@ -14,9 +14,9 @@ - - - + + + diff --git a/Examples/Complete/Picking/Core/Picking.cs b/Examples/Complete/Picking/Core/Picking.cs index 43029af33..f41299998 100644 --- a/Examples/Complete/Picking/Core/Picking.cs +++ b/Examples/Complete/Picking/Core/Picking.cs @@ -1,4 +1,4 @@ -using Fusee.Base.Common; +using Fusee.Base.Common; using Fusee.Base.Core; using Fusee.Engine.Common; using Fusee.Engine.Core; @@ -48,12 +48,12 @@ private async Task Load() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_scene); - _scenePicker = new ScenePicker(_scene); + _scenePicker = new ScenePicker(_scene, _sceneRenderer.PrePassVisitor.CameraPrepassResults); _gui = await FuseeGuiHelper.CreateDefaultGuiAsync(this, CanvasRenderMode.Screen, "FUSEE Picking Example"); // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); _guiRenderer = new SceneRendererForward(_gui); + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override async Task InitAsync() @@ -121,9 +121,7 @@ public override void RenderAFrame() //Picking if (_pick) { - float2 pickPosClip = (_pickPos * new float2(2.0f / Width, -2.0f / Height)) + new float2(-1, 1); - - PickResult newPick = _scenePicker.Pick(RC, pickPosClip).ToList().OrderBy(pr => pr.ClipPos.z) + PickResult newPick = _scenePicker.Pick(Input.Mouse.Position, RC.ViewportWidth, RC.ViewportHeight).ToList().OrderBy(pr => pr.ClipPos.z) .FirstOrDefault(); Diagnostics.Debug(newPick); @@ -152,11 +150,11 @@ public override void RenderAFrame() // Constantly check for interactive objects. if (!Input.Mouse.Desc.Contains("Android")) - _sih.CheckForInteractiveObjects(RC, Input.Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Input.Mouse.Position, Width, Height); if (Input.Touch != null && Input.Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Input.Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Input.Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Input.Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } // Swap buffers: Show the contents of the back buffer (containing the currently rendered frame) on the front buffer. diff --git a/Examples/Complete/PickingRayCast/Android/Fusee.Examples.PickingRayCast.Android.csproj b/Examples/Complete/PickingRayCast/Android/Fusee.Examples.PickingRayCast.Android.csproj index 19d0bb5c9..accde2a38 100644 --- a/Examples/Complete/PickingRayCast/Android/Fusee.Examples.PickingRayCast.Android.csproj +++ b/Examples/Complete/PickingRayCast/Android/Fusee.Examples.PickingRayCast.Android.csproj @@ -1,12 +1,12 @@ - + - Fusee.Examples.Picking.Android - Fusee.Examples.Picking.Android + Fusee.Examples.PickingRayCast.Android + Fusee.Examples.PickingRayCast.Android Debug AnyCPU ..\..\..\..\bin\$(Configuration) - $(BaseOutputPath)\Examples\Picking\Android + $(BaseOutputPath)\Examples\PickingRayCast\Android 8.0.30703 2.0 {027608D0-ACAF-488E-AA8B-F2BA547CBC71} @@ -16,47 +16,45 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 True - full + portable False DEBUG;TRACE;PLATFORM_ANDROID prompt 4 None None - true + True True False False False - armeabi-v7a;x86_64 Xamarin False True - ..\..\..\..\bin\Debug\Examples\PickingRayCast\Android\ - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/PickingRayCast/Android/Properties/AndroidManifest.xml b/Examples/Complete/PickingRayCast/Android/Properties/AndroidManifest.xml index 01463fb50..c95f65ca7 100644 --- a/Examples/Complete/PickingRayCast/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/PickingRayCast/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + - + \ No newline at end of file diff --git a/Examples/Complete/PickingRayCast/Blazor/Fusee.Examples.PickingRayCast.Blazor.csproj b/Examples/Complete/PickingRayCast/Blazor/Fusee.Examples.PickingRayCast.Blazor.csproj index 7bbeb9a50..f5cca5f10 100644 --- a/Examples/Complete/PickingRayCast/Blazor/Fusee.Examples.PickingRayCast.Blazor.csproj +++ b/Examples/Complete/PickingRayCast/Blazor/Fusee.Examples.PickingRayCast.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 service-worker-assets.js @@ -14,9 +14,9 @@ - - - + + + diff --git a/Examples/Complete/PickingRayCast/Core/PickingRayCast.cs b/Examples/Complete/PickingRayCast/Core/PickingRayCast.cs index 52e77bf0f..2b1d54b0c 100644 --- a/Examples/Complete/PickingRayCast/Core/PickingRayCast.cs +++ b/Examples/Complete/PickingRayCast/Core/PickingRayCast.cs @@ -83,7 +83,7 @@ private async void Load() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_scene); - _sceneRayCaster = new SceneRayCaster(_scene, Cull.Clockwise); + _sceneRayCaster = new SceneRayCaster(_scene, _sceneRenderer.PrePassVisitor.CameraPrepassResults, Cull.Clockwise); // Create the interaction handler _guiRenderer = new SceneRendererForward(_gui); @@ -147,7 +147,38 @@ public override void Update() // Check for hits if (_pick) { - var castHit = _sceneRayCaster.RayPick(RC, _pickPos).ToList().OrderBy(rr => rr.DistanceFromOrigin).FirstOrDefault(); + // prepare mouse coordinates; for each camera calculate clip space mouse position and create a new ray which travels through the scene and collects meshes + CameraResult pickCam = default; + Rectangle pickCamRect = new(); + + // check in which camera our mouse is currently positioned (left or right) + foreach (var camRes in _sceneRenderer.PrePassVisitor.CameraPrepassResults) + { + Rectangle camRect = new() + { + Left = (int)(camRes.Camera.Viewport.x * RC.ViewportWidth / 100), + Top = (int)(camRes.Camera.Viewport.y * RC.ViewportHeight / 100) + }; + camRect.Right = ((int)(camRes.Camera.Viewport.z * RC.ViewportWidth) / 100) + camRect.Left; + camRect.Bottom = ((int)(camRes.Camera.Viewport.w * RC.ViewportHeight) / 100) + camRect.Top; + + if (!float2.PointInRectangle(new float2(camRect.Left, camRect.Top), new float2(camRect.Right, camRect.Bottom), Input.Mouse.Position)) + continue; + + if (pickCam == default || camRes.Camera.Layer > pickCam.Camera.Layer) + { + pickCam = camRes; + pickCamRect = camRect; + } + } + + // generate fitting clip position with the currently used camera rectangle + var pickPosClip = ((Input.Mouse.Position - new float2(pickCamRect.Left, pickCamRect.Top)) * new float2(2.0f / pickCamRect.Width, -2.0f / pickCamRect.Height)) + new float2(-1, 1); + + // generate ray at mouse position and... + var ray = new RayF(pickPosClip, pickCam.View, pickCam.Camera.GetProjectionMat(RC.ViewportWidth, RC.ViewportHeight, out var _)); + // ... send it through the scene + var castHit = _sceneRayCaster.Traverse(ray).ToList().OrderBy(rr => rr.DistanceFromOrigin).FirstOrDefault(); if (castHit != null) castHit.Node.GetComponent().SurfaceInput.Albedo = (float4)ColorUint.LawnGreen; diff --git a/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2.cs b/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2.cs index d9774fcf3..c14c9a59c 100644 --- a/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2.cs +++ b/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2.cs @@ -1,6 +1,5 @@ using Fusee.Engine.Common; using Fusee.Engine.Core; -using Fusee.PointCloud.Common; namespace Fusee.Examples.PointCloudPotree2.Core { @@ -12,8 +11,6 @@ public class PointCloudPotree2 : RenderCanvas public bool IsInitialized { get; private set; } public bool IsAlive { get; private set; } - public RenderMode PointRenderMode = RenderMode.DynamicMesh; - public bool ClosingRequested { get => _pointRenderingCore.ClosingRequested; @@ -22,11 +19,6 @@ public bool ClosingRequested private PointCloudPotree2Core _pointRenderingCore; - public PointCloudPotree2() - { - - } - public override void Init() { VSync = false; @@ -65,13 +57,13 @@ public override void Update() return; } - _pointRenderingCore.Update(true); + _pointRenderingCore?.Update(true); } // Is called when the window was resized public override void Resize(ResizeEventArgs e) { - _pointRenderingCore.Resize(e.Width, e.Height); + _pointRenderingCore?.Resize(e.Width, e.Height); } public override void DeInit() diff --git a/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2Core.cs b/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2Core.cs index 98bca57ab..479d52512 100644 --- a/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2Core.cs +++ b/Examples/Complete/PointCloudPotree2/Core/PointCloudPotree2Core.cs @@ -1,13 +1,16 @@ -using Fusee.Base.Common; +using Fusee.Base.Common; using Fusee.Engine.Core; +using Fusee.Engine.Core.Primitives; using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using Fusee.PointCloud.Common; using Fusee.PointCloud.Core.Scene; using Fusee.PointCloud.Potree.V2; +using Fusee.PointCloud.Potree.V2.Data; using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace Fusee.Examples.PointCloudPotree2.Core { @@ -17,6 +20,16 @@ public class PointCloudPotree2Core public WritableTexture RenderTexture { get; private set; } + /// + /// This value is used, when we use this class as a + /// + public float2 ExternalMousePosition { get; set; } + + /// + /// This value is used, when we use this class as a + /// + public int2 ExternalCanvasSize { get; set; } + public bool ClosingRequested { get { return _closingRequested; } @@ -24,7 +37,7 @@ public bool ClosingRequested } private bool _closingRequested; - public RenderMode PointRenderMode = RenderMode.StaticMesh; + public RenderMode PointRenderMode = RenderMode.DynamicMesh; public string AssetsPath = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); private static float _angleHorz, _angleVert, _angleVelHorz, _angleVelVert; @@ -50,8 +63,12 @@ public bool ClosingRequested private Potree2Reader _potreeReader; private PotreeData _potreeData; + private ScenePicker _picker; + private readonly Transform _pickResultTransform = new(); + private readonly RenderContext _rc; + public void OnLoadNewFile(object sender, EventArgs e) { var path = PointRenderingParams.Instance.PathToOocFile; @@ -59,22 +76,32 @@ public void OnLoadNewFile(object sender, EventArgs e) if (path == null || path == string.Empty) return; - _potreeData = new PotreeData(path); - _potreeReader = new Potree2Reader(ref _potreeData); + _potreeData = _potreeReader.ReadNewFile(path); - _pointCloud = (PointCloudComponent)_potreeReader.GetPointCloudComponent(RenderMode.DynamicMesh); + _pointCloud = (PointCloudComponent)_potreeReader.GetPointCloudComponent(PointRenderMode); _pointCloud.PointCloudImp.MinProjSizeModifier = PointRenderingParams.Instance.ProjectedSizeModifier; _pointCloud.PointCloudImp.PointThreshold = PointRenderingParams.Instance.PointThreshold; _pointCloud.Camera = _cam; _pointCloudNode.Components[3] = _pointCloud; + + // re-generate scene picker + if (PointRenderMode == RenderMode.DynamicMesh) + { + _picker = new ScenePicker(_scene, _sceneRenderer.PrePassVisitor.CameraPrepassResults, Engine.Common.Cull.None, new List() + { + new PointCloudPickerModule(((PointCloud.Potree.Potree2CloudDynamic)_pointCloud.PointCloudImp).VisibilityTester.Octree, + (PointCloud.Potree.Potree2CloudDynamic)_pointCloud.PointCloudImp, + (float)_potreeData.Metadata.Spacing) + }); + } } public PointCloudPotree2Core(RenderContext rc) { - _potreeData = new PotreeData(Path.Combine(AssetsPath, PointRenderingParams.Instance.PathToOocFile)); - _potreeReader = new Potree2Reader(ref _potreeData); + _potreeReader = new Potree2Reader(Path.Combine(AssetsPath, PointRenderingParams.Instance.PathToOocFile)); + _potreeData = _potreeReader.PotreeData; _rc = rc; } @@ -157,6 +184,33 @@ public void Init() _sceneRenderer.VisitorModules.Add(new PointCloudRenderModule(_sceneRenderer.GetType() == typeof(SceneRendererForward))); _pointCloud.Camera = _cam; + + // Generate picker, pass camera prepass results and the picker module + // which needs the point cloud, the octree and the spacing + // does not work for instanced and static point cloud meshes + if (PointRenderMode == RenderMode.DynamicMesh) + { + _picker = new ScenePicker(_scene, _sceneRenderer.PrePassVisitor.CameraPrepassResults, Engine.Common.Cull.None, new List() + { + new PointCloudPickerModule(((PointCloud.Potree.Potree2CloudDynamic)_pointCloud.PointCloudImp).VisibilityTester.Octree, + (PointCloud.Potree.Potree2CloudDynamic)_pointCloud.PointCloudImp, + (float)_potreeData.Metadata.Spacing) + }); + + // Add sphere to visual identify the pick result + _scene.Children.Add(new SceneNode + { + Components = new List() + { + _pickResultTransform, + MakeEffect.FromDiffuse(new float4(1,0,0,1)), + new Sphere(10, 10) + } + }); + + _pickResultTransform.Translation = _pointCloud.PointCloudImp.Center; + _pickResultTransform.Scale = float3.One * 0.05f; + } } // RenderAFrame is called once a frame @@ -227,6 +281,30 @@ public void Update(bool allowInput) _angleVelVert = 0; _camTransform.FpsView(_angleHorz, _angleVert, Input.Keyboard.WSAxis, Input.Keyboard.ADAxis, Time.DeltaTimeUpdate * 20); + + + if (!_keys && Input.Mouse.RightButton && PointRenderMode == RenderMode.DynamicMesh) + { + var size = RenderToTexture ? ExternalCanvasSize : new int2(_rc.ViewportWidth, _rc.ViewportHeight); + var mousePos = RenderToTexture ? ExternalMousePosition : Input.Mouse.Position; + var result = _picker?.Pick(mousePos, size.x, size.y).Where(x => x is PointCloudPickResult).Cast(); + if (result != null && result.Any()) + { + var minElement = result.FirstOrDefault(); + + // get min x/y distance point + foreach (var r in result) + { + if (r.DistanceToRay.x < minElement.DistanceToRay.x && r.DistanceToRay.y < minElement.DistanceToRay.y) + { + minElement = r; + } + } + _pickResultTransform.Translation = minElement.Mesh.Vertices[minElement.VertIdx]; + } + + } + } private void OnThresholdChanged(int newValue) diff --git a/Examples/Complete/PointCloudPotree2/ImGui/ImGuiApp.cs b/Examples/Complete/PointCloudPotree2/ImGui/ImGuiApp.cs index b8f750998..4b8fedacd 100644 --- a/Examples/Complete/PointCloudPotree2/ImGui/ImGuiApp.cs +++ b/Examples/Complete/PointCloudPotree2/ImGui/ImGuiApp.cs @@ -57,12 +57,12 @@ public override async Task InitAsync() _fuControl.Init(); await base.InitAsync(); - _picker = new ImGuiFilePicker(Path.Combine(Environment.CurrentDirectory, ""), false, ".json"); + _picker = new ImGuiFilePicker(new DirectoryInfo(Environment.CurrentDirectory), ".json"); _picker.OnPicked += (s, file) => { - if (string.IsNullOrEmpty(file)) return; + if (file == null || !file.Exists) return; - PointRenderingParams.Instance.PathToOocFile = new FileInfo(file).Directory.FullName; + PointRenderingParams.Instance.PathToOocFile = file.DirectoryName; if (_fuControl != null) { diff --git a/Examples/Complete/PointCloudPotree2/ImGui/PointCloudRenderingControl.cs b/Examples/Complete/PointCloudPotree2/ImGui/PointCloudRenderingControl.cs index 70c384dcc..349708941 100644 --- a/Examples/Complete/PointCloudPotree2/ImGui/PointCloudRenderingControl.cs +++ b/Examples/Complete/PointCloudPotree2/ImGui/PointCloudRenderingControl.cs @@ -2,7 +2,9 @@ using Fusee.Engine.Core; using Fusee.Examples.PointCloudPotree2.Core; using Fusee.ImGuiImp.Desktop.Templates; +using Fusee.Math.Core; using Fusee.PointCloud.Common; +using ImGuiNET; using System; namespace Fusee.Examples.PointCloudPotree2.Gui @@ -46,6 +48,13 @@ public override void Init() protected override ITextureHandle RenderAFrame() { _pointRenderingCore.RenderAFrame(); + + // set mouse position with offset to canvas + var iScreenPos = new float2(ImGui.GetCursorScreenPos().X, ImGui.GetCursorScreenPos().Y); + var fbScale = ImGui.GetIO().DisplayFramebufferScale; + var scaledInput = new float2(Input.Mouse.Position.x / fbScale.X, Input.Mouse.Position.y / fbScale.Y); + _pointRenderingCore.ExternalMousePosition = scaledInput - iScreenPos; + return _pointRenderingCore.RenderTexture?.TextureHandle; } @@ -62,6 +71,8 @@ public override void Update(bool allowInput) // Is called when the window was resized protected override void Resize(int width, int height) { + // set size from extern + _pointRenderingCore.ExternalCanvasSize = new Math.Core.int2(width, height); _pointRenderingCore.Resize(width, height); } diff --git a/Examples/Complete/PointCloudPotree2/Wpf/Fusee.Examples.PointCloudPotree2.Wpf.csproj b/Examples/Complete/PointCloudPotree2/Wpf/Fusee.Examples.PointCloudPotree2.Wpf.csproj index 7cd743182..bded40a0f 100644 --- a/Examples/Complete/PointCloudPotree2/Wpf/Fusee.Examples.PointCloudPotree2.Wpf.csproj +++ b/Examples/Complete/PointCloudPotree2/Wpf/Fusee.Examples.PointCloudPotree2.Wpf.csproj @@ -32,7 +32,7 @@ - + \ No newline at end of file diff --git a/Examples/Complete/RenderContextOnly/Android/Fusee.Examples.RenderContextOnly.Android.csproj b/Examples/Complete/RenderContextOnly/Android/Fusee.Examples.RenderContextOnly.Android.csproj index 73972b9da..32b1fef4b 100644 --- a/Examples/Complete/RenderContextOnly/Android/Fusee.Examples.RenderContextOnly.Android.csproj +++ b/Examples/Complete/RenderContextOnly/Android/Fusee.Examples.RenderContextOnly.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.RenderContextOnly.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,35 +28,33 @@ 4 None None - true + True True False False False - armeabi-v7a;x86_64 Xamarin False True - ..\..\..\..\bin\Debug\Libraries\Fusee.Examples.RenderContextOnly.Android\ - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/RenderContextOnly/Android/Properties/AndroidManifest.xml b/Examples/Complete/RenderContextOnly/Android/Properties/AndroidManifest.xml index a466b7b93..8d9697f7a 100644 --- a/Examples/Complete/RenderContextOnly/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/RenderContextOnly/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + - + \ No newline at end of file diff --git a/Examples/Complete/RenderContextOnly/Blazor/Fusee.Examples.RenderContextOnly.Blazor.csproj b/Examples/Complete/RenderContextOnly/Blazor/Fusee.Examples.RenderContextOnly.Blazor.csproj index 274cc0ae0..7f704f9af 100644 --- a/Examples/Complete/RenderContextOnly/Blazor/Fusee.Examples.RenderContextOnly.Blazor.csproj +++ b/Examples/Complete/RenderContextOnly/Blazor/Fusee.Examples.RenderContextOnly.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 service-worker-assets.js @@ -14,9 +14,9 @@ - - - + + + diff --git a/Examples/Complete/RenderLayer/Android/Fusee.Examples.RenderLayer.Android.csproj b/Examples/Complete/RenderLayer/Android/Fusee.Examples.RenderLayer.Android.csproj index 1fb54ee84..b27b7f592 100644 --- a/Examples/Complete/RenderLayer/Android/Fusee.Examples.RenderLayer.Android.csproj +++ b/Examples/Complete/RenderLayer/Android/Fusee.Examples.RenderLayer.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.RenderLayer.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a;x86_64 Xamarin @@ -42,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/RenderLayer/Android/Properties/AndroidManifest.xml b/Examples/Complete/RenderLayer/Android/Properties/AndroidManifest.xml index a466b7b93..e32b6f66e 100644 --- a/Examples/Complete/RenderLayer/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/RenderLayer/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + - + \ No newline at end of file diff --git a/Examples/Complete/Simple/Android/Fusee.Examples.Simple.Android.csproj b/Examples/Complete/Simple/Android/Fusee.Examples.Simple.Android.csproj index da7bcfee9..3bd05f6f8 100644 --- a/Examples/Complete/Simple/Android/Fusee.Examples.Simple.Android.csproj +++ b/Examples/Complete/Simple/Android/Fusee.Examples.Simple.Android.csproj @@ -1,10 +1,10 @@ - + Fusee.Examples.Simple.Android Fusee.Examples.Simple.Android - Debug - AnyCPU + Debug + AnyCPU ..\..\..\..\bin\$(Configuration) $(BaseOutputPath)\Examples\Simple\Android 8.0.30703 @@ -14,48 +14,47 @@ Library Properties 512 + True + True + Resources\Resource.designer.cs + Resource + Off + false + v13.0 Properties\AndroidManifest.xml - Resources\Resource.Designer.cs - true - v11.0 + Resources + Assets + true + true + Xamarin.Android.Net.AndroidClientHandler True portable False DEBUG;TRACE;PLATFORM_ANDROID - prompt - 4 - None - None - true - True - - False - False - False - armeabi-v7a;x86_64 - - - Xamarin - False - True + prompt + 4 + None + False - pdbonly - True + True + portable + True TRACE;PLATFORM_ANDROID - prompt - 4 - False - Full - true + prompt + 4 + aab + true + SdkOnly + True - + diff --git a/Examples/Complete/Simple/Android/Properties/AndroidManifest.xml b/Examples/Complete/Simple/Android/Properties/AndroidManifest.xml index a466b7b93..3811d918d 100644 --- a/Examples/Complete/Simple/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/Simple/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/Simple/Blazor/Fusee.Examples.Simple.Blazor.csproj b/Examples/Complete/Simple/Blazor/Fusee.Examples.Simple.Blazor.csproj index 360c08eb6..ce3a8fac3 100644 --- a/Examples/Complete/Simple/Blazor/Fusee.Examples.Simple.Blazor.csproj +++ b/Examples/Complete/Simple/Blazor/Fusee.Examples.Simple.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 service-worker-assets.js @@ -14,9 +14,9 @@ - - - + + + diff --git a/Examples/Complete/Simple/Core/Simple.cs b/Examples/Complete/Simple/Core/Simple.cs index 129ce5edd..0cd7f125b 100644 --- a/Examples/Complete/Simple/Core/Simple.cs +++ b/Examples/Complete/Simple/Core/Simple.cs @@ -41,9 +41,6 @@ private async Task Load() _gui = await FuseeGuiHelper.CreateDefaultGuiAsync(this, CanvasRenderMode.Screen, "FUSEE Simple Example"); - // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); - // Load the rocket model _rocketScene = await AssetStorage.GetAsync("RocketFus.fus"); _camPivotTransform = new Transform(); @@ -72,6 +69,9 @@ private async Task Load() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_rocketScene); _guiRenderer = new SceneRendererForward(_gui); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override async Task InitAsync() @@ -136,10 +136,10 @@ public override void RenderAFrame() //Constantly check for interactive objects. if (!Mouse.Desc.Contains("Android")) - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); if (Touch != null && Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } // Swap buffers: Show the contents of the backbuffer (containing the currently rendered frame) on the front buffer. diff --git a/Examples/Complete/ThreeDFont/Android/Fusee.Examples.ThreeDFont.Android.csproj b/Examples/Complete/ThreeDFont/Android/Fusee.Examples.ThreeDFont.Android.csproj index 56dccb690..42a891732 100644 --- a/Examples/Complete/ThreeDFont/Android/Fusee.Examples.ThreeDFont.Android.csproj +++ b/Examples/Complete/ThreeDFont/Android/Fusee.Examples.ThreeDFont.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.ThreeDFont.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a,x86 Xamarin @@ -42,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/ThreeDFont/Android/Properties/AndroidManifest.xml b/Examples/Complete/ThreeDFont/Android/Properties/AndroidManifest.xml index 5f049f3b5..9ec8cfc81 100644 --- a/Examples/Complete/ThreeDFont/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/ThreeDFont/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/UI/Android/Fusee.Examples.UI.Android.csproj b/Examples/Complete/UI/Android/Fusee.Examples.UI.Android.csproj index 6756b5c85..c696e35b7 100644 --- a/Examples/Complete/UI/Android/Fusee.Examples.UI.Android.csproj +++ b/Examples/Complete/UI/Android/Fusee.Examples.UI.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Examples.UI.Android @@ -16,7 +16,7 @@ 512 Properties\AndroidManifest.xml Resources\Resource.Designer.cs - true + True v11.0 @@ -28,13 +28,12 @@ 4 None None - true + True True False False False - armeabi-v7a;x86_64 Xamarin @@ -42,20 +41,20 @@ True - pdbonly + portable True TRACE;PLATFORM_ANDROID prompt 4 False Full - true + True - + diff --git a/Examples/Complete/UI/Android/Properties/AndroidManifest.xml b/Examples/Complete/UI/Android/Properties/AndroidManifest.xml index a4357ce1d..b72fdf319 100644 --- a/Examples/Complete/UI/Android/Properties/AndroidManifest.xml +++ b/Examples/Complete/UI/Android/Properties/AndroidManifest.xml @@ -1,8 +1,6 @@  - - - + \ No newline at end of file diff --git a/Examples/Complete/UI/Core/UI.cs b/Examples/Complete/UI/Core/UI.cs index b9cfc19f9..89774297c 100644 --- a/Examples/Complete/UI/Core/UI.cs +++ b/Examples/Complete/UI/Core/UI.cs @@ -25,7 +25,7 @@ public class UI : RenderCanvas private const float Damping = 0.8f; private SceneContainer _scene; - private SceneRendererForward _sceneRenderer; + private SceneRendererForward _guiRenderer; private bool _keys; @@ -411,11 +411,11 @@ public override void Init() // Set the scene by creating a scene graph _scene = CreateNineSliceScene(); - // Create the interaction handler - _sih = new SceneInteractionHandler(_scene); - // Wrap a SceneRenderer around the model. - _sceneRenderer = new SceneRendererForward(_scene); + _guiRenderer = new SceneRendererForward(_scene); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_scene, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override void Update() @@ -465,15 +465,15 @@ public override void RenderAFrame() { _fpsText.Text = "FPS: " + Time.FramesPerSecond.ToString("0.00"); - _sceneRenderer.Render(RC); + _guiRenderer.Render(RC); // Constantly check for interactive objects. if (!Input.Mouse.Desc.Contains("Android")) - _sih.CheckForInteractiveObjects(RC, Input.Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Input.Mouse.Position, Width, Height); if (Input.Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Input.Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Input.Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Input.Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } // Swap buffers: Show the contents of the back buffer (containing the currently rendered frame) on the front buffer. diff --git a/Fusee.ruleset b/Fusee.ruleset new file mode 100644 index 000000000..3de447963 --- /dev/null +++ b/Fusee.ruleset @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/Fusee.sln b/Fusee.sln index 0803971ea..7b6e236c8 100644 --- a/Fusee.sln +++ b/Fusee.sln @@ -65,8 +65,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .gitignore = .gitignore BuildNuget.cmd = BuildNuget.cmd .github\workflows\ci.yml = .github\workflows\ci.yml + .github\dependabot.yml = .github\dependabot.yml Directory.Build.props = Directory.Build.props Directory.Build.targets = Directory.Build.targets + Fusee.ruleset = Fusee.ruleset + LICENSE = LICENSE + LICENSE-THIRD-PARTY = LICENSE-THIRD-PARTY NuGet.config = NuGet.config EndProjectSection EndProject @@ -258,8 +262,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.Tools.Build.Blazorpat EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.Tools.Build.Versionupdate", "src\Tools\Build\Versionupdate\Fusee.Tools.Build.Versionupdate.csproj", "{59693479-2561-45B0-BB9E-10337CA9DF5C}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Las", "Las", "{E364F712-2400-4D9F-8F40-8B5870086262}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PointCloudPotree2", "PointCloudPotree2", "{B2C0746D-E5CC-45F1-BD88-3E0D5A2A7191}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.Examples.PointCloudPotree2.Desktop", "Examples\Complete\PointCloudPotree2\Desktop\Fusee.Examples.PointCloudPotree2.Desktop.csproj", "{72974011-C264-4D3D-979A-A4E750C5787A}" @@ -270,14 +272,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.Examples.Deferred.Bla EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.Examples.PointCloudPotree2.Wpf", "Examples\Complete\PointCloudPotree2\Wpf\Fusee.Examples.PointCloudPotree2.Wpf.csproj", "{35839EA8-889B-4C26-94CB-9DB309E9C9F3}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Desktop", "Desktop", "{FA703895-434C-4ECF-8E0D-3145280DA2C6}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Shared", "Shared", "{6F281DDE-0376-40C4-9693-51C7CD2F49D4}" -EndProject -Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "Fusee.PointCloud.Las.Shared", "src\PointCloud\Las\Shared\Fusee.PointCloud.Las.Shared.shproj", "{DCC7DA71-3E2E-476C-8391-1F9651637503}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.PointCloud.Las.Desktop", "src\PointCloud\Las\Desktop\Fusee.PointCloud.Las.Desktop.csproj", "{F28634E7-52B8-4935-B19E-CB8A6844E6F1}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Fusee.PointCloud.Potree", "src\PointCloud\Potree\Fusee.PointCloud.Potree.csproj", "{1B99F3FD-C685-4D72-8EBA-94A1214469B5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "RenderContextOnly", "RenderContextOnly", "{E6E73351-7DED-4B97-B254-A0E903D769C1}" @@ -1595,20 +1589,6 @@ Global {35839EA8-889B-4C26-94CB-9DB309E9C9F3}.Release-Blazor|Any CPU.ActiveCfg = Release|Any CPU {35839EA8-889B-4C26-94CB-9DB309E9C9F3}.Release-Desktop|Any CPU.ActiveCfg = Release|Any CPU {35839EA8-889B-4C26-94CB-9DB309E9C9F3}.Release-NuGet|Any CPU.ActiveCfg = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Debug-Android|Any CPU.ActiveCfg = Debug|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Debug-Blazor|Any CPU.ActiveCfg = Debug|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Debug-Desktop|Any CPU.ActiveCfg = Debug|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Debug-Desktop|Any CPU.Build.0 = Debug|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release|Any CPU.Build.0 = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release-Android|Any CPU.ActiveCfg = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release-Blazor|Any CPU.ActiveCfg = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release-Desktop|Any CPU.ActiveCfg = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release-Desktop|Any CPU.Build.0 = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release-NuGet|Any CPU.ActiveCfg = Release|Any CPU - {F28634E7-52B8-4935-B19E-CB8A6844E6F1}.Release-NuGet|Any CPU.Build.0 = Release|Any CPU {1B99F3FD-C685-4D72-8EBA-94A1214469B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1B99F3FD-C685-4D72-8EBA-94A1214469B5}.Debug|Any CPU.Build.0 = Debug|Any CPU {1B99F3FD-C685-4D72-8EBA-94A1214469B5}.Debug-Android|Any CPU.ActiveCfg = Debug|Any CPU @@ -1854,16 +1834,11 @@ Global {594F2025-8C23-40D2-8100-C9989CA16DC9} = {0C344D5F-BE70-4828-B95A-1D313B894BC3} {8CF98098-64E7-421B-9CBF-67E02EDE6D32} = {594F2025-8C23-40D2-8100-C9989CA16DC9} {59693479-2561-45B0-BB9E-10337CA9DF5C} = {594F2025-8C23-40D2-8100-C9989CA16DC9} - {E364F712-2400-4D9F-8F40-8B5870086262} = {549D61C5-EFE6-4C2B-AD19-BB84A36C5917} {B2C0746D-E5CC-45F1-BD88-3E0D5A2A7191} = {E68628DA-312F-4171-A5ED-08072B5CCA3C} {72974011-C264-4D3D-979A-A4E750C5787A} = {B2C0746D-E5CC-45F1-BD88-3E0D5A2A7191} {76FE409A-DCA8-4714-8E95-4FE189751EE7} = {B2C0746D-E5CC-45F1-BD88-3E0D5A2A7191} {84C77C62-0721-4C6F-9789-88D5F9C98974} = {E7E4600F-D815-49F6-B79C-C456C87F01DC} {35839EA8-889B-4C26-94CB-9DB309E9C9F3} = {B2C0746D-E5CC-45F1-BD88-3E0D5A2A7191} - {FA703895-434C-4ECF-8E0D-3145280DA2C6} = {E364F712-2400-4D9F-8F40-8B5870086262} - {6F281DDE-0376-40C4-9693-51C7CD2F49D4} = {E364F712-2400-4D9F-8F40-8B5870086262} - {DCC7DA71-3E2E-476C-8391-1F9651637503} = {6F281DDE-0376-40C4-9693-51C7CD2F49D4} - {F28634E7-52B8-4935-B19E-CB8A6844E6F1} = {FA703895-434C-4ECF-8E0D-3145280DA2C6} {1B99F3FD-C685-4D72-8EBA-94A1214469B5} = {549D61C5-EFE6-4C2B-AD19-BB84A36C5917} {E6E73351-7DED-4B97-B254-A0E903D769C1} = {E68628DA-312F-4171-A5ED-08072B5CCA3C} {81209326-E0D1-48C7-878F-F3DEA131BFB1} = {E6E73351-7DED-4B97-B254-A0E903D769C1} @@ -1885,8 +1860,6 @@ Global src\Engine\Imp\Graphics\SharedAll\Fusee.Engine.Imp.Graphics.SharedAll.projitems*{7638e21e-193f-4ac7-8f01-b595fe8b3caa}*SharedItemsImports = 13 src\Engine\Imp\Graphics\SharedAll\Fusee.Engine.Imp.Graphics.SharedAll.projitems*{b3ce4f39-fcc4-4388-8130-9d0b9d65d034}*SharedItemsImports = 5 src\Engine\Imp\Graphics\Shared\Fusee.Engine.Imp.Graphics.Shared.projitems*{b3ce4f39-fcc4-4388-8130-9d0b9d65d034}*SharedItemsImports = 5 - src\PointCloud\Las\Shared\Fusee.PointCloud.Las.Shared.projitems*{dcc7da71-3e2e-476c-8391-1f9651637503}*SharedItemsImports = 13 - src\PointCloud\Las\Shared\Fusee.PointCloud.Las.Shared.projitems*{f28634e7-52b8-4935-b19e-cb8a6844e6f1}*SharedItemsImports = 5 src\Engine\Imp\Graphics\SharedAll\Fusee.Engine.Imp.Graphics.SharedAll.projitems*{f7ad2bb5-d2b0-4697-bddb-4cc26152a6b6}*SharedItemsImports = 5 src\Engine\Imp\Graphics\Shared\Fusee.Engine.Imp.Graphics.Shared.projitems*{f7ad2bb5-d2b0-4697-bddb-4cc26152a6b6}*SharedItemsImports = 5 EndGlobalSection diff --git a/LICENSE-THIRD-PARTY b/LICENSE-THIRD-PARTY index 89602ef2a..daf2af8e2 100644 --- a/LICENSE-THIRD-PARTY +++ b/LICENSE-THIRD-PARTY @@ -8,6 +8,7 @@ components are released under: - Json.NET - https://github.com/JamesNK/Newtonsoft.Json - MIT - LASlib - https://github.com/LAStools/LAStools/tree/master/LASlib - LGPL - Lato - https://fonts.google.com/specimen/Lato - SIL Open Font License 1.1 +- Math.NET - https://github.com/mathnet/mathnet-numerics - MIT - OpenTK - http://www.opentk.com/ - MIT - Potree - https://github.com/potree/potree - BSD 2-Clause "Simplified" License - Protobuf - https://code.google.com/p/protobuf/ - BSD New/Simplified diff --git a/ext/protobuf/Python/google/protobuf/__init__.py b/ext/protobuf/Python/google/protobuf/__init__.py index e7b197e5d..a301349e3 100644 --- a/ext/protobuf/Python/google/protobuf/__init__.py +++ b/ext/protobuf/Python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.21.9' +__version__ = '4.22.0' diff --git a/ext/protobuf/Python/google/protobuf/any_pb2.py b/ext/protobuf/Python/google/protobuf/any_pb2.py index a3aea5eae..c43ebb48f 100644 --- a/ext/protobuf/Python/google/protobuf/any_pb2.py +++ b/ext/protobuf/Python/google/protobuf/any_pb2.py @@ -15,12 +15,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05valueBv\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01Z,google.golang.org/protobuf/types/known/anypb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.any_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.any_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\010AnyProtoP\001Z,google.golang.org/protobuf/types/known/anypb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _ANY._serialized_start=46 - _ANY._serialized_end=100 + _globals['_ANY']._serialized_start=46 + _globals['_ANY']._serialized_end=100 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/api_pb2.py b/ext/protobuf/Python/google/protobuf/api_pb2.py index 8c3cec53a..70233ef40 100644 --- a/ext/protobuf/Python/google/protobuf/api_pb2.py +++ b/ext/protobuf/Python/google/protobuf/api_pb2.py @@ -17,16 +17,17 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19google/protobuf/api.proto\x12\x0fgoogle.protobuf\x1a$google/protobuf/source_context.proto\x1a\x1agoogle/protobuf/type.proto\"\xc1\x02\n\x03\x41pi\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x31\n\x07methods\x18\x02 \x03(\x0b\x32\x17.google.protobuf.MethodR\x07methods\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x18\n\x07version\x18\x04 \x01(\tR\x07version\x12\x45\n\x0esource_context\x18\x05 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12.\n\x06mixins\x18\x06 \x03(\x0b\x32\x16.google.protobuf.MixinR\x06mixins\x12/\n\x06syntax\x18\x07 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\"\xb2\x02\n\x06Method\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12(\n\x10request_type_url\x18\x02 \x01(\tR\x0erequestTypeUrl\x12+\n\x11request_streaming\x18\x03 \x01(\x08R\x10requestStreaming\x12*\n\x11response_type_url\x18\x04 \x01(\tR\x0fresponseTypeUrl\x12-\n\x12response_streaming\x18\x05 \x01(\x08R\x11responseStreaming\x12\x31\n\x07options\x18\x06 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12/\n\x06syntax\x18\x07 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\"/\n\x05Mixin\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x12\n\x04root\x18\x02 \x01(\tR\x04rootBv\n\x13\x63om.google.protobufB\x08\x41piProtoP\x01Z,google.golang.org/protobuf/types/known/apipb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.api_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.api_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\010ApiProtoP\001Z,google.golang.org/protobuf/types/known/apipb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _API._serialized_start=113 - _API._serialized_end=434 - _METHOD._serialized_start=437 - _METHOD._serialized_end=743 - _MIXIN._serialized_start=745 - _MIXIN._serialized_end=792 + _globals['_API']._serialized_start=113 + _globals['_API']._serialized_end=434 + _globals['_METHOD']._serialized_start=437 + _globals['_METHOD']._serialized_end=743 + _globals['_MIXIN']._serialized_start=745 + _globals['_MIXIN']._serialized_end=792 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/compiler/plugin_pb2.py b/ext/protobuf/Python/google/protobuf/compiler/plugin_pb2.py index 6684ec269..7dc1e3861 100644 --- a/ext/protobuf/Python/google/protobuf/compiler/plugin_pb2.py +++ b/ext/protobuf/Python/google/protobuf/compiler/plugin_pb2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/compiler/plugin.proto +# source: src/google/protobuf/compiler/plugin.proto """Generated protocol buffer code.""" from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor @@ -14,22 +14,23 @@ from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%google/protobuf/compiler/plugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"c\n\x07Version\x12\x14\n\x05major\x18\x01 \x01(\x05R\x05major\x12\x14\n\x05minor\x18\x02 \x01(\x05R\x05minor\x12\x14\n\x05patch\x18\x03 \x01(\x05R\x05patch\x12\x16\n\x06suffix\x18\x04 \x01(\tR\x06suffix\"\xf1\x01\n\x14\x43odeGeneratorRequest\x12(\n\x10\x66ile_to_generate\x18\x01 \x03(\tR\x0e\x66ileToGenerate\x12\x1c\n\tparameter\x18\x02 \x01(\tR\tparameter\x12\x43\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\tprotoFile\x12L\n\x10\x63ompiler_version\x18\x03 \x01(\x0b\x32!.google.protobuf.compiler.VersionR\x0f\x63ompilerVersion\"\x94\x03\n\x15\x43odeGeneratorResponse\x12\x14\n\x05\x65rror\x18\x01 \x01(\tR\x05\x65rror\x12-\n\x12supported_features\x18\x02 \x01(\x04R\x11supportedFeatures\x12H\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.FileR\x04\x66ile\x1a\xb1\x01\n\x04\x46ile\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\'\n\x0finsertion_point\x18\x02 \x01(\tR\x0einsertionPoint\x12\x18\n\x07\x63ontent\x18\x0f \x01(\tR\x07\x63ontent\x12R\n\x13generated_code_info\x18\x10 \x01(\x0b\x32\".google.protobuf.GeneratedCodeInfoR\x11generatedCodeInfo\"8\n\x07\x46\x65\x61ture\x12\x10\n\x0c\x46\x45\x41TURE_NONE\x10\x00\x12\x1b\n\x17\x46\x45\x41TURE_PROTO3_OPTIONAL\x10\x01\x42W\n\x1c\x63om.google.protobuf.compilerB\x0cPluginProtosZ)google.golang.org/protobuf/types/pluginpb') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)src/google/protobuf/compiler/plugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"c\n\x07Version\x12\x14\n\x05major\x18\x01 \x01(\x05R\x05major\x12\x14\n\x05minor\x18\x02 \x01(\x05R\x05minor\x12\x14\n\x05patch\x18\x03 \x01(\x05R\x05patch\x12\x16\n\x06suffix\x18\x04 \x01(\tR\x06suffix\"\xf1\x01\n\x14\x43odeGeneratorRequest\x12(\n\x10\x66ile_to_generate\x18\x01 \x03(\tR\x0e\x66ileToGenerate\x12\x1c\n\tparameter\x18\x02 \x01(\tR\tparameter\x12\x43\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\tprotoFile\x12L\n\x10\x63ompiler_version\x18\x03 \x01(\x0b\x32!.google.protobuf.compiler.VersionR\x0f\x63ompilerVersion\"\x94\x03\n\x15\x43odeGeneratorResponse\x12\x14\n\x05\x65rror\x18\x01 \x01(\tR\x05\x65rror\x12-\n\x12supported_features\x18\x02 \x01(\x04R\x11supportedFeatures\x12H\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.FileR\x04\x66ile\x1a\xb1\x01\n\x04\x46ile\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\'\n\x0finsertion_point\x18\x02 \x01(\tR\x0einsertionPoint\x12\x18\n\x07\x63ontent\x18\x0f \x01(\tR\x07\x63ontent\x12R\n\x13generated_code_info\x18\x10 \x01(\x0b\x32\".google.protobuf.GeneratedCodeInfoR\x11generatedCodeInfo\"8\n\x07\x46\x65\x61ture\x12\x10\n\x0c\x46\x45\x41TURE_NONE\x10\x00\x12\x1b\n\x17\x46\x45\x41TURE_PROTO3_OPTIONAL\x10\x01\x42r\n\x1c\x63om.google.protobuf.compilerB\x0cPluginProtosZ)google.golang.org/protobuf/types/pluginpb\xaa\x02\x18Google.Protobuf.Compiler') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.compiler.plugin_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'src.google.protobuf.compiler.plugin_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\034com.google.protobuf.compilerB\014PluginProtosZ)google.golang.org/protobuf/types/pluginpb' - _VERSION._serialized_start=101 - _VERSION._serialized_end=200 - _CODEGENERATORREQUEST._serialized_start=203 - _CODEGENERATORREQUEST._serialized_end=444 - _CODEGENERATORRESPONSE._serialized_start=447 - _CODEGENERATORRESPONSE._serialized_end=851 - _CODEGENERATORRESPONSE_FILE._serialized_start=616 - _CODEGENERATORRESPONSE_FILE._serialized_end=793 - _CODEGENERATORRESPONSE_FEATURE._serialized_start=795 - _CODEGENERATORRESPONSE_FEATURE._serialized_end=851 + DESCRIPTOR._serialized_options = b'\n\034com.google.protobuf.compilerB\014PluginProtosZ)google.golang.org/protobuf/types/pluginpb\252\002\030Google.Protobuf.Compiler' + _globals['_VERSION']._serialized_start=105 + _globals['_VERSION']._serialized_end=204 + _globals['_CODEGENERATORREQUEST']._serialized_start=207 + _globals['_CODEGENERATORREQUEST']._serialized_end=448 + _globals['_CODEGENERATORRESPONSE']._serialized_start=451 + _globals['_CODEGENERATORRESPONSE']._serialized_end=855 + _globals['_CODEGENERATORRESPONSE_FILE']._serialized_start=620 + _globals['_CODEGENERATORRESPONSE_FILE']._serialized_end=797 + _globals['_CODEGENERATORRESPONSE_FEATURE']._serialized_start=799 + _globals['_CODEGENERATORRESPONSE_FEATURE']._serialized_end=855 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/descriptor.py b/ext/protobuf/Python/google/protobuf/descriptor.py index f5a0caa6b..fcb87cab5 100644 --- a/ext/protobuf/Python/google/protobuf/descriptor.py +++ b/ext/protobuf/Python/google/protobuf/descriptor.py @@ -66,6 +66,7 @@ class TypeTransformationError(Error): # and make it return True when the descriptor is an instance of the extension # type written in C++. class DescriptorMetaclass(type): + def __instancecheck__(cls, obj): if super(DescriptorMetaclass, cls).__instancecheck__(obj): return True @@ -633,13 +634,29 @@ def has_presence(self): if (self.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE or self.containing_oneof): return True - if hasattr(self.file, 'syntax'): - return self.file.syntax == 'proto2' - if hasattr(self.message_type, 'syntax'): - return self.message_type.syntax == 'proto2' - raise RuntimeError( - 'has_presence is not ready to use because field %s is not' - ' linked with message type nor file' % self.full_name) + # self.containing_type is used here instead of self.file for legacy + # compatibility. FieldDescriptor.file was added in cl/153110619 + # Some old/generated code didn't link file to FieldDescriptor. + # TODO(jieluo): remove syntax usage b/240619313 + return self.containing_type.syntax == 'proto2' + + @property + def is_packed(self): + """Returns if the field is packed.""" + if self.label != FieldDescriptor.LABEL_REPEATED: + return False + field_type = self.type + if (field_type == FieldDescriptor.TYPE_STRING or + field_type == FieldDescriptor.TYPE_GROUP or + field_type == FieldDescriptor.TYPE_MESSAGE or + field_type == FieldDescriptor.TYPE_BYTES): + return False + if self.containing_type.syntax == 'proto2': + return self.has_options and self.GetOptions().packed + else: + return (not self.has_options or + not self.GetOptions().HasField('packed') or + self.GetOptions().packed) @staticmethod def ProtoTypeToCppProtoType(proto_type): @@ -720,6 +737,30 @@ def __init__(self, name, full_name, filename, values, # Values are reversed to ensure that the first alias is retained. self.values_by_number = dict((v.number, v) for v in reversed(values)) + @property + def is_closed(self): + """Returns true whether this is a "closed" enum. + + This means that it: + - Has a fixed set of values, rather than being equivalent to an int32. + - Encountering values not in this set causes them to be treated as unknown + fields. + - The first value (i.e., the default) may be nonzero. + + WARNING: Some runtimes currently have a quirk where non-closed enums are + treated as closed when used as the type of fields defined in a + `syntax = proto2;` file. This quirk is not present in all runtimes; as of + writing, we know that: + + - C++, Java, and C++-based Python share this quirk. + - UPB and UPB-based Python do not. + - PHP and Ruby treat all enums as open regardless of declaration. + + Care should be taken when using this function to respect the target + runtime's enum handling quirks. + """ + return self.file.syntax == 'proto2' + def CopyToProto(self, proto): """Copies this to a descriptor_pb2.EnumDescriptorProto. @@ -873,11 +914,14 @@ def FindMethodByName(self, name): Args: name (str): Name of the method. + Returns: - MethodDescriptor or None: the descriptor for the requested method, if - found. + MethodDescriptor: The descriptor for the requested method. + + Raises: + KeyError: if the method cannot be found in the service. """ - return self.methods_by_name.get(name, None) + return self.methods_by_name[name] def CopyToProto(self, proto): """Copies this to a descriptor_pb2.ServiceDescriptorProto. @@ -1018,13 +1062,7 @@ def __new__(cls, name, package, options=None, # FileDescriptor() is called from various places, not only from generated # files, to register dynamic proto files and messages. # pylint: disable=g-explicit-bool-comparison - if serialized_pb == b'': - # Cpp generated code must be linked in if serialized_pb is '' - try: - return _message.default_pool.FindFileByName(name) - except KeyError: - raise RuntimeError('Please link in cpp generated lib for %s' % (name)) - elif serialized_pb: + if serialized_pb: return _message.default_pool.AddSerializedFile(serialized_pb) else: return super(FileDescriptor, cls).__new__(cls) diff --git a/ext/protobuf/Python/google/protobuf/descriptor_pb2.py b/ext/protobuf/Python/google/protobuf/descriptor_pb2.py index 2eaeb7d13..84781d251 100644 --- a/ext/protobuf/Python/google/protobuf/descriptor_pb2.py +++ b/ext/protobuf/Python/google/protobuf/descriptor_pb2.py @@ -20,11 +20,12 @@ syntax='proto2', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"M\n\x11\x46ileDescriptorSet\x12\x38\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x04\x66ile\"\xe4\x04\n\x13\x46ileDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n\x07package\x18\x02 \x01(\tR\x07package\x12\x1e\n\ndependency\x18\x03 \x03(\tR\ndependency\x12+\n\x11public_dependency\x18\n \x03(\x05R\x10publicDependency\x12\'\n\x0fweak_dependency\x18\x0b \x03(\x05R\x0eweakDependency\x12\x43\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\x0bmessageType\x12\x41\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12\x41\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProtoR\x07service\x12\x43\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x36\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptionsR\x07options\x12I\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfoR\x0esourceCodeInfo\x12\x16\n\x06syntax\x18\x0c \x01(\tR\x06syntax\"\xb9\x06\n\x0f\x44\x65scriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12;\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\x05\x66ield\x12\x43\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x41\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\nnestedType\x12\x41\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12X\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRangeR\x0e\x65xtensionRange\x12\x44\n\noneof_decl\x18\x08 \x03(\x0b\x32%.google.protobuf.OneofDescriptorProtoR\toneofDecl\x12\x39\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptionsR\x07options\x12U\n\x0ereserved_range\x18\t \x03(\x0b\x32..google.protobuf.DescriptorProto.ReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\n \x03(\tR\x0creservedName\x1az\n\x0e\x45xtensionRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\x12@\n\x07options\x18\x03 \x01(\x0b\x32&.google.protobuf.ExtensionRangeOptionsR\x07options\x1a\x37\n\rReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"|\n\x15\x45xtensionRangeOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc1\x06\n\x14\x46ieldDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x41\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.LabelR\x05label\x12>\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.TypeR\x04type\x12\x1b\n\ttype_name\x18\x06 \x01(\tR\x08typeName\x12\x1a\n\x08\x65xtendee\x18\x02 \x01(\tR\x08\x65xtendee\x12#\n\rdefault_value\x18\x07 \x01(\tR\x0c\x64\x65\x66\x61ultValue\x12\x1f\n\x0boneof_index\x18\t \x01(\x05R\noneofIndex\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12\x37\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptionsR\x07options\x12\'\n\x0fproto3_optional\x18\x11 \x01(\x08R\x0eproto3Optional\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\x12\x12\n\x0eLABEL_REPEATED\x10\x03\"c\n\x14OneofDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\x07options\x18\x02 \x01(\x0b\x32\x1d.google.protobuf.OneofOptionsR\x07options\"\xe3\x02\n\x13\x45numDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12?\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProtoR\x05value\x12\x36\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptionsR\x07options\x12]\n\x0ereserved_range\x18\x04 \x03(\x0b\x32\x36.google.protobuf.EnumDescriptorProto.EnumReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\x05 \x03(\tR\x0creservedName\x1a;\n\x11\x45numReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\x83\x01\n\x18\x45numValueDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptionsR\x07options\"\xa7\x01\n\x16ServiceDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12>\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProtoR\x06method\x12\x39\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptionsR\x07options\"\x89\x02\n\x15MethodDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n\ninput_type\x18\x02 \x01(\tR\tinputType\x12\x1f\n\x0boutput_type\x18\x03 \x01(\tR\noutputType\x12\x38\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptionsR\x07options\x12\x30\n\x10\x63lient_streaming\x18\x05 \x01(\x08:\x05\x66\x61lseR\x0f\x63lientStreaming\x12\x30\n\x10server_streaming\x18\x06 \x01(\x08:\x05\x66\x61lseR\x0fserverStreaming\"\x91\t\n\x0b\x46ileOptions\x12!\n\x0cjava_package\x18\x01 \x01(\tR\x0bjavaPackage\x12\x30\n\x14java_outer_classname\x18\x08 \x01(\tR\x12javaOuterClassname\x12\x35\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lseR\x11javaMultipleFiles\x12\x44\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08\x42\x02\x18\x01R\x19javaGenerateEqualsAndHash\x12:\n\x16java_string_check_utf8\x18\x1b \x01(\x08:\x05\x66\x61lseR\x13javaStringCheckUtf8\x12S\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEEDR\x0boptimizeFor\x12\x1d\n\ngo_package\x18\x0b \x01(\tR\tgoPackage\x12\x35\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lseR\x11\x63\x63GenericServices\x12\x39\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lseR\x13javaGenericServices\x12\x35\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lseR\x11pyGenericServices\x12\x37\n\x14php_generic_services\x18* \x01(\x08:\x05\x66\x61lseR\x12phpGenericServices\x12%\n\ndeprecated\x18\x17 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12.\n\x10\x63\x63_enable_arenas\x18\x1f \x01(\x08:\x04trueR\x0e\x63\x63\x45nableArenas\x12*\n\x11objc_class_prefix\x18$ \x01(\tR\x0fobjcClassPrefix\x12)\n\x10\x63sharp_namespace\x18% \x01(\tR\x0f\x63sharpNamespace\x12!\n\x0cswift_prefix\x18\' \x01(\tR\x0bswiftPrefix\x12(\n\x10php_class_prefix\x18( \x01(\tR\x0ephpClassPrefix\x12#\n\rphp_namespace\x18) \x01(\tR\x0cphpNamespace\x12\x34\n\x16php_metadata_namespace\x18, \x01(\tR\x14phpMetadataNamespace\x12!\n\x0cruby_package\x18- \x01(\tR\x0brubyPackage\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08&\x10\'\"\xe3\x02\n\x0eMessageOptions\x12<\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lseR\x14messageSetWireFormat\x12L\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lseR\x1cnoStandardDescriptorAccessor\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1b\n\tmap_entry\x18\x07 \x01(\x08R\x08mapEntry\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\tJ\x04\x08\t\x10\n\"\x92\x04\n\x0c\x46ieldOptions\x12\x41\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRINGR\x05\x63type\x12\x16\n\x06packed\x18\x02 \x01(\x08R\x06packed\x12G\n\x06jstype\x18\x06 \x01(\x0e\x32$.google.protobuf.FieldOptions.JSType:\tJS_NORMALR\x06jstype\x12\x19\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lseR\x04lazy\x12.\n\x0funverified_lazy\x18\x0f \x01(\x08:\x05\x66\x61lseR\x0eunverifiedLazy\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x19\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lseR\x04weak\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02\"5\n\x06JSType\x12\r\n\tJS_NORMAL\x10\x00\x12\r\n\tJS_STRING\x10\x01\x12\r\n\tJS_NUMBER\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05\"s\n\x0cOneofOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc0\x01\n\x0b\x45numOptions\x12\x1f\n\x0b\x61llow_alias\x18\x02 \x01(\x08R\nallowAlias\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x05\x10\x06\"\x9e\x01\n\x10\x45numValueOptions\x12%\n\ndeprecated\x18\x01 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9c\x01\n\x0eServiceOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xe0\x02\n\rMethodOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12q\n\x11idempotency_level\x18\" \x01(\x0e\x32/.google.protobuf.MethodOptions.IdempotencyLevel:\x13IDEMPOTENCY_UNKNOWNR\x10idempotencyLevel\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"P\n\x10IdempotencyLevel\x12\x17\n\x13IDEMPOTENCY_UNKNOWN\x10\x00\x12\x13\n\x0fNO_SIDE_EFFECTS\x10\x01\x12\x0e\n\nIDEMPOTENT\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9a\x03\n\x13UninterpretedOption\x12\x41\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePartR\x04name\x12)\n\x10identifier_value\x18\x03 \x01(\tR\x0fidentifierValue\x12,\n\x12positive_int_value\x18\x04 \x01(\x04R\x10positiveIntValue\x12,\n\x12negative_int_value\x18\x05 \x01(\x03R\x10negativeIntValue\x12!\n\x0c\x64ouble_value\x18\x06 \x01(\x01R\x0b\x64oubleValue\x12!\n\x0cstring_value\x18\x07 \x01(\x0cR\x0bstringValue\x12\'\n\x0f\x61ggregate_value\x18\x08 \x01(\tR\x0e\x61ggregateValue\x1aJ\n\x08NamePart\x12\x1b\n\tname_part\x18\x01 \x02(\tR\x08namePart\x12!\n\x0cis_extension\x18\x02 \x02(\x08R\x0bisExtension\"\xa7\x02\n\x0eSourceCodeInfo\x12\x44\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.LocationR\x08location\x1a\xce\x01\n\x08Location\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x16\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01R\x04span\x12)\n\x10leading_comments\x18\x03 \x01(\tR\x0fleadingComments\x12+\n\x11trailing_comments\x18\x04 \x01(\tR\x10trailingComments\x12:\n\x19leading_detached_comments\x18\x06 \x03(\tR\x17leadingDetachedComments\"\xd1\x01\n\x11GeneratedCodeInfo\x12M\n\nannotation\x18\x01 \x03(\x0b\x32-.google.protobuf.GeneratedCodeInfo.AnnotationR\nannotation\x1am\n\nAnnotation\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x1f\n\x0bsource_file\x18\x02 \x01(\tR\nsourceFile\x12\x14\n\x05\x62\x65gin\x18\x03 \x01(\x05R\x05\x62\x65gin\x12\x10\n\x03\x65nd\x18\x04 \x01(\x05R\x03\x65ndB~\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01Z-google.golang.org/protobuf/types/descriptorpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1aGoogle.Protobuf.Reflection' + serialized_pb=b'\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"M\n\x11\x46ileDescriptorSet\x12\x38\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x04\x66ile\"\xfe\x04\n\x13\x46ileDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n\x07package\x18\x02 \x01(\tR\x07package\x12\x1e\n\ndependency\x18\x03 \x03(\tR\ndependency\x12+\n\x11public_dependency\x18\n \x03(\x05R\x10publicDependency\x12\'\n\x0fweak_dependency\x18\x0b \x03(\x05R\x0eweakDependency\x12\x43\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\x0bmessageType\x12\x41\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12\x41\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProtoR\x07service\x12\x43\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x36\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptionsR\x07options\x12I\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfoR\x0esourceCodeInfo\x12\x16\n\x06syntax\x18\x0c \x01(\tR\x06syntax\x12\x18\n\x07\x65\x64ition\x18\r \x01(\tR\x07\x65\x64ition\"\xb9\x06\n\x0f\x44\x65scriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12;\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\x05\x66ield\x12\x43\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x41\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\nnestedType\x12\x41\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12X\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRangeR\x0e\x65xtensionRange\x12\x44\n\noneof_decl\x18\x08 \x03(\x0b\x32%.google.protobuf.OneofDescriptorProtoR\toneofDecl\x12\x39\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptionsR\x07options\x12U\n\x0ereserved_range\x18\t \x03(\x0b\x32..google.protobuf.DescriptorProto.ReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\n \x03(\tR\x0creservedName\x1az\n\x0e\x45xtensionRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\x12@\n\x07options\x18\x03 \x01(\x0b\x32&.google.protobuf.ExtensionRangeOptionsR\x07options\x1a\x37\n\rReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"|\n\x15\x45xtensionRangeOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc1\x06\n\x14\x46ieldDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x41\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.LabelR\x05label\x12>\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.TypeR\x04type\x12\x1b\n\ttype_name\x18\x06 \x01(\tR\x08typeName\x12\x1a\n\x08\x65xtendee\x18\x02 \x01(\tR\x08\x65xtendee\x12#\n\rdefault_value\x18\x07 \x01(\tR\x0c\x64\x65\x66\x61ultValue\x12\x1f\n\x0boneof_index\x18\t \x01(\x05R\noneofIndex\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12\x37\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptionsR\x07options\x12\'\n\x0fproto3_optional\x18\x11 \x01(\x08R\x0eproto3Optional\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\x12\x12\n\x0eLABEL_REPEATED\x10\x03\"c\n\x14OneofDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\x07options\x18\x02 \x01(\x0b\x32\x1d.google.protobuf.OneofOptionsR\x07options\"\xe3\x02\n\x13\x45numDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12?\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProtoR\x05value\x12\x36\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptionsR\x07options\x12]\n\x0ereserved_range\x18\x04 \x03(\x0b\x32\x36.google.protobuf.EnumDescriptorProto.EnumReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\x05 \x03(\tR\x0creservedName\x1a;\n\x11\x45numReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\x83\x01\n\x18\x45numValueDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptionsR\x07options\"\xa7\x01\n\x16ServiceDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12>\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProtoR\x06method\x12\x39\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptionsR\x07options\"\x89\x02\n\x15MethodDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n\ninput_type\x18\x02 \x01(\tR\tinputType\x12\x1f\n\x0boutput_type\x18\x03 \x01(\tR\noutputType\x12\x38\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptionsR\x07options\x12\x30\n\x10\x63lient_streaming\x18\x05 \x01(\x08:\x05\x66\x61lseR\x0f\x63lientStreaming\x12\x30\n\x10server_streaming\x18\x06 \x01(\x08:\x05\x66\x61lseR\x0fserverStreaming\"\x91\t\n\x0b\x46ileOptions\x12!\n\x0cjava_package\x18\x01 \x01(\tR\x0bjavaPackage\x12\x30\n\x14java_outer_classname\x18\x08 \x01(\tR\x12javaOuterClassname\x12\x35\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lseR\x11javaMultipleFiles\x12\x44\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08\x42\x02\x18\x01R\x19javaGenerateEqualsAndHash\x12:\n\x16java_string_check_utf8\x18\x1b \x01(\x08:\x05\x66\x61lseR\x13javaStringCheckUtf8\x12S\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEEDR\x0boptimizeFor\x12\x1d\n\ngo_package\x18\x0b \x01(\tR\tgoPackage\x12\x35\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lseR\x11\x63\x63GenericServices\x12\x39\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lseR\x13javaGenericServices\x12\x35\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lseR\x11pyGenericServices\x12\x37\n\x14php_generic_services\x18* \x01(\x08:\x05\x66\x61lseR\x12phpGenericServices\x12%\n\ndeprecated\x18\x17 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12.\n\x10\x63\x63_enable_arenas\x18\x1f \x01(\x08:\x04trueR\x0e\x63\x63\x45nableArenas\x12*\n\x11objc_class_prefix\x18$ \x01(\tR\x0fobjcClassPrefix\x12)\n\x10\x63sharp_namespace\x18% \x01(\tR\x0f\x63sharpNamespace\x12!\n\x0cswift_prefix\x18\' \x01(\tR\x0bswiftPrefix\x12(\n\x10php_class_prefix\x18( \x01(\tR\x0ephpClassPrefix\x12#\n\rphp_namespace\x18) \x01(\tR\x0cphpNamespace\x12\x34\n\x16php_metadata_namespace\x18, \x01(\tR\x14phpMetadataNamespace\x12!\n\x0cruby_package\x18- \x01(\tR\x0brubyPackage\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08&\x10\'\"\xbb\x03\n\x0eMessageOptions\x12<\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lseR\x14messageSetWireFormat\x12L\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lseR\x1cnoStandardDescriptorAccessor\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1b\n\tmap_entry\x18\x07 \x01(\x08R\x08mapEntry\x12V\n&deprecated_legacy_json_field_conflicts\x18\x0b \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\tJ\x04\x08\t\x10\n\"\xb7\x08\n\x0c\x46ieldOptions\x12\x41\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRINGR\x05\x63type\x12\x16\n\x06packed\x18\x02 \x01(\x08R\x06packed\x12G\n\x06jstype\x18\x06 \x01(\x0e\x32$.google.protobuf.FieldOptions.JSType:\tJS_NORMALR\x06jstype\x12\x19\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lseR\x04lazy\x12.\n\x0funverified_lazy\x18\x0f \x01(\x08:\x05\x66\x61lseR\x0eunverifiedLazy\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x19\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lseR\x04weak\x12(\n\x0c\x64\x65\x62ug_redact\x18\x10 \x01(\x08:\x05\x66\x61lseR\x0b\x64\x65\x62ugRedact\x12K\n\tretention\x18\x11 \x01(\x0e\x32-.google.protobuf.FieldOptions.OptionRetentionR\tretention\x12\x46\n\x06target\x18\x12 \x01(\x0e\x32..google.protobuf.FieldOptions.OptionTargetTypeR\x06target\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02\"5\n\x06JSType\x12\r\n\tJS_NORMAL\x10\x00\x12\r\n\tJS_STRING\x10\x01\x12\r\n\tJS_NUMBER\x10\x02\"U\n\x0fOptionRetention\x12\x15\n\x11RETENTION_UNKNOWN\x10\x00\x12\x15\n\x11RETENTION_RUNTIME\x10\x01\x12\x14\n\x10RETENTION_SOURCE\x10\x02\"\x8c\x02\n\x10OptionTargetType\x12\x17\n\x13TARGET_TYPE_UNKNOWN\x10\x00\x12\x14\n\x10TARGET_TYPE_FILE\x10\x01\x12\x1f\n\x1bTARGET_TYPE_EXTENSION_RANGE\x10\x02\x12\x17\n\x13TARGET_TYPE_MESSAGE\x10\x03\x12\x15\n\x11TARGET_TYPE_FIELD\x10\x04\x12\x15\n\x11TARGET_TYPE_ONEOF\x10\x05\x12\x14\n\x10TARGET_TYPE_ENUM\x10\x06\x12\x1a\n\x16TARGET_TYPE_ENUM_ENTRY\x10\x07\x12\x17\n\x13TARGET_TYPE_SERVICE\x10\x08\x12\x16\n\x12TARGET_TYPE_METHOD\x10\t*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05\"s\n\x0cOneofOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x98\x02\n\x0b\x45numOptions\x12\x1f\n\x0b\x61llow_alias\x18\x02 \x01(\x08R\nallowAlias\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12V\n&deprecated_legacy_json_field_conflicts\x18\x06 \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x05\x10\x06\"\x9e\x01\n\x10\x45numValueOptions\x12%\n\ndeprecated\x18\x01 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9c\x01\n\x0eServiceOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xe0\x02\n\rMethodOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12q\n\x11idempotency_level\x18\" \x01(\x0e\x32/.google.protobuf.MethodOptions.IdempotencyLevel:\x13IDEMPOTENCY_UNKNOWNR\x10idempotencyLevel\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"P\n\x10IdempotencyLevel\x12\x17\n\x13IDEMPOTENCY_UNKNOWN\x10\x00\x12\x13\n\x0fNO_SIDE_EFFECTS\x10\x01\x12\x0e\n\nIDEMPOTENT\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9a\x03\n\x13UninterpretedOption\x12\x41\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePartR\x04name\x12)\n\x10identifier_value\x18\x03 \x01(\tR\x0fidentifierValue\x12,\n\x12positive_int_value\x18\x04 \x01(\x04R\x10positiveIntValue\x12,\n\x12negative_int_value\x18\x05 \x01(\x03R\x10negativeIntValue\x12!\n\x0c\x64ouble_value\x18\x06 \x01(\x01R\x0b\x64oubleValue\x12!\n\x0cstring_value\x18\x07 \x01(\x0cR\x0bstringValue\x12\'\n\x0f\x61ggregate_value\x18\x08 \x01(\tR\x0e\x61ggregateValue\x1aJ\n\x08NamePart\x12\x1b\n\tname_part\x18\x01 \x02(\tR\x08namePart\x12!\n\x0cis_extension\x18\x02 \x02(\x08R\x0bisExtension\"\xa7\x02\n\x0eSourceCodeInfo\x12\x44\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.LocationR\x08location\x1a\xce\x01\n\x08Location\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x16\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01R\x04span\x12)\n\x10leading_comments\x18\x03 \x01(\tR\x0fleadingComments\x12+\n\x11trailing_comments\x18\x04 \x01(\tR\x10trailingComments\x12:\n\x19leading_detached_comments\x18\x06 \x03(\tR\x17leadingDetachedComments\"\xd0\x02\n\x11GeneratedCodeInfo\x12M\n\nannotation\x18\x01 \x03(\x0b\x32-.google.protobuf.GeneratedCodeInfo.AnnotationR\nannotation\x1a\xeb\x01\n\nAnnotation\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x1f\n\x0bsource_file\x18\x02 \x01(\tR\nsourceFile\x12\x14\n\x05\x62\x65gin\x18\x03 \x01(\x05R\x05\x62\x65gin\x12\x10\n\x03\x65nd\x18\x04 \x01(\x05R\x03\x65nd\x12R\n\x08semantic\x18\x05 \x01(\x0e\x32\x36.google.protobuf.GeneratedCodeInfo.Annotation.SemanticR\x08semantic\"(\n\x08Semantic\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03SET\x10\x01\x12\t\n\x05\x41LIAS\x10\x02\x42~\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01Z-google.golang.org/protobuf/types/descriptorpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1aGoogle.Protobuf.Reflection' ) else: - DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"M\n\x11\x46ileDescriptorSet\x12\x38\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x04\x66ile\"\xe4\x04\n\x13\x46ileDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n\x07package\x18\x02 \x01(\tR\x07package\x12\x1e\n\ndependency\x18\x03 \x03(\tR\ndependency\x12+\n\x11public_dependency\x18\n \x03(\x05R\x10publicDependency\x12\'\n\x0fweak_dependency\x18\x0b \x03(\x05R\x0eweakDependency\x12\x43\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\x0bmessageType\x12\x41\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12\x41\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProtoR\x07service\x12\x43\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x36\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptionsR\x07options\x12I\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfoR\x0esourceCodeInfo\x12\x16\n\x06syntax\x18\x0c \x01(\tR\x06syntax\"\xb9\x06\n\x0f\x44\x65scriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12;\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\x05\x66ield\x12\x43\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x41\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\nnestedType\x12\x41\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12X\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRangeR\x0e\x65xtensionRange\x12\x44\n\noneof_decl\x18\x08 \x03(\x0b\x32%.google.protobuf.OneofDescriptorProtoR\toneofDecl\x12\x39\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptionsR\x07options\x12U\n\x0ereserved_range\x18\t \x03(\x0b\x32..google.protobuf.DescriptorProto.ReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\n \x03(\tR\x0creservedName\x1az\n\x0e\x45xtensionRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\x12@\n\x07options\x18\x03 \x01(\x0b\x32&.google.protobuf.ExtensionRangeOptionsR\x07options\x1a\x37\n\rReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"|\n\x15\x45xtensionRangeOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc1\x06\n\x14\x46ieldDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x41\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.LabelR\x05label\x12>\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.TypeR\x04type\x12\x1b\n\ttype_name\x18\x06 \x01(\tR\x08typeName\x12\x1a\n\x08\x65xtendee\x18\x02 \x01(\tR\x08\x65xtendee\x12#\n\rdefault_value\x18\x07 \x01(\tR\x0c\x64\x65\x66\x61ultValue\x12\x1f\n\x0boneof_index\x18\t \x01(\x05R\noneofIndex\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12\x37\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptionsR\x07options\x12\'\n\x0fproto3_optional\x18\x11 \x01(\x08R\x0eproto3Optional\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\x12\x12\n\x0eLABEL_REPEATED\x10\x03\"c\n\x14OneofDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\x07options\x18\x02 \x01(\x0b\x32\x1d.google.protobuf.OneofOptionsR\x07options\"\xe3\x02\n\x13\x45numDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12?\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProtoR\x05value\x12\x36\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptionsR\x07options\x12]\n\x0ereserved_range\x18\x04 \x03(\x0b\x32\x36.google.protobuf.EnumDescriptorProto.EnumReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\x05 \x03(\tR\x0creservedName\x1a;\n\x11\x45numReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\x83\x01\n\x18\x45numValueDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptionsR\x07options\"\xa7\x01\n\x16ServiceDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12>\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProtoR\x06method\x12\x39\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptionsR\x07options\"\x89\x02\n\x15MethodDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n\ninput_type\x18\x02 \x01(\tR\tinputType\x12\x1f\n\x0boutput_type\x18\x03 \x01(\tR\noutputType\x12\x38\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptionsR\x07options\x12\x30\n\x10\x63lient_streaming\x18\x05 \x01(\x08:\x05\x66\x61lseR\x0f\x63lientStreaming\x12\x30\n\x10server_streaming\x18\x06 \x01(\x08:\x05\x66\x61lseR\x0fserverStreaming\"\x91\t\n\x0b\x46ileOptions\x12!\n\x0cjava_package\x18\x01 \x01(\tR\x0bjavaPackage\x12\x30\n\x14java_outer_classname\x18\x08 \x01(\tR\x12javaOuterClassname\x12\x35\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lseR\x11javaMultipleFiles\x12\x44\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08\x42\x02\x18\x01R\x19javaGenerateEqualsAndHash\x12:\n\x16java_string_check_utf8\x18\x1b \x01(\x08:\x05\x66\x61lseR\x13javaStringCheckUtf8\x12S\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEEDR\x0boptimizeFor\x12\x1d\n\ngo_package\x18\x0b \x01(\tR\tgoPackage\x12\x35\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lseR\x11\x63\x63GenericServices\x12\x39\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lseR\x13javaGenericServices\x12\x35\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lseR\x11pyGenericServices\x12\x37\n\x14php_generic_services\x18* \x01(\x08:\x05\x66\x61lseR\x12phpGenericServices\x12%\n\ndeprecated\x18\x17 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12.\n\x10\x63\x63_enable_arenas\x18\x1f \x01(\x08:\x04trueR\x0e\x63\x63\x45nableArenas\x12*\n\x11objc_class_prefix\x18$ \x01(\tR\x0fobjcClassPrefix\x12)\n\x10\x63sharp_namespace\x18% \x01(\tR\x0f\x63sharpNamespace\x12!\n\x0cswift_prefix\x18\' \x01(\tR\x0bswiftPrefix\x12(\n\x10php_class_prefix\x18( \x01(\tR\x0ephpClassPrefix\x12#\n\rphp_namespace\x18) \x01(\tR\x0cphpNamespace\x12\x34\n\x16php_metadata_namespace\x18, \x01(\tR\x14phpMetadataNamespace\x12!\n\x0cruby_package\x18- \x01(\tR\x0brubyPackage\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08&\x10\'\"\xe3\x02\n\x0eMessageOptions\x12<\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lseR\x14messageSetWireFormat\x12L\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lseR\x1cnoStandardDescriptorAccessor\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1b\n\tmap_entry\x18\x07 \x01(\x08R\x08mapEntry\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\tJ\x04\x08\t\x10\n\"\x92\x04\n\x0c\x46ieldOptions\x12\x41\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRINGR\x05\x63type\x12\x16\n\x06packed\x18\x02 \x01(\x08R\x06packed\x12G\n\x06jstype\x18\x06 \x01(\x0e\x32$.google.protobuf.FieldOptions.JSType:\tJS_NORMALR\x06jstype\x12\x19\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lseR\x04lazy\x12.\n\x0funverified_lazy\x18\x0f \x01(\x08:\x05\x66\x61lseR\x0eunverifiedLazy\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x19\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lseR\x04weak\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02\"5\n\x06JSType\x12\r\n\tJS_NORMAL\x10\x00\x12\r\n\tJS_STRING\x10\x01\x12\r\n\tJS_NUMBER\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05\"s\n\x0cOneofOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc0\x01\n\x0b\x45numOptions\x12\x1f\n\x0b\x61llow_alias\x18\x02 \x01(\x08R\nallowAlias\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x05\x10\x06\"\x9e\x01\n\x10\x45numValueOptions\x12%\n\ndeprecated\x18\x01 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9c\x01\n\x0eServiceOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xe0\x02\n\rMethodOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12q\n\x11idempotency_level\x18\" \x01(\x0e\x32/.google.protobuf.MethodOptions.IdempotencyLevel:\x13IDEMPOTENCY_UNKNOWNR\x10idempotencyLevel\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"P\n\x10IdempotencyLevel\x12\x17\n\x13IDEMPOTENCY_UNKNOWN\x10\x00\x12\x13\n\x0fNO_SIDE_EFFECTS\x10\x01\x12\x0e\n\nIDEMPOTENT\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9a\x03\n\x13UninterpretedOption\x12\x41\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePartR\x04name\x12)\n\x10identifier_value\x18\x03 \x01(\tR\x0fidentifierValue\x12,\n\x12positive_int_value\x18\x04 \x01(\x04R\x10positiveIntValue\x12,\n\x12negative_int_value\x18\x05 \x01(\x03R\x10negativeIntValue\x12!\n\x0c\x64ouble_value\x18\x06 \x01(\x01R\x0b\x64oubleValue\x12!\n\x0cstring_value\x18\x07 \x01(\x0cR\x0bstringValue\x12\'\n\x0f\x61ggregate_value\x18\x08 \x01(\tR\x0e\x61ggregateValue\x1aJ\n\x08NamePart\x12\x1b\n\tname_part\x18\x01 \x02(\tR\x08namePart\x12!\n\x0cis_extension\x18\x02 \x02(\x08R\x0bisExtension\"\xa7\x02\n\x0eSourceCodeInfo\x12\x44\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.LocationR\x08location\x1a\xce\x01\n\x08Location\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x16\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01R\x04span\x12)\n\x10leading_comments\x18\x03 \x01(\tR\x0fleadingComments\x12+\n\x11trailing_comments\x18\x04 \x01(\tR\x10trailingComments\x12:\n\x19leading_detached_comments\x18\x06 \x03(\tR\x17leadingDetachedComments\"\xd1\x01\n\x11GeneratedCodeInfo\x12M\n\nannotation\x18\x01 \x03(\x0b\x32-.google.protobuf.GeneratedCodeInfo.AnnotationR\nannotation\x1am\n\nAnnotation\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x1f\n\x0bsource_file\x18\x02 \x01(\tR\nsourceFile\x12\x14\n\x05\x62\x65gin\x18\x03 \x01(\x05R\x05\x62\x65gin\x12\x10\n\x03\x65nd\x18\x04 \x01(\x05R\x03\x65ndB~\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01Z-google.golang.org/protobuf/types/descriptorpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1aGoogle.Protobuf.Reflection') + DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"M\n\x11\x46ileDescriptorSet\x12\x38\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x04\x66ile\"\xfe\x04\n\x13\x46ileDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n\x07package\x18\x02 \x01(\tR\x07package\x12\x1e\n\ndependency\x18\x03 \x03(\tR\ndependency\x12+\n\x11public_dependency\x18\n \x03(\x05R\x10publicDependency\x12\'\n\x0fweak_dependency\x18\x0b \x03(\x05R\x0eweakDependency\x12\x43\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\x0bmessageType\x12\x41\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12\x41\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProtoR\x07service\x12\x43\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x36\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptionsR\x07options\x12I\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfoR\x0esourceCodeInfo\x12\x16\n\x06syntax\x18\x0c \x01(\tR\x06syntax\x12\x18\n\x07\x65\x64ition\x18\r \x01(\tR\x07\x65\x64ition\"\xb9\x06\n\x0f\x44\x65scriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12;\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\x05\x66ield\x12\x43\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x41\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\nnestedType\x12\x41\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12X\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRangeR\x0e\x65xtensionRange\x12\x44\n\noneof_decl\x18\x08 \x03(\x0b\x32%.google.protobuf.OneofDescriptorProtoR\toneofDecl\x12\x39\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptionsR\x07options\x12U\n\x0ereserved_range\x18\t \x03(\x0b\x32..google.protobuf.DescriptorProto.ReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\n \x03(\tR\x0creservedName\x1az\n\x0e\x45xtensionRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\x12@\n\x07options\x18\x03 \x01(\x0b\x32&.google.protobuf.ExtensionRangeOptionsR\x07options\x1a\x37\n\rReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"|\n\x15\x45xtensionRangeOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc1\x06\n\x14\x46ieldDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x41\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.LabelR\x05label\x12>\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.TypeR\x04type\x12\x1b\n\ttype_name\x18\x06 \x01(\tR\x08typeName\x12\x1a\n\x08\x65xtendee\x18\x02 \x01(\tR\x08\x65xtendee\x12#\n\rdefault_value\x18\x07 \x01(\tR\x0c\x64\x65\x66\x61ultValue\x12\x1f\n\x0boneof_index\x18\t \x01(\x05R\noneofIndex\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12\x37\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptionsR\x07options\x12\'\n\x0fproto3_optional\x18\x11 \x01(\x08R\x0eproto3Optional\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\x12\x12\n\x0eLABEL_REPEATED\x10\x03\"c\n\x14OneofDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\x07options\x18\x02 \x01(\x0b\x32\x1d.google.protobuf.OneofOptionsR\x07options\"\xe3\x02\n\x13\x45numDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12?\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProtoR\x05value\x12\x36\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptionsR\x07options\x12]\n\x0ereserved_range\x18\x04 \x03(\x0b\x32\x36.google.protobuf.EnumDescriptorProto.EnumReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\x05 \x03(\tR\x0creservedName\x1a;\n\x11\x45numReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\x83\x01\n\x18\x45numValueDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptionsR\x07options\"\xa7\x01\n\x16ServiceDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12>\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProtoR\x06method\x12\x39\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptionsR\x07options\"\x89\x02\n\x15MethodDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n\ninput_type\x18\x02 \x01(\tR\tinputType\x12\x1f\n\x0boutput_type\x18\x03 \x01(\tR\noutputType\x12\x38\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptionsR\x07options\x12\x30\n\x10\x63lient_streaming\x18\x05 \x01(\x08:\x05\x66\x61lseR\x0f\x63lientStreaming\x12\x30\n\x10server_streaming\x18\x06 \x01(\x08:\x05\x66\x61lseR\x0fserverStreaming\"\x91\t\n\x0b\x46ileOptions\x12!\n\x0cjava_package\x18\x01 \x01(\tR\x0bjavaPackage\x12\x30\n\x14java_outer_classname\x18\x08 \x01(\tR\x12javaOuterClassname\x12\x35\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lseR\x11javaMultipleFiles\x12\x44\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08\x42\x02\x18\x01R\x19javaGenerateEqualsAndHash\x12:\n\x16java_string_check_utf8\x18\x1b \x01(\x08:\x05\x66\x61lseR\x13javaStringCheckUtf8\x12S\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEEDR\x0boptimizeFor\x12\x1d\n\ngo_package\x18\x0b \x01(\tR\tgoPackage\x12\x35\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lseR\x11\x63\x63GenericServices\x12\x39\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lseR\x13javaGenericServices\x12\x35\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lseR\x11pyGenericServices\x12\x37\n\x14php_generic_services\x18* \x01(\x08:\x05\x66\x61lseR\x12phpGenericServices\x12%\n\ndeprecated\x18\x17 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12.\n\x10\x63\x63_enable_arenas\x18\x1f \x01(\x08:\x04trueR\x0e\x63\x63\x45nableArenas\x12*\n\x11objc_class_prefix\x18$ \x01(\tR\x0fobjcClassPrefix\x12)\n\x10\x63sharp_namespace\x18% \x01(\tR\x0f\x63sharpNamespace\x12!\n\x0cswift_prefix\x18\' \x01(\tR\x0bswiftPrefix\x12(\n\x10php_class_prefix\x18( \x01(\tR\x0ephpClassPrefix\x12#\n\rphp_namespace\x18) \x01(\tR\x0cphpNamespace\x12\x34\n\x16php_metadata_namespace\x18, \x01(\tR\x14phpMetadataNamespace\x12!\n\x0cruby_package\x18- \x01(\tR\x0brubyPackage\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08&\x10\'\"\xbb\x03\n\x0eMessageOptions\x12<\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lseR\x14messageSetWireFormat\x12L\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lseR\x1cnoStandardDescriptorAccessor\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1b\n\tmap_entry\x18\x07 \x01(\x08R\x08mapEntry\x12V\n&deprecated_legacy_json_field_conflicts\x18\x0b \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\tJ\x04\x08\t\x10\n\"\xb7\x08\n\x0c\x46ieldOptions\x12\x41\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRINGR\x05\x63type\x12\x16\n\x06packed\x18\x02 \x01(\x08R\x06packed\x12G\n\x06jstype\x18\x06 \x01(\x0e\x32$.google.protobuf.FieldOptions.JSType:\tJS_NORMALR\x06jstype\x12\x19\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lseR\x04lazy\x12.\n\x0funverified_lazy\x18\x0f \x01(\x08:\x05\x66\x61lseR\x0eunverifiedLazy\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x19\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lseR\x04weak\x12(\n\x0c\x64\x65\x62ug_redact\x18\x10 \x01(\x08:\x05\x66\x61lseR\x0b\x64\x65\x62ugRedact\x12K\n\tretention\x18\x11 \x01(\x0e\x32-.google.protobuf.FieldOptions.OptionRetentionR\tretention\x12\x46\n\x06target\x18\x12 \x01(\x0e\x32..google.protobuf.FieldOptions.OptionTargetTypeR\x06target\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02\"5\n\x06JSType\x12\r\n\tJS_NORMAL\x10\x00\x12\r\n\tJS_STRING\x10\x01\x12\r\n\tJS_NUMBER\x10\x02\"U\n\x0fOptionRetention\x12\x15\n\x11RETENTION_UNKNOWN\x10\x00\x12\x15\n\x11RETENTION_RUNTIME\x10\x01\x12\x14\n\x10RETENTION_SOURCE\x10\x02\"\x8c\x02\n\x10OptionTargetType\x12\x17\n\x13TARGET_TYPE_UNKNOWN\x10\x00\x12\x14\n\x10TARGET_TYPE_FILE\x10\x01\x12\x1f\n\x1bTARGET_TYPE_EXTENSION_RANGE\x10\x02\x12\x17\n\x13TARGET_TYPE_MESSAGE\x10\x03\x12\x15\n\x11TARGET_TYPE_FIELD\x10\x04\x12\x15\n\x11TARGET_TYPE_ONEOF\x10\x05\x12\x14\n\x10TARGET_TYPE_ENUM\x10\x06\x12\x1a\n\x16TARGET_TYPE_ENUM_ENTRY\x10\x07\x12\x17\n\x13TARGET_TYPE_SERVICE\x10\x08\x12\x16\n\x12TARGET_TYPE_METHOD\x10\t*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05\"s\n\x0cOneofOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x98\x02\n\x0b\x45numOptions\x12\x1f\n\x0b\x61llow_alias\x18\x02 \x01(\x08R\nallowAlias\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12V\n&deprecated_legacy_json_field_conflicts\x18\x06 \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x05\x10\x06\"\x9e\x01\n\x10\x45numValueOptions\x12%\n\ndeprecated\x18\x01 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9c\x01\n\x0eServiceOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xe0\x02\n\rMethodOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12q\n\x11idempotency_level\x18\" \x01(\x0e\x32/.google.protobuf.MethodOptions.IdempotencyLevel:\x13IDEMPOTENCY_UNKNOWNR\x10idempotencyLevel\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"P\n\x10IdempotencyLevel\x12\x17\n\x13IDEMPOTENCY_UNKNOWN\x10\x00\x12\x13\n\x0fNO_SIDE_EFFECTS\x10\x01\x12\x0e\n\nIDEMPOTENT\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9a\x03\n\x13UninterpretedOption\x12\x41\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePartR\x04name\x12)\n\x10identifier_value\x18\x03 \x01(\tR\x0fidentifierValue\x12,\n\x12positive_int_value\x18\x04 \x01(\x04R\x10positiveIntValue\x12,\n\x12negative_int_value\x18\x05 \x01(\x03R\x10negativeIntValue\x12!\n\x0c\x64ouble_value\x18\x06 \x01(\x01R\x0b\x64oubleValue\x12!\n\x0cstring_value\x18\x07 \x01(\x0cR\x0bstringValue\x12\'\n\x0f\x61ggregate_value\x18\x08 \x01(\tR\x0e\x61ggregateValue\x1aJ\n\x08NamePart\x12\x1b\n\tname_part\x18\x01 \x02(\tR\x08namePart\x12!\n\x0cis_extension\x18\x02 \x02(\x08R\x0bisExtension\"\xa7\x02\n\x0eSourceCodeInfo\x12\x44\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.LocationR\x08location\x1a\xce\x01\n\x08Location\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x16\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01R\x04span\x12)\n\x10leading_comments\x18\x03 \x01(\tR\x0fleadingComments\x12+\n\x11trailing_comments\x18\x04 \x01(\tR\x10trailingComments\x12:\n\x19leading_detached_comments\x18\x06 \x03(\tR\x17leadingDetachedComments\"\xd0\x02\n\x11GeneratedCodeInfo\x12M\n\nannotation\x18\x01 \x03(\x0b\x32-.google.protobuf.GeneratedCodeInfo.AnnotationR\nannotation\x1a\xeb\x01\n\nAnnotation\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x1f\n\x0bsource_file\x18\x02 \x01(\tR\nsourceFile\x12\x14\n\x05\x62\x65gin\x18\x03 \x01(\x05R\x05\x62\x65gin\x12\x10\n\x03\x65nd\x18\x04 \x01(\x05R\x03\x65nd\x12R\n\x08semantic\x18\x05 \x01(\x0e\x32\x36.google.protobuf.GeneratedCodeInfo.Annotation.SemanticR\x08semantic\"(\n\x08Semantic\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03SET\x10\x01\x12\t\n\x05\x41LIAS\x10\x02\x42~\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01Z-google.golang.org/protobuf/types/descriptorpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1aGoogle.Protobuf.Reflection') +_globals = globals() if _descriptor._USE_C_DESCRIPTORS == False: _FIELDDESCRIPTORPROTO_TYPE = _descriptor.EnumDescriptor( name='Type', @@ -241,6 +242,97 @@ ) _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_JSTYPE) + _FIELDOPTIONS_OPTIONRETENTION = _descriptor.EnumDescriptor( + name='OptionRetention', + full_name='google.protobuf.FieldOptions.OptionRetention', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='RETENTION_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='RETENTION_RUNTIME', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='RETENTION_SOURCE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_OPTIONRETENTION) + + _FIELDOPTIONS_OPTIONTARGETTYPE = _descriptor.EnumDescriptor( + name='OptionTargetType', + full_name='google.protobuf.FieldOptions.OptionTargetType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_FILE', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_EXTENSION_RANGE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_MESSAGE', index=3, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_FIELD', index=4, number=4, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_ONEOF', index=5, number=5, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_ENUM', index=6, number=6, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_ENUM_ENTRY', index=7, number=7, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_SERVICE', index=8, number=8, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_METHOD', index=9, number=9, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_OPTIONTARGETTYPE) + _METHODOPTIONS_IDEMPOTENCYLEVEL = _descriptor.EnumDescriptor( name='IdempotencyLevel', full_name='google.protobuf.MethodOptions.IdempotencyLevel', @@ -269,6 +361,34 @@ ) _sym_db.RegisterEnumDescriptor(_METHODOPTIONS_IDEMPOTENCYLEVEL) + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC = _descriptor.EnumDescriptor( + name='Semantic', + full_name='google.protobuf.GeneratedCodeInfo.Annotation.Semantic', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='SET', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ALIAS', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_GENERATEDCODEINFO_ANNOTATION_SEMANTIC) + _FILEDESCRIPTORSET = _descriptor.Descriptor( name='FileDescriptorSet', @@ -392,6 +512,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, json_name='syntax', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='edition', full_name='google.protobuf.FileDescriptorProto.edition', index=12, + number=13, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='edition', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -1203,7 +1330,14 @@ is_extension=False, extension_scope=None, serialized_options=None, json_name='mapEntry', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.MessageOptions.uninterpreted_option', index=4, + name='deprecated_legacy_json_field_conflicts', full_name='google.protobuf.MessageOptions.deprecated_legacy_json_field_conflicts', index=4, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecatedLegacyJsonFieldConflicts', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.MessageOptions.uninterpreted_option', index=5, number=999, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -1282,7 +1416,28 @@ is_extension=False, extension_scope=None, serialized_options=None, json_name='weak', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.FieldOptions.uninterpreted_option', index=7, + name='debug_redact', full_name='google.protobuf.FieldOptions.debug_redact', index=7, + number=16, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='debugRedact', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='retention', full_name='google.protobuf.FieldOptions.retention', index=8, + number=17, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='retention', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='target', full_name='google.protobuf.FieldOptions.target', index=9, + number=18, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='target', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.FieldOptions.uninterpreted_option', index=10, number=999, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -1295,6 +1450,8 @@ enum_types=[ _FIELDOPTIONS_CTYPE, _FIELDOPTIONS_JSTYPE, + _FIELDOPTIONS_OPTIONRETENTION, + _FIELDOPTIONS_OPTIONTARGETTYPE, ], serialized_options=None, is_extendable=True, @@ -1358,7 +1515,14 @@ is_extension=False, extension_scope=None, serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.EnumOptions.uninterpreted_option', index=2, + name='deprecated_legacy_json_field_conflicts', full_name='google.protobuf.EnumOptions.deprecated_legacy_json_field_conflicts', index=2, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecatedLegacyJsonFieldConflicts', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.EnumOptions.uninterpreted_option', index=3, number=999, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -1729,11 +1893,19 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, json_name='end', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='semantic', full_name='google.protobuf.GeneratedCodeInfo.Annotation.semantic', index=4, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='semantic', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], nested_types=[], enum_types=[ + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC, ], serialized_options=None, is_extendable=False, @@ -1811,9 +1983,13 @@ _MESSAGEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION _FIELDOPTIONS.fields_by_name['ctype'].enum_type = _FIELDOPTIONS_CTYPE _FIELDOPTIONS.fields_by_name['jstype'].enum_type = _FIELDOPTIONS_JSTYPE + _FIELDOPTIONS.fields_by_name['retention'].enum_type = _FIELDOPTIONS_OPTIONRETENTION + _FIELDOPTIONS.fields_by_name['target'].enum_type = _FIELDOPTIONS_OPTIONTARGETTYPE _FIELDOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION _FIELDOPTIONS_CTYPE.containing_type = _FIELDOPTIONS _FIELDOPTIONS_JSTYPE.containing_type = _FIELDOPTIONS + _FIELDOPTIONS_OPTIONRETENTION.containing_type = _FIELDOPTIONS + _FIELDOPTIONS_OPTIONTARGETTYPE.containing_type = _FIELDOPTIONS _ONEOFOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION _ENUMOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION _ENUMVALUEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION @@ -1825,7 +2001,9 @@ _UNINTERPRETEDOPTION.fields_by_name['name'].message_type = _UNINTERPRETEDOPTION_NAMEPART _SOURCECODEINFO_LOCATION.containing_type = _SOURCECODEINFO _SOURCECODEINFO.fields_by_name['location'].message_type = _SOURCECODEINFO_LOCATION + _GENERATEDCODEINFO_ANNOTATION.fields_by_name['semantic'].enum_type = _GENERATEDCODEINFO_ANNOTATION_SEMANTIC _GENERATEDCODEINFO_ANNOTATION.containing_type = _GENERATEDCODEINFO + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC.containing_type = _GENERATEDCODEINFO_ANNOTATION _GENERATEDCODEINFO.fields_by_name['annotation'].message_type = _GENERATEDCODEINFO_ANNOTATION DESCRIPTOR.message_types_by_name['FileDescriptorSet'] = _FILEDESCRIPTORSET DESCRIPTOR.message_types_by_name['FileDescriptorProto'] = _FILEDESCRIPTORPROTO @@ -1851,75 +2029,81 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) else: - _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.descriptor_pb2', globals()) + _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.descriptor_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _FILEDESCRIPTORSET._serialized_start=53 - _FILEDESCRIPTORSET._serialized_end=130 - _FILEDESCRIPTORPROTO._serialized_start=133 - _FILEDESCRIPTORPROTO._serialized_end=745 - _DESCRIPTORPROTO._serialized_start=748 - _DESCRIPTORPROTO._serialized_end=1573 - _DESCRIPTORPROTO_EXTENSIONRANGE._serialized_start=1394 - _DESCRIPTORPROTO_EXTENSIONRANGE._serialized_end=1516 - _DESCRIPTORPROTO_RESERVEDRANGE._serialized_start=1518 - _DESCRIPTORPROTO_RESERVEDRANGE._serialized_end=1573 - _EXTENSIONRANGEOPTIONS._serialized_start=1575 - _EXTENSIONRANGEOPTIONS._serialized_end=1699 - _FIELDDESCRIPTORPROTO._serialized_start=1702 - _FIELDDESCRIPTORPROTO._serialized_end=2535 - _FIELDDESCRIPTORPROTO_TYPE._serialized_start=2156 - _FIELDDESCRIPTORPROTO_TYPE._serialized_end=2466 - _FIELDDESCRIPTORPROTO_LABEL._serialized_start=2468 - _FIELDDESCRIPTORPROTO_LABEL._serialized_end=2535 - _ONEOFDESCRIPTORPROTO._serialized_start=2537 - _ONEOFDESCRIPTORPROTO._serialized_end=2636 - _ENUMDESCRIPTORPROTO._serialized_start=2639 - _ENUMDESCRIPTORPROTO._serialized_end=2994 - _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE._serialized_start=2935 - _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE._serialized_end=2994 - _ENUMVALUEDESCRIPTORPROTO._serialized_start=2997 - _ENUMVALUEDESCRIPTORPROTO._serialized_end=3128 - _SERVICEDESCRIPTORPROTO._serialized_start=3131 - _SERVICEDESCRIPTORPROTO._serialized_end=3298 - _METHODDESCRIPTORPROTO._serialized_start=3301 - _METHODDESCRIPTORPROTO._serialized_end=3566 - _FILEOPTIONS._serialized_start=3569 - _FILEOPTIONS._serialized_end=4738 - _FILEOPTIONS_OPTIMIZEMODE._serialized_start=4663 - _FILEOPTIONS_OPTIMIZEMODE._serialized_end=4721 - _MESSAGEOPTIONS._serialized_start=4741 - _MESSAGEOPTIONS._serialized_end=5096 - _FIELDOPTIONS._serialized_start=5099 - _FIELDOPTIONS._serialized_end=5629 - _FIELDOPTIONS_CTYPE._serialized_start=5510 - _FIELDOPTIONS_CTYPE._serialized_end=5557 - _FIELDOPTIONS_JSTYPE._serialized_start=5559 - _FIELDOPTIONS_JSTYPE._serialized_end=5612 - _ONEOFOPTIONS._serialized_start=5631 - _ONEOFOPTIONS._serialized_end=5746 - _ENUMOPTIONS._serialized_start=5749 - _ENUMOPTIONS._serialized_end=5941 - _ENUMVALUEOPTIONS._serialized_start=5944 - _ENUMVALUEOPTIONS._serialized_end=6102 - _SERVICEOPTIONS._serialized_start=6105 - _SERVICEOPTIONS._serialized_end=6261 - _METHODOPTIONS._serialized_start=6264 - _METHODOPTIONS._serialized_end=6616 - _METHODOPTIONS_IDEMPOTENCYLEVEL._serialized_start=6525 - _METHODOPTIONS_IDEMPOTENCYLEVEL._serialized_end=6605 - _UNINTERPRETEDOPTION._serialized_start=6619 - _UNINTERPRETEDOPTION._serialized_end=7029 - _UNINTERPRETEDOPTION_NAMEPART._serialized_start=6955 - _UNINTERPRETEDOPTION_NAMEPART._serialized_end=7029 - _SOURCECODEINFO._serialized_start=7032 - _SOURCECODEINFO._serialized_end=7327 - _SOURCECODEINFO_LOCATION._serialized_start=7121 - _SOURCECODEINFO_LOCATION._serialized_end=7327 - _GENERATEDCODEINFO._serialized_start=7330 - _GENERATEDCODEINFO._serialized_end=7539 - _GENERATEDCODEINFO_ANNOTATION._serialized_start=7430 - _GENERATEDCODEINFO_ANNOTATION._serialized_end=7539 + _globals['_FILEDESCRIPTORSET']._serialized_start=53 + _globals['_FILEDESCRIPTORSET']._serialized_end=130 + _globals['_FILEDESCRIPTORPROTO']._serialized_start=133 + _globals['_FILEDESCRIPTORPROTO']._serialized_end=771 + _globals['_DESCRIPTORPROTO']._serialized_start=774 + _globals['_DESCRIPTORPROTO']._serialized_end=1599 + _globals['_DESCRIPTORPROTO_EXTENSIONRANGE']._serialized_start=1420 + _globals['_DESCRIPTORPROTO_EXTENSIONRANGE']._serialized_end=1542 + _globals['_DESCRIPTORPROTO_RESERVEDRANGE']._serialized_start=1544 + _globals['_DESCRIPTORPROTO_RESERVEDRANGE']._serialized_end=1599 + _globals['_EXTENSIONRANGEOPTIONS']._serialized_start=1601 + _globals['_EXTENSIONRANGEOPTIONS']._serialized_end=1725 + _globals['_FIELDDESCRIPTORPROTO']._serialized_start=1728 + _globals['_FIELDDESCRIPTORPROTO']._serialized_end=2561 + _globals['_FIELDDESCRIPTORPROTO_TYPE']._serialized_start=2182 + _globals['_FIELDDESCRIPTORPROTO_TYPE']._serialized_end=2492 + _globals['_FIELDDESCRIPTORPROTO_LABEL']._serialized_start=2494 + _globals['_FIELDDESCRIPTORPROTO_LABEL']._serialized_end=2561 + _globals['_ONEOFDESCRIPTORPROTO']._serialized_start=2563 + _globals['_ONEOFDESCRIPTORPROTO']._serialized_end=2662 + _globals['_ENUMDESCRIPTORPROTO']._serialized_start=2665 + _globals['_ENUMDESCRIPTORPROTO']._serialized_end=3020 + _globals['_ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE']._serialized_start=2961 + _globals['_ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE']._serialized_end=3020 + _globals['_ENUMVALUEDESCRIPTORPROTO']._serialized_start=3023 + _globals['_ENUMVALUEDESCRIPTORPROTO']._serialized_end=3154 + _globals['_SERVICEDESCRIPTORPROTO']._serialized_start=3157 + _globals['_SERVICEDESCRIPTORPROTO']._serialized_end=3324 + _globals['_METHODDESCRIPTORPROTO']._serialized_start=3327 + _globals['_METHODDESCRIPTORPROTO']._serialized_end=3592 + _globals['_FILEOPTIONS']._serialized_start=3595 + _globals['_FILEOPTIONS']._serialized_end=4764 + _globals['_FILEOPTIONS_OPTIMIZEMODE']._serialized_start=4689 + _globals['_FILEOPTIONS_OPTIMIZEMODE']._serialized_end=4747 + _globals['_MESSAGEOPTIONS']._serialized_start=4767 + _globals['_MESSAGEOPTIONS']._serialized_end=5210 + _globals['_FIELDOPTIONS']._serialized_start=5213 + _globals['_FIELDOPTIONS']._serialized_end=6292 + _globals['_FIELDOPTIONS_CTYPE']._serialized_start=5815 + _globals['_FIELDOPTIONS_CTYPE']._serialized_end=5862 + _globals['_FIELDOPTIONS_JSTYPE']._serialized_start=5864 + _globals['_FIELDOPTIONS_JSTYPE']._serialized_end=5917 + _globals['_FIELDOPTIONS_OPTIONRETENTION']._serialized_start=5919 + _globals['_FIELDOPTIONS_OPTIONRETENTION']._serialized_end=6004 + _globals['_FIELDOPTIONS_OPTIONTARGETTYPE']._serialized_start=6007 + _globals['_FIELDOPTIONS_OPTIONTARGETTYPE']._serialized_end=6275 + _globals['_ONEOFOPTIONS']._serialized_start=6294 + _globals['_ONEOFOPTIONS']._serialized_end=6409 + _globals['_ENUMOPTIONS']._serialized_start=6412 + _globals['_ENUMOPTIONS']._serialized_end=6692 + _globals['_ENUMVALUEOPTIONS']._serialized_start=6695 + _globals['_ENUMVALUEOPTIONS']._serialized_end=6853 + _globals['_SERVICEOPTIONS']._serialized_start=6856 + _globals['_SERVICEOPTIONS']._serialized_end=7012 + _globals['_METHODOPTIONS']._serialized_start=7015 + _globals['_METHODOPTIONS']._serialized_end=7367 + _globals['_METHODOPTIONS_IDEMPOTENCYLEVEL']._serialized_start=7276 + _globals['_METHODOPTIONS_IDEMPOTENCYLEVEL']._serialized_end=7356 + _globals['_UNINTERPRETEDOPTION']._serialized_start=7370 + _globals['_UNINTERPRETEDOPTION']._serialized_end=7780 + _globals['_UNINTERPRETEDOPTION_NAMEPART']._serialized_start=7706 + _globals['_UNINTERPRETEDOPTION_NAMEPART']._serialized_end=7780 + _globals['_SOURCECODEINFO']._serialized_start=7783 + _globals['_SOURCECODEINFO']._serialized_end=8078 + _globals['_SOURCECODEINFO_LOCATION']._serialized_start=7872 + _globals['_SOURCECODEINFO_LOCATION']._serialized_end=8078 + _globals['_GENERATEDCODEINFO']._serialized_start=8081 + _globals['_GENERATEDCODEINFO']._serialized_end=8417 + _globals['_GENERATEDCODEINFO_ANNOTATION']._serialized_start=8182 + _globals['_GENERATEDCODEINFO_ANNOTATION']._serialized_end=8417 + _globals['_GENERATEDCODEINFO_ANNOTATION_SEMANTIC']._serialized_start=8377 + _globals['_GENERATEDCODEINFO_ANNOTATION_SEMANTIC']._serialized_end=8417 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/descriptor_pool.py b/ext/protobuf/Python/google/protobuf/descriptor_pool.py index 911372a8b..1ebf11834 100644 --- a/ext/protobuf/Python/google/protobuf/descriptor_pool.py +++ b/ext/protobuf/Python/google/protobuf/descriptor_pool.py @@ -120,11 +120,13 @@ class DescriptorPool(object): if _USE_C_DESCRIPTORS: - def __new__(cls, descriptor_db=None): - # pylint: disable=protected-access - return descriptor._message.DescriptorPool(descriptor_db) + def __new__(cls, descriptor_db=None): + # pylint: disable=protected-access + return descriptor._message.DescriptorPool(descriptor_db) - def __init__(self, descriptor_db=None): + def __init__( + self, descriptor_db=None, use_deprecated_legacy_json_field_conflicts=False + ): """Initializes a Pool of proto buffs. The descriptor_db argument to the constructor is provided to allow @@ -135,6 +137,8 @@ def __init__(self, descriptor_db=None): Args: descriptor_db: A secondary source of file descriptors. + use_deprecated_legacy_json_field_conflicts: Unused, for compatibility with + C++. """ self._internal_db = descriptor_database.DescriptorDatabase() @@ -144,9 +148,6 @@ def __init__(self, descriptor_db=None): self._service_descriptors = {} self._file_descriptors = {} self._toplevel_extensions = {} - # TODO(jieluo): Remove _file_desc_by_toplevel_extension after - # maybe year 2020 for compatibility issue (with 3.4.1 only). - self._file_desc_by_toplevel_extension = {} self._top_enum_values = {} # We store extensions in two two-level mappings: The first key is the # descriptor of the message being extended, the second key is the extension @@ -220,7 +221,7 @@ def AddSerializedFile(self, serialized_file_desc_proto): file_desc.serialized_pb = serialized_file_desc_proto return file_desc - # Add Descriptor to descriptor pool is dreprecated. Please use Add() + # Add Descriptor to descriptor pool is deprecated. Please use Add() # or AddSerializedFile() to add a FileDescriptorProto instead. @_Deprecated def AddDescriptor(self, desc): @@ -245,7 +246,7 @@ def _AddDescriptor(self, desc): self._descriptors[desc.full_name] = desc self._AddFileDescriptor(desc.file) - # Add EnumDescriptor to descriptor pool is dreprecated. Please use Add() + # Add EnumDescriptor to descriptor pool is deprecated. Please use Add() # or AddSerializedFile() to add a FileDescriptorProto instead. @_Deprecated def AddEnumDescriptor(self, enum_desc): @@ -286,7 +287,7 @@ def _AddEnumDescriptor(self, enum_desc): self._top_enum_values[full_name] = enum_value self._AddFileDescriptor(enum_desc.file) - # Add ServiceDescriptor to descriptor pool is dreprecated. Please use Add() + # Add ServiceDescriptor to descriptor pool is deprecated. Please use Add() # or AddSerializedFile() to add a FileDescriptorProto instead. @_Deprecated def AddServiceDescriptor(self, service_desc): @@ -307,7 +308,7 @@ def _AddServiceDescriptor(self, service_desc): service_desc.file.name) self._service_descriptors[service_desc.full_name] = service_desc - # Add ExtensionDescriptor to descriptor pool is dreprecated. Please use Add() + # Add ExtensionDescriptor to descriptor pool is deprecated. Please use Add() # or AddSerializedFile() to add a FileDescriptorProto instead. @_Deprecated def AddExtensionDescriptor(self, extension): @@ -331,6 +332,8 @@ def _AddExtensionDescriptor(self, extension): raise TypeError('Expected an extension descriptor.') if extension.extension_scope is None: + self._CheckConflictRegister( + extension, extension.full_name, extension.file.name) self._toplevel_extensions[extension.full_name] = extension try: @@ -372,12 +375,6 @@ def _InternalAddFileDescriptor(self, file_desc): """ self._AddFileDescriptor(file_desc) - # TODO(jieluo): This is a temporary solution for FieldDescriptor.file. - # FieldDescriptor.file is added in code gen. Remove this solution after - # maybe 2020 for compatibility reason (with 3.4.1 only). - for extension in file_desc.extensions_by_name.values(): - self._file_desc_by_toplevel_extension[ - extension.full_name] = file_desc def _AddFileDescriptor(self, file_desc): """Adds a FileDescriptor to the pool, non-recursively. @@ -483,7 +480,7 @@ def _InternalFindFileContainingSymbol(self, symbol): pass try: - return self._file_desc_by_toplevel_extension[symbol] + return self._toplevel_extensions[symbol].file except KeyError: pass @@ -792,8 +789,6 @@ def _ConvertFileProtoToFileDescriptor(self, file_proto): file_descriptor.package, scope) file_descriptor.extensions_by_name[extension_desc.name] = ( extension_desc) - self._file_desc_by_toplevel_extension[extension_desc.full_name] = ( - file_descriptor) for desc_proto in file_proto.message_type: self._SetAllFieldTypes(file_proto.package, desc_proto, scope) diff --git a/ext/protobuf/Python/google/protobuf/duration_pb2.py b/ext/protobuf/Python/google/protobuf/duration_pb2.py index d11409e5d..f5309c1c4 100644 --- a/ext/protobuf/Python/google/protobuf/duration_pb2.py +++ b/ext/protobuf/Python/google/protobuf/duration_pb2.py @@ -15,12 +15,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\":\n\x08\x44uration\x12\x18\n\x07seconds\x18\x01 \x01(\x03R\x07seconds\x12\x14\n\x05nanos\x18\x02 \x01(\x05R\x05nanosB\x83\x01\n\x13\x63om.google.protobufB\rDurationProtoP\x01Z1google.golang.org/protobuf/types/known/durationpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.duration_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.duration_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\rDurationProtoP\001Z1google.golang.org/protobuf/types/known/durationpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _DURATION._serialized_start=51 - _DURATION._serialized_end=109 + _globals['_DURATION']._serialized_start=51 + _globals['_DURATION']._serialized_end=109 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/empty_pb2.py b/ext/protobuf/Python/google/protobuf/empty_pb2.py index 0b4d554db..d4d36580e 100644 --- a/ext/protobuf/Python/google/protobuf/empty_pb2.py +++ b/ext/protobuf/Python/google/protobuf/empty_pb2.py @@ -15,12 +15,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB}\n\x13\x63om.google.protobufB\nEmptyProtoP\x01Z.google.golang.org/protobuf/types/known/emptypb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.empty_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.empty_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\nEmptyProtoP\001Z.google.golang.org/protobuf/types/known/emptypb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _EMPTY._serialized_start=48 - _EMPTY._serialized_end=55 + _globals['_EMPTY']._serialized_start=48 + _globals['_EMPTY']._serialized_end=55 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/field_mask_pb2.py b/ext/protobuf/Python/google/protobuf/field_mask_pb2.py index a4f60fd02..645aae888 100644 --- a/ext/protobuf/Python/google/protobuf/field_mask_pb2.py +++ b/ext/protobuf/Python/google/protobuf/field_mask_pb2.py @@ -15,12 +15,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n google/protobuf/field_mask.proto\x12\x0fgoogle.protobuf\"!\n\tFieldMask\x12\x14\n\x05paths\x18\x01 \x03(\tR\x05pathsB\x85\x01\n\x13\x63om.google.protobufB\x0e\x46ieldMaskProtoP\x01Z2google.golang.org/protobuf/types/known/fieldmaskpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.field_mask_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.field_mask_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\016FieldMaskProtoP\001Z2google.golang.org/protobuf/types/known/fieldmaskpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _FIELDMASK._serialized_start=53 - _FIELDMASK._serialized_end=86 + _globals['_FIELDMASK']._serialized_start=53 + _globals['_FIELDMASK']._serialized_end=86 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/internal/__init__.py b/ext/protobuf/Python/google/protobuf/internal/__init__.py index e69de29bb..7d2e571a1 100644 --- a/ext/protobuf/Python/google/protobuf/internal/__init__.py +++ b/ext/protobuf/Python/google/protobuf/internal/__init__.py @@ -0,0 +1,30 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/ext/protobuf/Python/google/protobuf/internal/_parameterized.py b/ext/protobuf/Python/google/protobuf/internal/_parameterized.py index afdbb78c3..2f4a3b6b7 100644 --- a/ext/protobuf/Python/google/protobuf/internal/_parameterized.py +++ b/ext/protobuf/Python/google/protobuf/internal/_parameterized.py @@ -37,8 +37,8 @@ A simple example: - class AdditionExample(parameterized.TestCase): - @parameterized.parameters( + class AdditionExample(_parameterized.TestCase): + @_parameterized.parameters( (1, 2, 3), (4, 5, 9), (1, 1, 3)) @@ -54,8 +54,8 @@ def testAddition(self, op1, op2, result): Parameters for individual test cases can be tuples (with positional parameters) or dictionaries (with named parameters): - class AdditionExample(parameterized.TestCase): - @parameterized.parameters( + class AdditionExample(_parameterized.TestCase): + @_parameterized.parameters( {'op1': 1, 'op2': 2, 'result': 3}, {'op1': 4, 'op2': 5, 'result': 9}, ) @@ -82,8 +82,8 @@ def testAddition(self, op1, op2, result): be a string (or an object that returns an apt name when converted via str()): - class NamedExample(parameterized.TestCase): - @parameterized.named_parameters( + class NamedExample(_parameterized.TestCase): + @_parameterized.named_parameters( ('Normal', 'aa', 'aaa', True), ('EmptyPrefix', '', 'abc', True), ('BothEmpty', '', '', True)) @@ -106,10 +106,10 @@ def testStartsWith(self, prefix, string, result): TestCase class, instead of decorating all test methods individually, the class itself can be decorated: - @parameterized.parameters( + @_parameterized.parameters( (1, 2, 3) (4, 5, 9)) - class ArithmeticTest(parameterized.TestCase): + class ArithmeticTest(_parameterized.TestCase): def testAdd(self, arg1, arg2, result): self.assertEqual(arg1 + arg2, result) @@ -122,8 +122,8 @@ def testSubtract(self, arg2, arg2, result): created from other sources, a single non-tuple iterable can be passed into the decorator. This iterable will be used to obtain the test cases: - class AdditionExample(parameterized.TestCase): - @parameterized.parameters( + class AdditionExample(_parameterized.TestCase): + @_parameterized.parameters( c.op1, c.op2, c.result for c in testcases ) def testAddition(self, op1, op2, result): @@ -135,8 +135,8 @@ def testAddition(self, op1, op2, result): If a test method takes only one argument, the single argument does not need to be wrapped into a tuple: - class NegativeNumberExample(parameterized.TestCase): - @parameterized.parameters( + class NegativeNumberExample(_parameterized.TestCase): + @_parameterized.parameters( -1, -3, -4, -5 ) def testIsNegative(self, arg): @@ -423,7 +423,7 @@ def CoopTestCase(other_base_class): import google3 import mox - from google3.testing.pybase import parameterized + from google.protobuf.internal import _parameterized class ExampleTest(parameterized.CoopTestCase(mox.MoxTestBase)): ... diff --git a/ext/protobuf/Python/google/protobuf/internal/api_implementation.py b/ext/protobuf/Python/google/protobuf/internal/api_implementation.py index 74586487a..7d20bd221 100644 --- a/ext/protobuf/Python/google/protobuf/internal/api_implementation.py +++ b/ext/protobuf/Python/google/protobuf/internal/api_implementation.py @@ -102,6 +102,7 @@ def _CanImport(mod_name): try: # pylint: disable=g-import-not-at-top from google.protobuf.pyext import _message + sys.modules['google3.net.proto2.python.internal.cpp._message'] = _message _c_module = _message del _message except ImportError: @@ -151,12 +152,6 @@ def Type(): return _implementation_type -def _SetType(implementation_type): - """Never use! Only for protobuf benchmark.""" - global _implementation_type - _implementation_type = implementation_type - - # See comment on 'Type' above. # TODO(jieluo): Remove the API, it returns a constant. b/228102101 def Version(): diff --git a/ext/protobuf/Python/google/protobuf/internal/decoder.py b/ext/protobuf/Python/google/protobuf/internal/decoder.py index a91627631..8ff549381 100644 --- a/ext/protobuf/Python/google/protobuf/internal/decoder.py +++ b/ext/protobuf/Python/google/protobuf/internal/decoder.py @@ -806,8 +806,7 @@ def DecodeItem(buffer, pos, end, message, field_dict): if value is None: message_type = extension.message_type if not hasattr(message_type, '_concrete_class'): - # pylint: disable=protected-access - message._FACTORY.GetPrototype(message_type) + message_factory.GetMessageClass(message_type) value = field_dict.setdefault( extension, message_type._concrete_class()) if value._InternalParse(buffer, message_start,message_end) != message_end: diff --git a/ext/protobuf/Python/google/protobuf/internal/descriptor_database_test.py b/ext/protobuf/Python/google/protobuf/internal/descriptor_database_test.py deleted file mode 100644 index 3c086b924..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/descriptor_database_test.py +++ /dev/null @@ -1,127 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.descriptor_database.""" - -__author__ = 'matthewtoia@google.com (Matt Toia)' - -import unittest -import warnings - -from google.protobuf import unittest_pb2 -from google.protobuf import descriptor_pb2 -from google.protobuf.internal import factory_test2_pb2 -from google.protobuf.internal import no_package_pb2 -from google.protobuf.internal import testing_refleaks -from google.protobuf import descriptor_database - - -@testing_refleaks.TestCase -class DescriptorDatabaseTest(unittest.TestCase): - - def testAdd(self): - db = descriptor_database.DescriptorDatabase() - file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString( - factory_test2_pb2.DESCRIPTOR.serialized_pb) - file_desc_proto2 = descriptor_pb2.FileDescriptorProto.FromString( - no_package_pb2.DESCRIPTOR.serialized_pb) - db.Add(file_desc_proto) - db.Add(file_desc_proto2) - - self.assertEqual(file_desc_proto, db.FindFileByName( - 'google/protobuf/internal/factory_test2.proto')) - # Can find message type. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message')) - # Can find nested message type. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message.NestedFactory2Message')) - # Can find enum type. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Enum')) - # Can find nested enum type. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum')) - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.MessageWithNestedEnumOnly.NestedEnum')) - # Can find field. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message.list_field')) - # Can find enum value. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Enum.FACTORY_2_VALUE_0')) - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.FACTORY_2_VALUE_0')) - self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol( - '.NO_PACKAGE_VALUE_0')) - # Can find top level extension. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.another_field')) - # Can find nested extension inside a message. - self.assertEqual(file_desc_proto, db.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message.one_more_field')) - - # Can find service. - file_desc_proto2 = descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb) - db.Add(file_desc_proto2) - self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol( - 'protobuf_unittest.TestService')) - - # Non-existent field under a valid top level symbol can also be - # found. The behavior is the same with protobuf C++. - self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol( - 'protobuf_unittest.TestAllTypes.none_field')) - - with self.assertRaisesRegex(KeyError, r'\'protobuf_unittest\.NoneMessage\''): - db.FindFileContainingSymbol('protobuf_unittest.NoneMessage') - - def testConflictRegister(self): - db = descriptor_database.DescriptorDatabase() - unittest_fd = descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb) - db.Add(unittest_fd) - conflict_fd = descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb) - conflict_fd.name = 'other_file2' - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter('always') - db.Add(conflict_fd) - self.assertTrue(len(w)) - self.assertIs(w[0].category, RuntimeWarning) - self.assertIn('Conflict register for file "other_file2": ', - str(w[0].message)) - self.assertIn('already defined in file ' - '"google/protobuf/unittest.proto"', - str(w[0].message)) - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/descriptor_pool_test.py b/ext/protobuf/Python/google/protobuf/internal/descriptor_pool_test.py deleted file mode 100644 index 9e451b45b..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/descriptor_pool_test.py +++ /dev/null @@ -1,1149 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.descriptor_pool.""" - -__author__ = 'matthewtoia@google.com (Matt Toia)' - -import copy -import os -import unittest -import warnings - -from google.protobuf import unittest_import_pb2 -from google.protobuf import unittest_import_public_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import descriptor_pb2 -from google.protobuf.internal import api_implementation -from google.protobuf.internal import descriptor_pool_test1_pb2 -from google.protobuf.internal import descriptor_pool_test2_pb2 -from google.protobuf.internal import factory_test1_pb2 -from google.protobuf.internal import factory_test2_pb2 -from google.protobuf.internal import file_options_test_pb2 -from google.protobuf.internal import more_messages_pb2 -from google.protobuf.internal import no_package_pb2 -from google.protobuf.internal import testing_refleaks -from google.protobuf import descriptor -from google.protobuf import descriptor_database -from google.protobuf import descriptor_pool -from google.protobuf import message_factory -from google.protobuf import symbol_database - - - -warnings.simplefilter('error', DeprecationWarning) - - -class DescriptorPoolTestBase(object): - - def testFindFileByName(self): - name1 = 'google/protobuf/internal/factory_test1.proto' - file_desc1 = self.pool.FindFileByName(name1) - self.assertIsInstance(file_desc1, descriptor.FileDescriptor) - self.assertEqual(name1, file_desc1.name) - self.assertEqual('google.protobuf.python.internal', file_desc1.package) - self.assertIn('Factory1Message', file_desc1.message_types_by_name) - - name2 = 'google/protobuf/internal/factory_test2.proto' - file_desc2 = self.pool.FindFileByName(name2) - self.assertIsInstance(file_desc2, descriptor.FileDescriptor) - self.assertEqual(name2, file_desc2.name) - self.assertEqual('google.protobuf.python.internal', file_desc2.package) - self.assertIn('Factory2Message', file_desc2.message_types_by_name) - - def testFindFileByNameFailure(self): - with self.assertRaises(KeyError): - self.pool.FindFileByName('Does not exist') - - def testFindFileContainingSymbol(self): - file_desc1 = self.pool.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory1Message') - self.assertIsInstance(file_desc1, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/internal/factory_test1.proto', - file_desc1.name) - self.assertEqual('google.protobuf.python.internal', file_desc1.package) - self.assertIn('Factory1Message', file_desc1.message_types_by_name) - - file_desc2 = self.pool.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message') - self.assertIsInstance(file_desc2, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/internal/factory_test2.proto', - file_desc2.name) - self.assertEqual('google.protobuf.python.internal', file_desc2.package) - self.assertIn('Factory2Message', file_desc2.message_types_by_name) - - # Tests top level extension. - file_desc3 = self.pool.FindFileContainingSymbol( - 'google.protobuf.python.internal.another_field') - self.assertIsInstance(file_desc3, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/internal/factory_test2.proto', - file_desc3.name) - - # Tests nested extension inside a message. - file_desc4 = self.pool.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message.one_more_field') - self.assertIsInstance(file_desc4, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/internal/factory_test2.proto', - file_desc4.name) - - file_desc5 = self.pool.FindFileContainingSymbol( - 'protobuf_unittest.TestService') - self.assertIsInstance(file_desc5, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/unittest.proto', - file_desc5.name) - # Tests the generated pool. - assert descriptor_pool.Default().FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory2Message.one_more_field') - assert descriptor_pool.Default().FindFileContainingSymbol( - 'google.protobuf.python.internal.another_field') - assert descriptor_pool.Default().FindFileContainingSymbol( - 'protobuf_unittest.TestService') - - # Can find field. - file_desc6 = self.pool.FindFileContainingSymbol( - 'google.protobuf.python.internal.Factory1Message.list_value') - self.assertIsInstance(file_desc6, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/internal/factory_test1.proto', - file_desc6.name) - - # Can find top level Enum value. - file_desc7 = self.pool.FindFileContainingSymbol( - 'google.protobuf.python.internal.FACTORY_1_VALUE_0') - self.assertIsInstance(file_desc7, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/internal/factory_test1.proto', - file_desc7.name) - - # Can find nested Enum value. - file_desc8 = self.pool.FindFileContainingSymbol( - 'protobuf_unittest.TestAllTypes.FOO') - self.assertIsInstance(file_desc8, descriptor.FileDescriptor) - self.assertEqual('google/protobuf/unittest.proto', - file_desc8.name) - - # TODO(jieluo): Add tests for no package when b/13860351 is fixed. - - self.assertRaises(KeyError, self.pool.FindFileContainingSymbol, - 'google.protobuf.python.internal.Factory1Message.none_field') - - def testFindFileContainingSymbolFailure(self): - with self.assertRaises(KeyError): - self.pool.FindFileContainingSymbol('Does not exist') - - def testFindMessageTypeByName(self): - msg1 = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory1Message') - self.assertIsInstance(msg1, descriptor.Descriptor) - self.assertEqual('Factory1Message', msg1.name) - self.assertEqual('google.protobuf.python.internal.Factory1Message', - msg1.full_name) - self.assertEqual(None, msg1.containing_type) - self.assertFalse(msg1.has_options) - - nested_msg1 = msg1.nested_types[0] - self.assertEqual('NestedFactory1Message', nested_msg1.name) - self.assertEqual(msg1, nested_msg1.containing_type) - - nested_enum1 = msg1.enum_types[0] - self.assertEqual('NestedFactory1Enum', nested_enum1.name) - self.assertEqual(msg1, nested_enum1.containing_type) - - self.assertEqual(nested_msg1, msg1.fields_by_name[ - 'nested_factory_1_message'].message_type) - self.assertEqual(nested_enum1, msg1.fields_by_name[ - 'nested_factory_1_enum'].enum_type) - - msg2 = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message') - self.assertIsInstance(msg2, descriptor.Descriptor) - self.assertEqual('Factory2Message', msg2.name) - self.assertEqual('google.protobuf.python.internal.Factory2Message', - msg2.full_name) - self.assertIsNone(msg2.containing_type) - - nested_msg2 = msg2.nested_types[0] - self.assertEqual('NestedFactory2Message', nested_msg2.name) - self.assertEqual(msg2, nested_msg2.containing_type) - - nested_enum2 = msg2.enum_types[0] - self.assertEqual('NestedFactory2Enum', nested_enum2.name) - self.assertEqual(msg2, nested_enum2.containing_type) - - self.assertEqual(nested_msg2, msg2.fields_by_name[ - 'nested_factory_2_message'].message_type) - self.assertEqual(nested_enum2, msg2.fields_by_name[ - 'nested_factory_2_enum'].enum_type) - - self.assertTrue(msg2.fields_by_name['int_with_default'].has_default_value) - self.assertEqual( - 1776, msg2.fields_by_name['int_with_default'].default_value) - - self.assertTrue( - msg2.fields_by_name['double_with_default'].has_default_value) - self.assertEqual( - 9.99, msg2.fields_by_name['double_with_default'].default_value) - - self.assertTrue( - msg2.fields_by_name['string_with_default'].has_default_value) - self.assertEqual( - 'hello world', msg2.fields_by_name['string_with_default'].default_value) - - self.assertTrue(msg2.fields_by_name['bool_with_default'].has_default_value) - self.assertFalse(msg2.fields_by_name['bool_with_default'].default_value) - - self.assertTrue(msg2.fields_by_name['enum_with_default'].has_default_value) - self.assertEqual( - 1, msg2.fields_by_name['enum_with_default'].default_value) - - msg3 = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message.NestedFactory2Message') - self.assertEqual(nested_msg2, msg3) - - self.assertTrue(msg2.fields_by_name['bytes_with_default'].has_default_value) - self.assertEqual( - b'a\xfb\x00c', - msg2.fields_by_name['bytes_with_default'].default_value) - - self.assertEqual(1, len(msg2.oneofs)) - self.assertEqual(1, len(msg2.oneofs_by_name)) - self.assertEqual(2, len(msg2.oneofs[0].fields)) - for name in ['oneof_int', 'oneof_string']: - self.assertEqual(msg2.oneofs[0], - msg2.fields_by_name[name].containing_oneof) - self.assertIn(msg2.fields_by_name[name], msg2.oneofs[0].fields) - - def testFindTypeErrors(self): - self.assertRaises(TypeError, self.pool.FindExtensionByNumber, '') - self.assertRaises(KeyError, self.pool.FindMethodByName, '') - - # TODO(jieluo): Fix python to raise correct errors. - if api_implementation.Type() == 'python': - error_type = AttributeError - else: - error_type = TypeError - self.assertRaises(error_type, self.pool.FindMessageTypeByName, 0) - self.assertRaises(error_type, self.pool.FindFieldByName, 0) - self.assertRaises(error_type, self.pool.FindExtensionByName, 0) - self.assertRaises(error_type, self.pool.FindEnumTypeByName, 0) - self.assertRaises(error_type, self.pool.FindOneofByName, 0) - self.assertRaises(error_type, self.pool.FindServiceByName, 0) - self.assertRaises(error_type, self.pool.FindMethodByName, 0) - self.assertRaises(error_type, self.pool.FindFileContainingSymbol, 0) - if api_implementation.Type() == 'python': - error_type = KeyError - self.assertRaises(error_type, self.pool.FindFileByName, 0) - - def testFindMessageTypeByNameFailure(self): - with self.assertRaises(KeyError): - self.pool.FindMessageTypeByName('Does not exist') - - def testFindEnumTypeByName(self): - enum1 = self.pool.FindEnumTypeByName( - 'google.protobuf.python.internal.Factory1Enum') - self.assertIsInstance(enum1, descriptor.EnumDescriptor) - self.assertEqual(0, enum1.values_by_name['FACTORY_1_VALUE_0'].number) - self.assertEqual(1, enum1.values_by_name['FACTORY_1_VALUE_1'].number) - self.assertFalse(enum1.has_options) - - nested_enum1 = self.pool.FindEnumTypeByName( - 'google.protobuf.python.internal.Factory1Message.NestedFactory1Enum') - self.assertIsInstance(nested_enum1, descriptor.EnumDescriptor) - self.assertEqual( - 0, nested_enum1.values_by_name['NESTED_FACTORY_1_VALUE_0'].number) - self.assertEqual( - 1, nested_enum1.values_by_name['NESTED_FACTORY_1_VALUE_1'].number) - - enum2 = self.pool.FindEnumTypeByName( - 'google.protobuf.python.internal.Factory2Enum') - self.assertIsInstance(enum2, descriptor.EnumDescriptor) - self.assertEqual(0, enum2.values_by_name['FACTORY_2_VALUE_0'].number) - self.assertEqual(1, enum2.values_by_name['FACTORY_2_VALUE_1'].number) - - nested_enum2 = self.pool.FindEnumTypeByName( - 'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum') - self.assertIsInstance(nested_enum2, descriptor.EnumDescriptor) - self.assertEqual( - 0, nested_enum2.values_by_name['NESTED_FACTORY_2_VALUE_0'].number) - self.assertEqual( - 1, nested_enum2.values_by_name['NESTED_FACTORY_2_VALUE_1'].number) - - def testFindEnumTypeByNameFailure(self): - with self.assertRaises(KeyError): - self.pool.FindEnumTypeByName('Does not exist') - - def testFindFieldByName(self): - field = self.pool.FindFieldByName( - 'google.protobuf.python.internal.Factory1Message.list_value') - self.assertEqual(field.name, 'list_value') - self.assertEqual(field.label, field.LABEL_REPEATED) - self.assertFalse(field.has_options) - - with self.assertRaises(KeyError): - self.pool.FindFieldByName('Does not exist') - - def testFindOneofByName(self): - oneof = self.pool.FindOneofByName( - 'google.protobuf.python.internal.Factory2Message.oneof_field') - self.assertEqual(oneof.name, 'oneof_field') - with self.assertRaises(KeyError): - self.pool.FindOneofByName('Does not exist') - - def testFindExtensionByName(self): - # An extension defined in a message. - extension = self.pool.FindExtensionByName( - 'google.protobuf.python.internal.Factory2Message.one_more_field') - self.assertEqual(extension.name, 'one_more_field') - # An extension defined at file scope. - extension = self.pool.FindExtensionByName( - 'google.protobuf.python.internal.another_field') - self.assertEqual(extension.name, 'another_field') - self.assertEqual(extension.number, 1002) - with self.assertRaises(KeyError): - self.pool.FindFieldByName('Does not exist') - - def testFindAllExtensions(self): - factory1_message = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory1Message') - factory2_message = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message') - # An extension defined in a message. - one_more_field = factory2_message.extensions_by_name['one_more_field'] - # An extension defined at file scope. - factory_test2 = self.pool.FindFileByName( - 'google/protobuf/internal/factory_test2.proto') - another_field = factory_test2.extensions_by_name['another_field'] - - extensions = self.pool.FindAllExtensions(factory1_message) - expected_extension_numbers = set([one_more_field, another_field]) - self.assertEqual(expected_extension_numbers, set(extensions)) - # Verify that mutating the returned list does not affect the pool. - extensions.append('unexpected_element') - # Get the extensions again, the returned value does not contain the - # 'unexpected_element'. - extensions = self.pool.FindAllExtensions(factory1_message) - self.assertEqual(expected_extension_numbers, set(extensions)) - - def testFindExtensionByNumber(self): - factory1_message = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory1Message') - # Build factory_test2.proto which will put extensions to the pool - self.pool.FindFileByName( - 'google/protobuf/internal/factory_test2.proto') - - # An extension defined in a message. - extension = self.pool.FindExtensionByNumber(factory1_message, 1001) - self.assertEqual(extension.name, 'one_more_field') - # An extension defined at file scope. - extension = self.pool.FindExtensionByNumber(factory1_message, 1002) - self.assertEqual(extension.name, 'another_field') - with self.assertRaises(KeyError): - extension = self.pool.FindExtensionByNumber(factory1_message, 1234567) - - def testExtensionsAreNotFields(self): - with self.assertRaises(KeyError): - self.pool.FindFieldByName('google.protobuf.python.internal.another_field') - with self.assertRaises(KeyError): - self.pool.FindFieldByName( - 'google.protobuf.python.internal.Factory2Message.one_more_field') - with self.assertRaises(KeyError): - self.pool.FindExtensionByName( - 'google.protobuf.python.internal.Factory1Message.list_value') - - def testFindService(self): - service = self.pool.FindServiceByName('protobuf_unittest.TestService') - self.assertEqual(service.full_name, 'protobuf_unittest.TestService') - with self.assertRaises(KeyError): - self.pool.FindServiceByName('Does not exist') - - method = self.pool.FindMethodByName('protobuf_unittest.TestService.Foo') - self.assertIs(method.containing_service, service) - with self.assertRaises(KeyError): - self.pool.FindMethodByName('protobuf_unittest.TestService.Doesnotexist') - - def testUserDefinedDB(self): - db = descriptor_database.DescriptorDatabase() - self.pool = descriptor_pool.DescriptorPool(db) - db.Add(self.factory_test1_fd) - db.Add(self.factory_test2_fd) - self.testFindMessageTypeByName() - - def testAddSerializedFile(self): - if isinstance(self, SecondaryDescriptorFromDescriptorDB): - if api_implementation.Type() != 'python': - # Cpp extension cannot call Add on a DescriptorPool - # that uses a DescriptorDatabase. - # TODO(jieluo): Fix python and cpp extension diff. - return - self.pool = descriptor_pool.DescriptorPool() - file1 = self.pool.AddSerializedFile( - self.factory_test1_fd.SerializeToString()) - file2 = self.pool.AddSerializedFile( - self.factory_test2_fd.SerializeToString()) - self.assertEqual(file1.name, - 'google/protobuf/internal/factory_test1.proto') - self.assertEqual(file2.name, - 'google/protobuf/internal/factory_test2.proto') - self.testFindMessageTypeByName() - file_json = self.pool.AddSerializedFile( - more_messages_pb2.DESCRIPTOR.serialized_pb) - field = file_json.message_types_by_name['class'].fields_by_name['int_field'] - self.assertEqual(field.json_name, 'json_int') - - - def testEnumDefaultValue(self): - """Test the default value of enums which don't start at zero.""" - def _CheckDefaultValue(file_descriptor): - default_value = (file_descriptor - .message_types_by_name['DescriptorPoolTest1'] - .fields_by_name['nested_enum'] - .default_value) - self.assertEqual(default_value, - descriptor_pool_test1_pb2.DescriptorPoolTest1.BETA) - # First check what the generated descriptor contains. - _CheckDefaultValue(descriptor_pool_test1_pb2.DESCRIPTOR) - # Then check the generated pool. Normally this is the same descriptor. - file_descriptor = symbol_database.Default().pool.FindFileByName( - 'google/protobuf/internal/descriptor_pool_test1.proto') - self.assertIs(file_descriptor, descriptor_pool_test1_pb2.DESCRIPTOR) - _CheckDefaultValue(file_descriptor) - - if isinstance(self, SecondaryDescriptorFromDescriptorDB): - if api_implementation.Type() != 'python': - # Cpp extension cannot call Add on a DescriptorPool - # that uses a DescriptorDatabase. - # TODO(jieluo): Fix python and cpp extension diff. - return - # Then check the dynamic pool and its internal DescriptorDatabase. - descriptor_proto = descriptor_pb2.FileDescriptorProto.FromString( - descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb) - self.pool.Add(descriptor_proto) - # And do the same check as above - file_descriptor = self.pool.FindFileByName( - 'google/protobuf/internal/descriptor_pool_test1.proto') - _CheckDefaultValue(file_descriptor) - - def testDefaultValueForCustomMessages(self): - """Check the value returned by non-existent fields.""" - def _CheckValueAndType(value, expected_value, expected_type): - self.assertEqual(value, expected_value) - self.assertIsInstance(value, expected_type) - - def _CheckDefaultValues(msg): - try: - int64 = long - except NameError: # Python3 - int64 = int - try: - unicode_type = unicode - except NameError: # Python3 - unicode_type = str - _CheckValueAndType(msg.optional_int32, 0, int) - _CheckValueAndType(msg.optional_uint64, 0, (int64, int)) - _CheckValueAndType(msg.optional_float, 0, (float, int)) - _CheckValueAndType(msg.optional_double, 0, (float, int)) - _CheckValueAndType(msg.optional_bool, False, bool) - _CheckValueAndType(msg.optional_string, u'', unicode_type) - _CheckValueAndType(msg.optional_bytes, b'', bytes) - _CheckValueAndType(msg.optional_nested_enum, msg.FOO, int) - # First for the generated message - _CheckDefaultValues(unittest_pb2.TestAllTypes()) - # Then for a message built with from the DescriptorPool. - pool = descriptor_pool.DescriptorPool() - pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_import_public_pb2.DESCRIPTOR.serialized_pb)) - pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_import_pb2.DESCRIPTOR.serialized_pb)) - pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb)) - message_class = message_factory.MessageFactory(pool).GetPrototype( - pool.FindMessageTypeByName( - unittest_pb2.TestAllTypes.DESCRIPTOR.full_name)) - _CheckDefaultValues(message_class()) - - def testAddFileDescriptor(self): - if isinstance(self, SecondaryDescriptorFromDescriptorDB): - if api_implementation.Type() != 'python': - # Cpp extension cannot call Add on a DescriptorPool - # that uses a DescriptorDatabase. - # TODO(jieluo): Fix python and cpp extension diff. - return - file_desc = descriptor_pb2.FileDescriptorProto(name='some/file.proto') - self.pool.Add(file_desc) - self.pool.AddSerializedFile(file_desc.SerializeToString()) - - def testComplexNesting(self): - if isinstance(self, SecondaryDescriptorFromDescriptorDB): - if api_implementation.Type() != 'python': - # Cpp extension cannot call Add on a DescriptorPool - # that uses a DescriptorDatabase. - # TODO(jieluo): Fix python and cpp extension diff. - return - more_messages_desc = descriptor_pb2.FileDescriptorProto.FromString( - more_messages_pb2.DESCRIPTOR.serialized_pb) - test1_desc = descriptor_pb2.FileDescriptorProto.FromString( - descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb) - test2_desc = descriptor_pb2.FileDescriptorProto.FromString( - descriptor_pool_test2_pb2.DESCRIPTOR.serialized_pb) - self.pool.Add(more_messages_desc) - self.pool.Add(test1_desc) - self.pool.Add(test2_desc) - TEST1_FILE.CheckFile(self, self.pool) - TEST2_FILE.CheckFile(self, self.pool) - - def testConflictRegister(self): - if isinstance(self, SecondaryDescriptorFromDescriptorDB): - if api_implementation.Type() != 'python': - # Cpp extension cannot call Add on a DescriptorPool - # that uses a DescriptorDatabase. - # TODO(jieluo): Fix python and cpp extension diff. - return - unittest_fd = descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb) - conflict_fd = copy.deepcopy(unittest_fd) - conflict_fd.name = 'other_file' - if api_implementation.Type() != 'python': - pass - else: - pool = copy.deepcopy(self.pool) - file_descriptor = unittest_pb2.DESCRIPTOR - pool._AddDescriptor( - file_descriptor.message_types_by_name['TestAllTypes']) - pool._AddEnumDescriptor( - file_descriptor.enum_types_by_name['ForeignEnum']) - pool._AddServiceDescriptor( - file_descriptor.services_by_name['TestService']) - pool._AddExtensionDescriptor( - file_descriptor.extensions_by_name['optional_int32_extension']) - pool.Add(unittest_fd) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - pool.Add(conflict_fd) - self.assertTrue(len(w)) - self.assertIs(w[0].category, RuntimeWarning) - self.assertIn('Conflict register for file "other_file": ', - str(w[0].message)) - pool.FindFileByName(unittest_fd.name) - with self.assertRaises(TypeError): - pool.FindFileByName(conflict_fd.name) - - -@testing_refleaks.TestCase -class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase): - - def setUp(self): - self.pool = descriptor_pool.Default() - self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test1_pb2.DESCRIPTOR.serialized_pb) - self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test2_pb2.DESCRIPTOR.serialized_pb) - - def testFindMethods(self): - self.assertIs( - self.pool.FindFileByName('google/protobuf/unittest.proto'), - unittest_pb2.DESCRIPTOR) - self.assertIs( - self.pool.FindMessageTypeByName('protobuf_unittest.TestAllTypes'), - unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertIs( - self.pool.FindFieldByName( - 'protobuf_unittest.TestAllTypes.optional_int32'), - unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name['optional_int32']) - self.assertIs( - self.pool.FindEnumTypeByName('protobuf_unittest.ForeignEnum'), - unittest_pb2.ForeignEnum.DESCRIPTOR) - self.assertIs( - self.pool.FindExtensionByName( - 'protobuf_unittest.optional_int32_extension'), - unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension']) - self.assertIs( - self.pool.FindOneofByName('protobuf_unittest.TestAllTypes.oneof_field'), - unittest_pb2.TestAllTypes.DESCRIPTOR.oneofs_by_name['oneof_field']) - self.assertIs( - self.pool.FindServiceByName('protobuf_unittest.TestService'), - unittest_pb2.DESCRIPTOR.services_by_name['TestService']) - - -@testing_refleaks.TestCase -class CreateDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase): - - def setUp(self): - self.pool = descriptor_pool.DescriptorPool() - self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test1_pb2.DESCRIPTOR.serialized_pb) - self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test2_pb2.DESCRIPTOR.serialized_pb) - self.pool.Add(self.factory_test1_fd) - self.pool.Add(self.factory_test2_fd) - - self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_import_public_pb2.DESCRIPTOR.serialized_pb)) - self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_import_pb2.DESCRIPTOR.serialized_pb)) - self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb)) - self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( - no_package_pb2.DESCRIPTOR.serialized_pb)) - - -@testing_refleaks.TestCase -class SecondaryDescriptorFromDescriptorDB(DescriptorPoolTestBase, - unittest.TestCase): - - def setUp(self): - self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test1_pb2.DESCRIPTOR.serialized_pb) - self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test2_pb2.DESCRIPTOR.serialized_pb) - self.db = descriptor_database.DescriptorDatabase() - self.db.Add(self.factory_test1_fd) - self.db.Add(self.factory_test2_fd) - self.db.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_import_public_pb2.DESCRIPTOR.serialized_pb)) - self.db.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_import_pb2.DESCRIPTOR.serialized_pb)) - self.db.Add(descriptor_pb2.FileDescriptorProto.FromString( - unittest_pb2.DESCRIPTOR.serialized_pb)) - self.db.Add(descriptor_pb2.FileDescriptorProto.FromString( - no_package_pb2.DESCRIPTOR.serialized_pb)) - self.pool = descriptor_pool.DescriptorPool(descriptor_db=self.db) - - def testErrorCollector(self): - file_proto = descriptor_pb2.FileDescriptorProto() - file_proto.package = 'collector' - file_proto.name = 'error_file' - message_type = file_proto.message_type.add() - message_type.name = 'ErrorMessage' - field = message_type.field.add() - field.number = 1 - field.name = 'nested_message_field' - field.label = descriptor.FieldDescriptor.LABEL_OPTIONAL - field.type = descriptor.FieldDescriptor.TYPE_MESSAGE - field.type_name = 'SubMessage' - oneof = message_type.oneof_decl.add() - oneof.name = 'MyOneof' - enum_type = file_proto.enum_type.add() - enum_type.name = 'MyEnum' - enum_value = enum_type.value.add() - enum_value.name = 'MyEnumValue' - enum_value.number = 0 - self.db.Add(file_proto) - - self.assertRaisesRegex(KeyError, 'SubMessage', - self.pool.FindMessageTypeByName, - 'collector.ErrorMessage') - self.assertRaisesRegex(KeyError, 'SubMessage', self.pool.FindFileByName, - 'error_file') - with self.assertRaises(KeyError) as exc: - self.pool.FindFileByName('none_file') - self.assertIn(str(exc.exception), ('\'none_file\'', - '\"Couldn\'t find file none_file\"')) - - # Pure python _ConvertFileProtoToFileDescriptor() method has side effect - # that all the symbols found in the file will load into the pool even the - # file can not build. So when FindMessageTypeByName('ErrorMessage') was - # called the first time, a KeyError will be raised but call the find - # method later will return a descriptor which is not build. - # TODO(jieluo): fix pure python to revert the load if file can not be build - if api_implementation.Type() != 'python': - error_msg = ('Invalid proto descriptor for file "error_file":\\n ' - 'collector.ErrorMessage.nested_message_field: "SubMessage" ' - 'is not defined.\\n collector.ErrorMessage.MyOneof: Oneof ' - 'must have at least one field.\\n\'') - with self.assertRaises(KeyError) as exc: - self.pool.FindMessageTypeByName('collector.ErrorMessage') - self.assertEqual(str(exc.exception), '\'Couldn\\\'t build file for ' - 'message collector.ErrorMessage\\n' + error_msg) - - with self.assertRaises(KeyError) as exc: - self.pool.FindFieldByName('collector.ErrorMessage.nested_message_field') - self.assertEqual(str(exc.exception), '\'Couldn\\\'t build file for field' - ' collector.ErrorMessage.nested_message_field\\n' - + error_msg) - - with self.assertRaises(KeyError) as exc: - self.pool.FindEnumTypeByName('collector.MyEnum') - self.assertEqual(str(exc.exception), '\'Couldn\\\'t build file for enum' - ' collector.MyEnum\\n' + error_msg) - - with self.assertRaises(KeyError) as exc: - self.pool.FindFileContainingSymbol('collector.MyEnumValue') - self.assertEqual(str(exc.exception), '\'Couldn\\\'t build file for symbol' - ' collector.MyEnumValue\\n' + error_msg) - - with self.assertRaises(KeyError) as exc: - self.pool.FindOneofByName('collector.ErrorMessage.MyOneof') - self.assertEqual(str(exc.exception), '\'Couldn\\\'t build file for oneof' - ' collector.ErrorMessage.MyOneof\\n' + error_msg) - - -class ProtoFile(object): - - def __init__(self, name, package, messages, dependencies=None, - public_dependencies=None): - self.name = name - self.package = package - self.messages = messages - self.dependencies = dependencies or [] - self.public_dependencies = public_dependencies or [] - - def CheckFile(self, test, pool): - file_desc = pool.FindFileByName(self.name) - test.assertEqual(self.name, file_desc.name) - test.assertEqual(self.package, file_desc.package) - dependencies_names = [f.name for f in file_desc.dependencies] - test.assertEqual(self.dependencies, dependencies_names) - public_dependencies_names = [f.name for f in file_desc.public_dependencies] - test.assertEqual(self.public_dependencies, public_dependencies_names) - for name, msg_type in self.messages.items(): - msg_type.CheckType(test, None, name, file_desc) - - -class EnumType(object): - - def __init__(self, values): - self.values = values - - def CheckType(self, test, msg_desc, name, file_desc): - enum_desc = msg_desc.enum_types_by_name[name] - test.assertEqual(name, enum_desc.name) - expected_enum_full_name = '.'.join([msg_desc.full_name, name]) - test.assertEqual(expected_enum_full_name, enum_desc.full_name) - test.assertEqual(msg_desc, enum_desc.containing_type) - test.assertEqual(file_desc, enum_desc.file) - for index, (value, number) in enumerate(self.values): - value_desc = enum_desc.values_by_name[value] - test.assertEqual(value, value_desc.name) - test.assertEqual(index, value_desc.index) - test.assertEqual(number, value_desc.number) - test.assertEqual(enum_desc, value_desc.type) - test.assertIn(value, msg_desc.enum_values_by_name) - - -class MessageType(object): - - def __init__(self, type_dict, field_list, is_extendable=False, - extensions=None): - self.type_dict = type_dict - self.field_list = field_list - self.is_extendable = is_extendable - self.extensions = extensions or [] - - def CheckType(self, test, containing_type_desc, name, file_desc): - if containing_type_desc is None: - desc = file_desc.message_types_by_name[name] - expected_full_name = '.'.join([file_desc.package, name]) - else: - desc = containing_type_desc.nested_types_by_name[name] - expected_full_name = '.'.join([containing_type_desc.full_name, name]) - - test.assertEqual(name, desc.name) - test.assertEqual(expected_full_name, desc.full_name) - test.assertEqual(containing_type_desc, desc.containing_type) - test.assertEqual(desc.file, file_desc) - test.assertEqual(self.is_extendable, desc.is_extendable) - for name, subtype in self.type_dict.items(): - subtype.CheckType(test, desc, name, file_desc) - - for index, (name, field) in enumerate(self.field_list): - field.CheckField(test, desc, name, index, file_desc) - - for index, (name, field) in enumerate(self.extensions): - field.CheckField(test, desc, name, index, file_desc) - - -class EnumField(object): - - def __init__(self, number, type_name, default_value): - self.number = number - self.type_name = type_name - self.default_value = default_value - - def CheckField(self, test, msg_desc, name, index, file_desc): - field_desc = msg_desc.fields_by_name[name] - enum_desc = msg_desc.enum_types_by_name[self.type_name] - test.assertEqual(name, field_desc.name) - expected_field_full_name = '.'.join([msg_desc.full_name, name]) - test.assertEqual(expected_field_full_name, field_desc.full_name) - test.assertEqual(index, field_desc.index) - test.assertEqual(self.number, field_desc.number) - test.assertEqual(descriptor.FieldDescriptor.TYPE_ENUM, field_desc.type) - test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_ENUM, - field_desc.cpp_type) - test.assertTrue(field_desc.has_default_value) - test.assertEqual(enum_desc.values_by_name[self.default_value].number, - field_desc.default_value) - test.assertFalse(enum_desc.values_by_name[self.default_value].has_options) - test.assertEqual(msg_desc, field_desc.containing_type) - test.assertEqual(enum_desc, field_desc.enum_type) - test.assertEqual(file_desc, enum_desc.file) - - -class MessageField(object): - - def __init__(self, number, type_name): - self.number = number - self.type_name = type_name - - def CheckField(self, test, msg_desc, name, index, file_desc): - field_desc = msg_desc.fields_by_name[name] - field_type_desc = msg_desc.nested_types_by_name[self.type_name] - test.assertEqual(name, field_desc.name) - expected_field_full_name = '.'.join([msg_desc.full_name, name]) - test.assertEqual(expected_field_full_name, field_desc.full_name) - test.assertEqual(index, field_desc.index) - test.assertEqual(self.number, field_desc.number) - test.assertEqual(descriptor.FieldDescriptor.TYPE_MESSAGE, field_desc.type) - test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_MESSAGE, - field_desc.cpp_type) - test.assertFalse(field_desc.has_default_value) - test.assertEqual(msg_desc, field_desc.containing_type) - test.assertEqual(field_type_desc, field_desc.message_type) - test.assertEqual(file_desc, field_desc.file) - test.assertEqual(field_desc.default_value, None) - - -class StringField(object): - - def __init__(self, number, default_value): - self.number = number - self.default_value = default_value - - def CheckField(self, test, msg_desc, name, index, file_desc): - field_desc = msg_desc.fields_by_name[name] - test.assertEqual(name, field_desc.name) - expected_field_full_name = '.'.join([msg_desc.full_name, name]) - test.assertEqual(expected_field_full_name, field_desc.full_name) - test.assertEqual(index, field_desc.index) - test.assertEqual(self.number, field_desc.number) - test.assertEqual(descriptor.FieldDescriptor.TYPE_STRING, field_desc.type) - test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_STRING, - field_desc.cpp_type) - test.assertTrue(field_desc.has_default_value) - test.assertEqual(self.default_value, field_desc.default_value) - test.assertEqual(file_desc, field_desc.file) - - -class ExtensionField(object): - - def __init__(self, number, extended_type): - self.number = number - self.extended_type = extended_type - - def CheckField(self, test, msg_desc, name, index, file_desc): - field_desc = msg_desc.extensions_by_name[name] - test.assertEqual(name, field_desc.name) - expected_field_full_name = '.'.join([msg_desc.full_name, name]) - test.assertEqual(expected_field_full_name, field_desc.full_name) - test.assertEqual(self.number, field_desc.number) - test.assertEqual(index, field_desc.index) - test.assertEqual(descriptor.FieldDescriptor.TYPE_MESSAGE, field_desc.type) - test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_MESSAGE, - field_desc.cpp_type) - test.assertFalse(field_desc.has_default_value) - test.assertTrue(field_desc.is_extension) - test.assertEqual(msg_desc, field_desc.extension_scope) - test.assertEqual(msg_desc, field_desc.message_type) - test.assertEqual(self.extended_type, field_desc.containing_type.name) - test.assertEqual(file_desc, field_desc.file) - - -@testing_refleaks.TestCase -class AddDescriptorTest(unittest.TestCase): - - def _TestMessage(self, prefix): - pool = descriptor_pool.DescriptorPool() - pool._AddDescriptor(unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertEqual( - 'protobuf_unittest.TestAllTypes', - pool.FindMessageTypeByName( - prefix + 'protobuf_unittest.TestAllTypes').full_name) - - # AddDescriptor is not recursive. - with self.assertRaises(KeyError): - pool.FindMessageTypeByName( - prefix + 'protobuf_unittest.TestAllTypes.NestedMessage') - - pool._AddDescriptor(unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR) - self.assertEqual( - 'protobuf_unittest.TestAllTypes.NestedMessage', - pool.FindMessageTypeByName( - prefix + 'protobuf_unittest.TestAllTypes.NestedMessage').full_name) - - # Files are implicitly also indexed when messages are added. - self.assertEqual( - 'google/protobuf/unittest.proto', - pool.FindFileByName( - 'google/protobuf/unittest.proto').name) - - self.assertEqual( - 'google/protobuf/unittest.proto', - pool.FindFileContainingSymbol( - prefix + 'protobuf_unittest.TestAllTypes.NestedMessage').name) - - @unittest.skipIf(api_implementation.Type() != 'python', - 'Only pure python allows _Add*()') - def testMessage(self): - self._TestMessage('') - self._TestMessage('.') - - def _TestEnum(self, prefix): - pool = descriptor_pool.DescriptorPool() - if api_implementation.Type() == 'cpp': - pool.AddEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) - else: - pool._AddEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) - self.assertEqual( - 'protobuf_unittest.ForeignEnum', - pool.FindEnumTypeByName( - prefix + 'protobuf_unittest.ForeignEnum').full_name) - - # AddEnumDescriptor is not recursive. - with self.assertRaises(KeyError): - pool.FindEnumTypeByName( - prefix + 'protobuf_unittest.ForeignEnum.NestedEnum') - - if api_implementation.Type() == 'cpp': - pool.AddEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) - else: - pool._AddEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) - self.assertEqual( - 'protobuf_unittest.TestAllTypes.NestedEnum', - pool.FindEnumTypeByName( - prefix + 'protobuf_unittest.TestAllTypes.NestedEnum').full_name) - - # Files are implicitly also indexed when enums are added. - self.assertEqual( - 'google/protobuf/unittest.proto', - pool.FindFileByName( - 'google/protobuf/unittest.proto').name) - - self.assertEqual( - 'google/protobuf/unittest.proto', - pool.FindFileContainingSymbol( - prefix + 'protobuf_unittest.TestAllTypes.NestedEnum').name) - - @unittest.skipIf(api_implementation.Type() != 'python', - 'Only pure python allows _Add*()') - def testEnum(self): - self._TestEnum('') - self._TestEnum('.') - - @unittest.skipIf(api_implementation.Type() != 'python', - 'Only pure python allows _Add*()') - def testService(self): - pool = descriptor_pool.DescriptorPool() - with self.assertRaises(KeyError): - pool.FindServiceByName('protobuf_unittest.TestService') - pool._AddServiceDescriptor(unittest_pb2._TESTSERVICE) - self.assertEqual( - 'protobuf_unittest.TestService', - pool.FindServiceByName('protobuf_unittest.TestService').full_name) - - @unittest.skipIf(api_implementation.Type() != 'python', - 'Only pure python allows _Add*()') - def testFile(self): - pool = descriptor_pool.DescriptorPool() - pool._AddFileDescriptor(unittest_pb2.DESCRIPTOR) - self.assertEqual( - 'google/protobuf/unittest.proto', - pool.FindFileByName( - 'google/protobuf/unittest.proto').name) - - # AddFileDescriptor is not recursive; messages and enums within files must - # be explicitly registered. - with self.assertRaises(KeyError): - pool.FindFileContainingSymbol( - 'protobuf_unittest.TestAllTypes') - - def testEmptyDescriptorPool(self): - # Check that an empty DescriptorPool() contains no messages. - pool = descriptor_pool.DescriptorPool() - proto_file_name = descriptor_pb2.DESCRIPTOR.name - self.assertRaises(KeyError, pool.FindFileByName, proto_file_name) - # Add the above file to the pool - file_descriptor = descriptor_pb2.FileDescriptorProto() - descriptor_pb2.DESCRIPTOR.CopyToProto(file_descriptor) - pool.Add(file_descriptor) - # Now it exists. - self.assertTrue(pool.FindFileByName(proto_file_name)) - - def testCustomDescriptorPool(self): - # Create a new pool, and add a file descriptor. - pool = descriptor_pool.DescriptorPool() - file_desc = descriptor_pb2.FileDescriptorProto( - name='some/file.proto', package='package') - file_desc.message_type.add(name='Message') - pool.Add(file_desc) - self.assertEqual(pool.FindFileByName('some/file.proto').name, - 'some/file.proto') - self.assertEqual(pool.FindMessageTypeByName('package.Message').name, - 'Message') - # Test no package - file_proto = descriptor_pb2.FileDescriptorProto( - name='some/filename/container.proto') - message_proto = file_proto.message_type.add( - name='TopMessage') - message_proto.field.add( - name='bb', - number=1, - type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL) - enum_proto = file_proto.enum_type.add(name='TopEnum') - enum_proto.value.add(name='FOREIGN_FOO', number=4) - file_proto.service.add(name='TopService') - pool = descriptor_pool.DescriptorPool() - pool.Add(file_proto) - self.assertEqual('TopMessage', - pool.FindMessageTypeByName('TopMessage').name) - self.assertEqual('TopEnum', pool.FindEnumTypeByName('TopEnum').name) - self.assertEqual('TopService', pool.FindServiceByName('TopService').name) - - def testFileDescriptorOptionsWithCustomDescriptorPool(self): - # Create a descriptor pool, and add a new FileDescriptorProto to it. - pool = descriptor_pool.DescriptorPool() - file_name = 'file_descriptor_options_with_custom_descriptor_pool.proto' - file_descriptor_proto = descriptor_pb2.FileDescriptorProto(name=file_name) - extension_id = file_options_test_pb2.foo_options - file_descriptor_proto.options.Extensions[extension_id].foo_name = 'foo' - pool.Add(file_descriptor_proto) - # The options set on the FileDescriptorProto should be available in the - # descriptor even if they contain extensions that cannot be deserialized - # using the pool. - file_descriptor = pool.FindFileByName(file_name) - options = file_descriptor.GetOptions() - self.assertEqual('foo', options.Extensions[extension_id].foo_name) - # The object returned by GetOptions() is cached. - self.assertIs(options, file_descriptor.GetOptions()) - - def testAddTypeError(self): - pool = descriptor_pool.DescriptorPool() - if api_implementation.Type() != 'python': - with self.assertRaises(TypeError): - pool.AddDescriptor(0) - with self.assertRaises(TypeError): - pool.AddEnumDescriptor(0) - with self.assertRaises(TypeError): - pool.AddServiceDescriptor(0) - with self.assertRaises(TypeError): - pool.AddExtensionDescriptor(0) - with self.assertRaises(TypeError): - pool.AddFileDescriptor(0) - else: - with self.assertRaises(TypeError): - pool._AddDescriptor(0) - with self.assertRaises(TypeError): - pool._AddEnumDescriptor(0) - with self.assertRaises(TypeError): - pool._AddServiceDescriptor(0) - with self.assertRaises(TypeError): - pool._AddExtensionDescriptor(0) - with self.assertRaises(TypeError): - pool._AddFileDescriptor(0) - - -TEST1_FILE = ProtoFile( - 'google/protobuf/internal/descriptor_pool_test1.proto', - 'google.protobuf.python.internal', - { - 'DescriptorPoolTest1': MessageType({ - 'NestedEnum': EnumType([('ALPHA', 1), ('BETA', 2)]), - 'NestedMessage': MessageType({ - 'NestedEnum': EnumType([('EPSILON', 5), ('ZETA', 6)]), - 'DeepNestedMessage': MessageType({ - 'NestedEnum': EnumType([('ETA', 7), ('THETA', 8)]), - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'ETA')), - ('nested_field', StringField(2, 'theta')), - ]), - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'ZETA')), - ('nested_field', StringField(2, 'beta')), - ('deep_nested_message', MessageField(3, 'DeepNestedMessage')), - ]) - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'BETA')), - ('nested_message', MessageField(2, 'NestedMessage')), - ], is_extendable=True), - - 'DescriptorPoolTest2': MessageType({ - 'NestedEnum': EnumType([('GAMMA', 3), ('DELTA', 4)]), - 'NestedMessage': MessageType({ - 'NestedEnum': EnumType([('IOTA', 9), ('KAPPA', 10)]), - 'DeepNestedMessage': MessageType({ - 'NestedEnum': EnumType([('LAMBDA', 11), ('MU', 12)]), - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'MU')), - ('nested_field', StringField(2, 'lambda')), - ]), - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'IOTA')), - ('nested_field', StringField(2, 'delta')), - ('deep_nested_message', MessageField(3, 'DeepNestedMessage')), - ]) - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'GAMMA')), - ('nested_message', MessageField(2, 'NestedMessage')), - ]), - }) - - -TEST2_FILE = ProtoFile( - 'google/protobuf/internal/descriptor_pool_test2.proto', - 'google.protobuf.python.internal', - { - 'DescriptorPoolTest3': MessageType({ - 'NestedEnum': EnumType([('NU', 13), ('XI', 14)]), - 'NestedMessage': MessageType({ - 'NestedEnum': EnumType([('OMICRON', 15), ('PI', 16)]), - 'DeepNestedMessage': MessageType({ - 'NestedEnum': EnumType([('RHO', 17), ('SIGMA', 18)]), - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'RHO')), - ('nested_field', StringField(2, 'sigma')), - ]), - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'PI')), - ('nested_field', StringField(2, 'nu')), - ('deep_nested_message', MessageField(3, 'DeepNestedMessage')), - ]) - }, [ - ('nested_enum', EnumField(1, 'NestedEnum', 'XI')), - ('nested_message', MessageField(2, 'NestedMessage')), - ], extensions=[ - ('descriptor_pool_test', - ExtensionField(1001, 'DescriptorPoolTest1')), - ]), - }, - dependencies=['google/protobuf/internal/descriptor_pool_test1.proto', - 'google/protobuf/internal/more_messages.proto'], - public_dependencies=['google/protobuf/internal/more_messages.proto']) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/descriptor_test.py b/ext/protobuf/Python/google/protobuf/internal/descriptor_test.py deleted file mode 100644 index 6a8532c40..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/descriptor_test.py +++ /dev/null @@ -1,1076 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unittest for google.protobuf.internal.descriptor.""" - -__author__ = 'robinson@google.com (Will Robinson)' - -import unittest -import warnings - -from google.protobuf import unittest_custom_options_pb2 -from google.protobuf import unittest_import_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import descriptor_pb2 -from google.protobuf.internal import api_implementation -from google.protobuf.internal import test_util -from google.protobuf import descriptor -from google.protobuf import descriptor_pool -from google.protobuf import symbol_database -from google.protobuf import text_format - - -TEST_EMPTY_MESSAGE_DESCRIPTOR_ASCII = """ -name: 'TestEmptyMessage' -""" - -TEST_FILE_DESCRIPTOR_DEBUG = """syntax = "proto2"; - -package protobuf_unittest; - -message NestedMessage { - enum ForeignEnum { - FOREIGN_FOO = 4; - FOREIGN_BAR = 5; - FOREIGN_BAZ = 6; - } - optional int32 bb = 1; -} - -message ResponseMessage { -} - -service Service { - rpc CallMethod(.protobuf_unittest.NestedMessage) returns (.protobuf_unittest.ResponseMessage); -} - -""" - - -warnings.simplefilter('error', DeprecationWarning) - - -class DescriptorTest(unittest.TestCase): - - def setUp(self): - file_proto = descriptor_pb2.FileDescriptorProto( - name='some/filename/some.proto', - package='protobuf_unittest') - message_proto = file_proto.message_type.add( - name='NestedMessage') - message_proto.field.add( - name='bb', - number=1, - type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL) - enum_proto = message_proto.enum_type.add( - name='ForeignEnum') - enum_proto.value.add(name='FOREIGN_FOO', number=4) - enum_proto.value.add(name='FOREIGN_BAR', number=5) - enum_proto.value.add(name='FOREIGN_BAZ', number=6) - - file_proto.message_type.add(name='ResponseMessage') - service_proto = file_proto.service.add( - name='Service') - method_proto = service_proto.method.add( - name='CallMethod', - input_type='.protobuf_unittest.NestedMessage', - output_type='.protobuf_unittest.ResponseMessage') - - # Note: Calling DescriptorPool.Add() multiple times with the same file only - # works if the input is canonical; in particular, all type names must be - # fully qualified. - self.pool = self.GetDescriptorPool() - self.pool.Add(file_proto) - self.my_file = self.pool.FindFileByName(file_proto.name) - self.my_message = self.my_file.message_types_by_name[message_proto.name] - self.my_enum = self.my_message.enum_types_by_name[enum_proto.name] - self.my_service = self.my_file.services_by_name[service_proto.name] - self.my_method = self.my_service.methods_by_name[method_proto.name] - - def GetDescriptorPool(self): - return symbol_database.Default().pool - - def testEnumValueName(self): - self.assertEqual(self.my_message.EnumValueName('ForeignEnum', 4), - 'FOREIGN_FOO') - - self.assertEqual( - self.my_message.enum_types_by_name[ - 'ForeignEnum'].values_by_number[4].name, - self.my_message.EnumValueName('ForeignEnum', 4)) - with self.assertRaises(KeyError): - self.my_message.EnumValueName('ForeignEnum', 999) - with self.assertRaises(KeyError): - self.my_message.EnumValueName('NoneEnum', 999) - with self.assertRaises(TypeError): - self.my_message.EnumValueName() - - def testEnumFixups(self): - self.assertEqual(self.my_enum, self.my_enum.values[0].type) - - def testContainingTypeFixups(self): - self.assertEqual(self.my_message, self.my_message.fields[0].containing_type) - self.assertEqual(self.my_message, self.my_enum.containing_type) - - def testContainingServiceFixups(self): - self.assertEqual(self.my_service, self.my_method.containing_service) - - @unittest.skipIf( - api_implementation.Type() == 'python', - 'GetDebugString is only available with the cpp implementation', - ) - def testGetDebugString(self): - self.assertEqual(self.my_file.GetDebugString(), TEST_FILE_DESCRIPTOR_DEBUG) - - def testGetOptions(self): - self.assertEqual(self.my_enum.GetOptions(), - descriptor_pb2.EnumOptions()) - self.assertEqual(self.my_enum.values[0].GetOptions(), - descriptor_pb2.EnumValueOptions()) - self.assertEqual(self.my_message.GetOptions(), - descriptor_pb2.MessageOptions()) - self.assertEqual(self.my_message.fields[0].GetOptions(), - descriptor_pb2.FieldOptions()) - self.assertEqual(self.my_method.GetOptions(), - descriptor_pb2.MethodOptions()) - self.assertEqual(self.my_service.GetOptions(), - descriptor_pb2.ServiceOptions()) - - def testSimpleCustomOptions(self): - file_descriptor = unittest_custom_options_pb2.DESCRIPTOR - message_descriptor = (unittest_custom_options_pb2. - TestMessageWithCustomOptions.DESCRIPTOR) - field_descriptor = message_descriptor.fields_by_name['field1'] - oneof_descriptor = message_descriptor.oneofs_by_name['AnOneof'] - enum_descriptor = message_descriptor.enum_types_by_name['AnEnum'] - enum_value_descriptor = (message_descriptor. - enum_values_by_name['ANENUM_VAL2']) - other_enum_value_descriptor = (message_descriptor. - enum_values_by_name['ANENUM_VAL1']) - service_descriptor = (unittest_custom_options_pb2. - TestServiceWithCustomOptions.DESCRIPTOR) - method_descriptor = service_descriptor.FindMethodByName('Foo') - - file_options = file_descriptor.GetOptions() - file_opt1 = unittest_custom_options_pb2.file_opt1 - self.assertEqual(9876543210, file_options.Extensions[file_opt1]) - message_options = message_descriptor.GetOptions() - message_opt1 = unittest_custom_options_pb2.message_opt1 - self.assertEqual(-56, message_options.Extensions[message_opt1]) - field_options = field_descriptor.GetOptions() - field_opt1 = unittest_custom_options_pb2.field_opt1 - self.assertEqual(8765432109, field_options.Extensions[field_opt1]) - field_opt2 = unittest_custom_options_pb2.field_opt2 - self.assertEqual(42, field_options.Extensions[field_opt2]) - oneof_options = oneof_descriptor.GetOptions() - oneof_opt1 = unittest_custom_options_pb2.oneof_opt1 - self.assertEqual(-99, oneof_options.Extensions[oneof_opt1]) - enum_options = enum_descriptor.GetOptions() - enum_opt1 = unittest_custom_options_pb2.enum_opt1 - self.assertEqual(-789, enum_options.Extensions[enum_opt1]) - enum_value_options = enum_value_descriptor.GetOptions() - enum_value_opt1 = unittest_custom_options_pb2.enum_value_opt1 - self.assertEqual(123, enum_value_options.Extensions[enum_value_opt1]) - - service_options = service_descriptor.GetOptions() - service_opt1 = unittest_custom_options_pb2.service_opt1 - self.assertEqual(-9876543210, service_options.Extensions[service_opt1]) - method_options = method_descriptor.GetOptions() - method_opt1 = unittest_custom_options_pb2.method_opt1 - self.assertEqual(unittest_custom_options_pb2.METHODOPT1_VAL2, - method_options.Extensions[method_opt1]) - - message_descriptor = ( - unittest_custom_options_pb2.DummyMessageContainingEnum.DESCRIPTOR) - self.assertTrue(file_descriptor.has_options) - self.assertFalse(message_descriptor.has_options) - self.assertTrue(field_descriptor.has_options) - self.assertTrue(oneof_descriptor.has_options) - self.assertTrue(enum_descriptor.has_options) - self.assertTrue(enum_value_descriptor.has_options) - self.assertFalse(other_enum_value_descriptor.has_options) - - def testCustomOptionsCopyTo(self): - message_descriptor = (unittest_custom_options_pb2. - TestMessageWithCustomOptions.DESCRIPTOR) - message_proto = descriptor_pb2.DescriptorProto() - message_descriptor.CopyToProto(message_proto) - self.assertEqual(len(message_proto.options.ListFields()), - 2) - - def testDifferentCustomOptionTypes(self): - kint32min = -2**31 - kint64min = -2**63 - kint32max = 2**31 - 1 - kint64max = 2**63 - 1 - kuint32max = 2**32 - 1 - kuint64max = 2**64 - 1 - - message_descriptor =\ - unittest_custom_options_pb2.CustomOptionMinIntegerValues.DESCRIPTOR - message_options = message_descriptor.GetOptions() - self.assertEqual(False, message_options.Extensions[ - unittest_custom_options_pb2.bool_opt]) - self.assertEqual(kint32min, message_options.Extensions[ - unittest_custom_options_pb2.int32_opt]) - self.assertEqual(kint64min, message_options.Extensions[ - unittest_custom_options_pb2.int64_opt]) - self.assertEqual(0, message_options.Extensions[ - unittest_custom_options_pb2.uint32_opt]) - self.assertEqual(0, message_options.Extensions[ - unittest_custom_options_pb2.uint64_opt]) - self.assertEqual(kint32min, message_options.Extensions[ - unittest_custom_options_pb2.sint32_opt]) - self.assertEqual(kint64min, message_options.Extensions[ - unittest_custom_options_pb2.sint64_opt]) - self.assertEqual(0, message_options.Extensions[ - unittest_custom_options_pb2.fixed32_opt]) - self.assertEqual(0, message_options.Extensions[ - unittest_custom_options_pb2.fixed64_opt]) - self.assertEqual(kint32min, message_options.Extensions[ - unittest_custom_options_pb2.sfixed32_opt]) - self.assertEqual(kint64min, message_options.Extensions[ - unittest_custom_options_pb2.sfixed64_opt]) - - message_descriptor =\ - unittest_custom_options_pb2.CustomOptionMaxIntegerValues.DESCRIPTOR - message_options = message_descriptor.GetOptions() - self.assertEqual(True, message_options.Extensions[ - unittest_custom_options_pb2.bool_opt]) - self.assertEqual(kint32max, message_options.Extensions[ - unittest_custom_options_pb2.int32_opt]) - self.assertEqual(kint64max, message_options.Extensions[ - unittest_custom_options_pb2.int64_opt]) - self.assertEqual(kuint32max, message_options.Extensions[ - unittest_custom_options_pb2.uint32_opt]) - self.assertEqual(kuint64max, message_options.Extensions[ - unittest_custom_options_pb2.uint64_opt]) - self.assertEqual(kint32max, message_options.Extensions[ - unittest_custom_options_pb2.sint32_opt]) - self.assertEqual(kint64max, message_options.Extensions[ - unittest_custom_options_pb2.sint64_opt]) - self.assertEqual(kuint32max, message_options.Extensions[ - unittest_custom_options_pb2.fixed32_opt]) - self.assertEqual(kuint64max, message_options.Extensions[ - unittest_custom_options_pb2.fixed64_opt]) - self.assertEqual(kint32max, message_options.Extensions[ - unittest_custom_options_pb2.sfixed32_opt]) - self.assertEqual(kint64max, message_options.Extensions[ - unittest_custom_options_pb2.sfixed64_opt]) - - message_descriptor =\ - unittest_custom_options_pb2.CustomOptionOtherValues.DESCRIPTOR - message_options = message_descriptor.GetOptions() - self.assertEqual(-100, message_options.Extensions[ - unittest_custom_options_pb2.int32_opt]) - self.assertAlmostEqual(12.3456789, message_options.Extensions[ - unittest_custom_options_pb2.float_opt], 6) - self.assertAlmostEqual(1.234567890123456789, message_options.Extensions[ - unittest_custom_options_pb2.double_opt]) - self.assertEqual("Hello, \"World\"", message_options.Extensions[ - unittest_custom_options_pb2.string_opt]) - self.assertEqual(b"Hello\0World", message_options.Extensions[ - unittest_custom_options_pb2.bytes_opt]) - dummy_enum = unittest_custom_options_pb2.DummyMessageContainingEnum - self.assertEqual( - dummy_enum.TEST_OPTION_ENUM_TYPE2, - message_options.Extensions[unittest_custom_options_pb2.enum_opt]) - - message_descriptor =\ - unittest_custom_options_pb2.SettingRealsFromPositiveInts.DESCRIPTOR - message_options = message_descriptor.GetOptions() - self.assertAlmostEqual(12, message_options.Extensions[ - unittest_custom_options_pb2.float_opt], 6) - self.assertAlmostEqual(154, message_options.Extensions[ - unittest_custom_options_pb2.double_opt]) - - message_descriptor =\ - unittest_custom_options_pb2.SettingRealsFromNegativeInts.DESCRIPTOR - message_options = message_descriptor.GetOptions() - self.assertAlmostEqual(-12, message_options.Extensions[ - unittest_custom_options_pb2.float_opt], 6) - self.assertAlmostEqual(-154, message_options.Extensions[ - unittest_custom_options_pb2.double_opt]) - - def testComplexExtensionOptions(self): - descriptor =\ - unittest_custom_options_pb2.VariousComplexOptions.DESCRIPTOR - options = descriptor.GetOptions() - self.assertEqual(42, options.Extensions[ - unittest_custom_options_pb2.complex_opt1].foo) - self.assertEqual(324, options.Extensions[ - unittest_custom_options_pb2.complex_opt1].Extensions[ - unittest_custom_options_pb2.mooo]) - self.assertEqual(876, options.Extensions[ - unittest_custom_options_pb2.complex_opt1].Extensions[ - unittest_custom_options_pb2.corge].moo) - self.assertEqual(987, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].baz) - self.assertEqual(654, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].Extensions[ - unittest_custom_options_pb2.grault]) - self.assertEqual(743, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].bar.foo) - self.assertEqual(1999, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].bar.Extensions[ - unittest_custom_options_pb2.mooo]) - self.assertEqual(2008, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].bar.Extensions[ - unittest_custom_options_pb2.corge].moo) - self.assertEqual(741, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].Extensions[ - unittest_custom_options_pb2.garply].foo) - self.assertEqual(1998, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].Extensions[ - unittest_custom_options_pb2.garply].Extensions[ - unittest_custom_options_pb2.mooo]) - self.assertEqual(2121, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].Extensions[ - unittest_custom_options_pb2.garply].Extensions[ - unittest_custom_options_pb2.corge].moo) - self.assertEqual(1971, options.Extensions[ - unittest_custom_options_pb2.ComplexOptionType2 - .ComplexOptionType4.complex_opt4].waldo) - self.assertEqual(321, options.Extensions[ - unittest_custom_options_pb2.complex_opt2].fred.waldo) - self.assertEqual(9, options.Extensions[ - unittest_custom_options_pb2.complex_opt3].moo) - self.assertEqual(22, options.Extensions[ - unittest_custom_options_pb2.complex_opt3].complexoptiontype5.plugh) - self.assertEqual(24, options.Extensions[ - unittest_custom_options_pb2.complexopt6].xyzzy) - - # Check that aggregate options were parsed and saved correctly in - # the appropriate descriptors. - def testAggregateOptions(self): - file_descriptor = unittest_custom_options_pb2.DESCRIPTOR - message_descriptor =\ - unittest_custom_options_pb2.AggregateMessage.DESCRIPTOR - field_descriptor = message_descriptor.fields_by_name["fieldname"] - enum_descriptor = unittest_custom_options_pb2.AggregateEnum.DESCRIPTOR - enum_value_descriptor = enum_descriptor.values_by_name["VALUE"] - service_descriptor =\ - unittest_custom_options_pb2.AggregateService.DESCRIPTOR - method_descriptor = service_descriptor.FindMethodByName("Method") - - # Tests for the different types of data embedded in fileopt - file_options = file_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.fileopt] - self.assertEqual(100, file_options.i) - self.assertEqual("FileAnnotation", file_options.s) - self.assertEqual("NestedFileAnnotation", file_options.sub.s) - self.assertEqual("FileExtensionAnnotation", file_options.file.Extensions[ - unittest_custom_options_pb2.fileopt].s) - self.assertEqual("EmbeddedMessageSetElement", file_options.mset.Extensions[ - unittest_custom_options_pb2.AggregateMessageSetElement - .message_set_extension].s) - - # Simple tests for all the other types of annotations - self.assertEqual( - "MessageAnnotation", - message_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.msgopt].s) - self.assertEqual( - "FieldAnnotation", - field_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.fieldopt].s) - self.assertEqual( - "EnumAnnotation", - enum_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.enumopt].s) - self.assertEqual( - "EnumValueAnnotation", - enum_value_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.enumvalopt].s) - self.assertEqual( - "ServiceAnnotation", - service_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.serviceopt].s) - self.assertEqual( - "MethodAnnotation", - method_descriptor.GetOptions().Extensions[ - unittest_custom_options_pb2.methodopt].s) - - def testNestedOptions(self): - nested_message =\ - unittest_custom_options_pb2.NestedOptionType.NestedMessage.DESCRIPTOR - self.assertEqual(1001, nested_message.GetOptions().Extensions[ - unittest_custom_options_pb2.message_opt1]) - nested_field = nested_message.fields_by_name["nested_field"] - self.assertEqual(1002, nested_field.GetOptions().Extensions[ - unittest_custom_options_pb2.field_opt1]) - outer_message =\ - unittest_custom_options_pb2.NestedOptionType.DESCRIPTOR - nested_enum = outer_message.enum_types_by_name["NestedEnum"] - self.assertEqual(1003, nested_enum.GetOptions().Extensions[ - unittest_custom_options_pb2.enum_opt1]) - nested_enum_value = outer_message.enum_values_by_name["NESTED_ENUM_VALUE"] - self.assertEqual(1004, nested_enum_value.GetOptions().Extensions[ - unittest_custom_options_pb2.enum_value_opt1]) - nested_extension = outer_message.extensions_by_name["nested_extension"] - self.assertEqual(1005, nested_extension.GetOptions().Extensions[ - unittest_custom_options_pb2.field_opt2]) - - def testFileDescriptorReferences(self): - self.assertEqual(self.my_enum.file, self.my_file) - self.assertEqual(self.my_message.file, self.my_file) - - def testFileDescriptor(self): - self.assertEqual(self.my_file.name, 'some/filename/some.proto') - self.assertEqual(self.my_file.package, 'protobuf_unittest') - self.assertEqual(self.my_file.pool, self.pool) - self.assertFalse(self.my_file.has_options) - self.assertEqual('proto2', self.my_file.syntax) - file_proto = descriptor_pb2.FileDescriptorProto() - self.my_file.CopyToProto(file_proto) - self.assertEqual(self.my_file.serialized_pb, - file_proto.SerializeToString()) - # Generated modules also belong to the default pool. - self.assertEqual(unittest_pb2.DESCRIPTOR.pool, descriptor_pool.Default()) - - @unittest.skipIf( - api_implementation.Type() == 'python', - 'Immutability of descriptors is only enforced in v2 implementation') - def testImmutableCppDescriptor(self): - file_descriptor = unittest_pb2.DESCRIPTOR - message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - field_descriptor = message_descriptor.fields_by_name['optional_int32'] - enum_descriptor = message_descriptor.enum_types_by_name['NestedEnum'] - oneof_descriptor = message_descriptor.oneofs_by_name['oneof_field'] - with self.assertRaises(AttributeError): - message_descriptor.fields_by_name = None - with self.assertRaises(TypeError): - message_descriptor.fields_by_name['Another'] = None - with self.assertRaises(TypeError): - message_descriptor.fields.append(None) - with self.assertRaises(AttributeError): - field_descriptor.containing_type = message_descriptor - with self.assertRaises(AttributeError): - file_descriptor.has_options = False - with self.assertRaises(AttributeError): - field_descriptor.has_options = False - with self.assertRaises(AttributeError): - oneof_descriptor.has_options = False - with self.assertRaises(AttributeError): - enum_descriptor.has_options = False - with self.assertRaises(AttributeError) as e: - message_descriptor.has_options = True - self.assertEqual('attribute is not writable: has_options', - str(e.exception)) - - def testDefault(self): - message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - field = message_descriptor.fields_by_name['repeated_int32'] - self.assertEqual(field.default_value, []) - field = message_descriptor.fields_by_name['repeated_nested_message'] - self.assertEqual(field.default_value, []) - field = message_descriptor.fields_by_name['optionalgroup'] - self.assertEqual(field.default_value, None) - field = message_descriptor.fields_by_name['optional_nested_message'] - self.assertEqual(field.default_value, None) - - -class NewDescriptorTest(DescriptorTest): - """Redo the same tests as above, but with a separate DescriptorPool.""" - - def GetDescriptorPool(self): - return descriptor_pool.DescriptorPool() - - -class GeneratedDescriptorTest(unittest.TestCase): - """Tests for the properties of descriptors in generated code.""" - - def CheckMessageDescriptor(self, message_descriptor): - # Basic properties - self.assertEqual(message_descriptor.name, 'TestAllTypes') - self.assertEqual(message_descriptor.full_name, - 'protobuf_unittest.TestAllTypes') - # Test equality and hashability - self.assertEqual(message_descriptor, message_descriptor) - self.assertEqual(message_descriptor.fields[0].containing_type, - message_descriptor) - self.assertIn(message_descriptor, [message_descriptor]) - self.assertIn(message_descriptor, {message_descriptor: None}) - # Test field containers - self.CheckDescriptorSequence(message_descriptor.fields) - self.CheckDescriptorMapping(message_descriptor.fields_by_name) - self.CheckDescriptorMapping(message_descriptor.fields_by_number) - self.CheckDescriptorMapping(message_descriptor.fields_by_camelcase_name) - self.CheckDescriptorMapping(message_descriptor.enum_types_by_name) - self.CheckDescriptorMapping(message_descriptor.enum_values_by_name) - self.CheckDescriptorMapping(message_descriptor.oneofs_by_name) - self.CheckDescriptorMapping(message_descriptor.enum_types[0].values_by_name) - # Test extension range - self.assertEqual(message_descriptor.extension_ranges, []) - - def CheckFieldDescriptor(self, field_descriptor): - # Basic properties - self.assertEqual(field_descriptor.name, 'optional_int32') - self.assertEqual(field_descriptor.camelcase_name, 'optionalInt32') - self.assertEqual(field_descriptor.full_name, - 'protobuf_unittest.TestAllTypes.optional_int32') - self.assertEqual(field_descriptor.containing_type.name, 'TestAllTypes') - self.assertEqual(field_descriptor.file, unittest_pb2.DESCRIPTOR) - # Test equality and hashability - self.assertEqual(field_descriptor, field_descriptor) - self.assertEqual( - field_descriptor.containing_type.fields_by_name['optional_int32'], - field_descriptor) - self.assertEqual( - field_descriptor.containing_type.fields_by_camelcase_name[ - 'optionalInt32'], - field_descriptor) - self.assertIn(field_descriptor, [field_descriptor]) - self.assertIn(field_descriptor, {field_descriptor: None}) - self.assertEqual(None, field_descriptor.extension_scope) - self.assertEqual(None, field_descriptor.enum_type) - self.assertTrue(field_descriptor.has_presence) - if api_implementation.Type() == 'cpp': - # For test coverage only - self.assertEqual(field_descriptor.id, field_descriptor.id) - - def CheckDescriptorSequence(self, sequence): - # Verifies that a property like 'messageDescriptor.fields' has all the - # properties of an immutable abc.Sequence. - self.assertNotEqual(sequence, - unittest_pb2.TestAllExtensions.DESCRIPTOR.fields) - self.assertNotEqual(sequence, []) - self.assertNotEqual(sequence, 1) - self.assertFalse(sequence == 1) # Only for cpp test coverage - self.assertEqual(sequence, sequence) - expected_list = list(sequence) - self.assertEqual(expected_list, sequence) - self.assertGreater(len(sequence), 0) # Sized - self.assertEqual(len(sequence), len(expected_list)) # Iterable - self.assertEqual(sequence[len(sequence) -1], sequence[-1]) - item = sequence[0] - self.assertEqual(item, sequence[0]) - self.assertIn(item, sequence) # Container - self.assertEqual(sequence.index(item), 0) - self.assertEqual(sequence.count(item), 1) - other_item = unittest_pb2.NestedTestAllTypes.DESCRIPTOR.fields[0] - self.assertNotIn(other_item, sequence) - self.assertEqual(sequence.count(other_item), 0) - self.assertRaises(ValueError, sequence.index, other_item) - self.assertRaises(ValueError, sequence.index, []) - reversed_iterator = reversed(sequence) - self.assertEqual(list(reversed_iterator), list(sequence)[::-1]) - self.assertRaises(StopIteration, next, reversed_iterator) - expected_list[0] = 'change value' - self.assertNotEqual(expected_list, sequence) - # TODO(jieluo): Change __repr__ support for DescriptorSequence. - if api_implementation.Type() == 'python': - self.assertEqual(str(list(sequence)), str(sequence)) - else: - self.assertEqual(str(sequence)[0], '<') - - def CheckDescriptorMapping(self, mapping): - # Verifies that a property like 'messageDescriptor.fields' has all the - # properties of an immutable abc.Mapping. - self.assertNotEqual( - mapping, unittest_pb2.TestAllExtensions.DESCRIPTOR.fields_by_name) - self.assertNotEqual(mapping, {}) - self.assertNotEqual(mapping, 1) - self.assertFalse(mapping == 1) # Only for cpp test coverage - excepted_dict = dict(mapping.items()) - self.assertEqual(mapping, excepted_dict) - self.assertEqual(mapping, mapping) - self.assertGreater(len(mapping), 0) # Sized - self.assertEqual(len(mapping), len(excepted_dict)) # Iterable - key, item = next(iter(mapping.items())) - self.assertIn(key, mapping) # Container - self.assertEqual(mapping.get(key), item) - with self.assertRaises(TypeError): - mapping.get() - # TODO(jieluo): Fix python and cpp extension diff. - if api_implementation.Type() == 'python': - self.assertRaises(TypeError, mapping.get, []) - else: - self.assertEqual(None, mapping.get([])) - # keys(), iterkeys() &co - item = (next(iter(mapping.keys())), next(iter(mapping.values()))) - self.assertEqual(item, next(iter(mapping.items()))) - excepted_dict[key] = 'change value' - self.assertNotEqual(mapping, excepted_dict) - del excepted_dict[key] - excepted_dict['new_key'] = 'new' - self.assertNotEqual(mapping, excepted_dict) - self.assertRaises(KeyError, mapping.__getitem__, 'key_error') - self.assertRaises(KeyError, mapping.__getitem__, len(mapping) + 1) - # TODO(jieluo): Add __repr__ support for DescriptorMapping. - if api_implementation.Type() == 'python': - self.assertEqual(len(str(dict(mapping.items()))), len(str(mapping))) - else: - self.assertEqual(str(mapping)[0], '<') - - def testDescriptor(self): - message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - self.CheckMessageDescriptor(message_descriptor) - field_descriptor = message_descriptor.fields_by_name['optional_int32'] - self.CheckFieldDescriptor(field_descriptor) - field_descriptor = message_descriptor.fields_by_camelcase_name[ - 'optionalInt32'] - self.CheckFieldDescriptor(field_descriptor) - enum_descriptor = unittest_pb2.DESCRIPTOR.enum_types_by_name[ - 'ForeignEnum'] - self.assertEqual(None, enum_descriptor.containing_type) - # Test extension range - self.assertEqual( - unittest_pb2.TestAllExtensions.DESCRIPTOR.extension_ranges, - [(1, 536870912)]) - self.assertEqual( - unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR.extension_ranges, - [(42, 43), (4143, 4244), (65536, 536870912)]) - - def testCppDescriptorContainer(self): - containing_file = unittest_pb2.DESCRIPTOR - self.CheckDescriptorSequence(containing_file.dependencies) - self.CheckDescriptorMapping(containing_file.message_types_by_name) - self.CheckDescriptorMapping(containing_file.enum_types_by_name) - self.CheckDescriptorMapping(containing_file.services_by_name) - self.CheckDescriptorMapping(containing_file.extensions_by_name) - self.CheckDescriptorMapping( - unittest_pb2.TestNestedExtension.DESCRIPTOR.extensions_by_name) - - def testCppDescriptorContainer_Iterator(self): - # Same test with the iterator - enum = unittest_pb2.TestAllTypes.DESCRIPTOR.enum_types_by_name['NestedEnum'] - values_iter = iter(enum.values) - del enum - self.assertEqual('FOO', next(values_iter).name) - - def testDescriptorNestedTypesContainer(self): - message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - nested_message_descriptor = unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR - self.assertEqual(len(message_descriptor.nested_types), 3) - self.assertFalse(None in message_descriptor.nested_types) - self.assertTrue( - nested_message_descriptor in message_descriptor.nested_types) - - def testServiceDescriptor(self): - service_descriptor = unittest_pb2.DESCRIPTOR.services_by_name['TestService'] - self.assertEqual(service_descriptor.name, 'TestService') - self.assertEqual(service_descriptor.methods[0].name, 'Foo') - self.assertIs(service_descriptor.file, unittest_pb2.DESCRIPTOR) - self.assertEqual(service_descriptor.index, 0) - self.CheckDescriptorMapping(service_descriptor.methods_by_name) - - def testOneofDescriptor(self): - message_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - oneof_descriptor = message_descriptor.oneofs_by_name['oneof_field'] - self.assertFalse(oneof_descriptor.has_options) - self.assertEqual(message_descriptor, oneof_descriptor.containing_type) - self.assertEqual('oneof_field', oneof_descriptor.name) - self.assertEqual('protobuf_unittest.TestAllTypes.oneof_field', - oneof_descriptor.full_name) - self.assertEqual(0, oneof_descriptor.index) - - -class DescriptorCopyToProtoTest(unittest.TestCase): - """Tests for CopyTo functions of Descriptor.""" - - def _AssertProtoEqual(self, actual_proto, expected_class, expected_ascii): - expected_proto = expected_class() - text_format.Merge(expected_ascii, expected_proto) - - self.assertEqual( - actual_proto, expected_proto, - 'Not equal,\nActual:\n%s\nExpected:\n%s\n' - % (str(actual_proto), str(expected_proto))) - - def _InternalTestCopyToProto(self, desc, expected_proto_class, - expected_proto_ascii): - actual = expected_proto_class() - desc.CopyToProto(actual) - self._AssertProtoEqual( - actual, expected_proto_class, expected_proto_ascii) - - def testCopyToProto_EmptyMessage(self): - self._InternalTestCopyToProto( - unittest_pb2.TestEmptyMessage.DESCRIPTOR, - descriptor_pb2.DescriptorProto, - TEST_EMPTY_MESSAGE_DESCRIPTOR_ASCII) - - def testCopyToProto_NestedMessage(self): - TEST_NESTED_MESSAGE_ASCII = """ - name: 'NestedMessage' - field: < - name: 'bb' - number: 1 - label: 1 # Optional - type: 5 # TYPE_INT32 - > - """ - - self._InternalTestCopyToProto( - unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR, - descriptor_pb2.DescriptorProto, - TEST_NESTED_MESSAGE_ASCII) - - def testCopyToProto_ForeignNestedMessage(self): - TEST_FOREIGN_NESTED_ASCII = """ - name: 'TestForeignNested' - field: < - name: 'foreign_nested' - number: 1 - label: 1 # Optional - type: 11 # TYPE_MESSAGE - type_name: '.protobuf_unittest.TestAllTypes.NestedMessage' - > - """ - - self._InternalTestCopyToProto( - unittest_pb2.TestForeignNested.DESCRIPTOR, - descriptor_pb2.DescriptorProto, - TEST_FOREIGN_NESTED_ASCII) - - def testCopyToProto_ForeignEnum(self): - TEST_FOREIGN_ENUM_ASCII = """ - name: 'ForeignEnum' - value: < - name: 'FOREIGN_FOO' - number: 4 - > - value: < - name: 'FOREIGN_BAR' - number: 5 - > - value: < - name: 'FOREIGN_BAZ' - number: 6 - > - """ - - self._InternalTestCopyToProto( - unittest_pb2.ForeignEnum.DESCRIPTOR, - descriptor_pb2.EnumDescriptorProto, - TEST_FOREIGN_ENUM_ASCII) - - def testCopyToProto_Options(self): - TEST_DEPRECATED_FIELDS_ASCII = """ - name: 'TestDeprecatedFields' - field: < - name: 'deprecated_int32' - number: 1 - label: 1 # Optional - type: 5 # TYPE_INT32 - options: < - deprecated: true - > - > - field { - name: "deprecated_int32_in_oneof" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_INT32 - options { - deprecated: true - } - oneof_index: 0 - } - oneof_decl { - name: "oneof_fields" - } - """ - - self._InternalTestCopyToProto( - unittest_pb2.TestDeprecatedFields.DESCRIPTOR, - descriptor_pb2.DescriptorProto, - TEST_DEPRECATED_FIELDS_ASCII) - - def testCopyToProto_AllExtensions(self): - TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII = """ - name: 'TestEmptyMessageWithExtensions' - extension_range: < - start: 1 - end: 536870912 - > - """ - - self._InternalTestCopyToProto( - unittest_pb2.TestEmptyMessageWithExtensions.DESCRIPTOR, - descriptor_pb2.DescriptorProto, - TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII) - - def testCopyToProto_SeveralExtensions(self): - TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII = """ - name: 'TestMultipleExtensionRanges' - extension_range: < - start: 42 - end: 43 - > - extension_range: < - start: 4143 - end: 4244 - > - extension_range: < - start: 65536 - end: 536870912 - > - """ - - self._InternalTestCopyToProto( - unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR, - descriptor_pb2.DescriptorProto, - TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII) - - def testCopyToProto_FileDescriptor(self): - UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = (""" - name: 'google/protobuf/unittest_import.proto' - package: 'protobuf_unittest_import' - dependency: 'google/protobuf/unittest_import_public.proto' - message_type: < - name: 'ImportMessage' - field: < - name: 'd' - number: 1 - label: 1 # Optional - type: 5 # TYPE_INT32 - > - > - """ + - """enum_type: < - name: 'ImportEnum' - value: < - name: 'IMPORT_FOO' - number: 7 - > - value: < - name: 'IMPORT_BAR' - number: 8 - > - value: < - name: 'IMPORT_BAZ' - number: 9 - > - > - enum_type: < - name: 'ImportEnumForMap' - value: < - name: 'UNKNOWN' - number: 0 - > - value: < - name: 'FOO' - number: 1 - > - value: < - name: 'BAR' - number: 2 - > - > - options: < - java_package: 'com.google.protobuf.test' - optimize_for: 1 # SPEED - """ + - """ - cc_enable_arenas: true - > - public_dependency: 0 - """) - self._InternalTestCopyToProto( - unittest_import_pb2.DESCRIPTOR, - descriptor_pb2.FileDescriptorProto, - UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII) - - def testCopyToProto_ServiceDescriptor(self): - TEST_SERVICE_ASCII = """ - name: 'TestService' - method: < - name: 'Foo' - input_type: '.protobuf_unittest.FooRequest' - output_type: '.protobuf_unittest.FooResponse' - > - method: < - name: 'Bar' - input_type: '.protobuf_unittest.BarRequest' - output_type: '.protobuf_unittest.BarResponse' - > - """ - self._InternalTestCopyToProto( - unittest_pb2.TestService.DESCRIPTOR, - descriptor_pb2.ServiceDescriptorProto, - TEST_SERVICE_ASCII) - - def testCopyToProto_MethodDescriptor(self): - expected_ascii = """ - name: 'Foo' - input_type: '.protobuf_unittest.FooRequest' - output_type: '.protobuf_unittest.FooResponse' - """ - method_descriptor = unittest_pb2.TestService.DESCRIPTOR.FindMethodByName( - 'Foo') - self._InternalTestCopyToProto( - method_descriptor, - descriptor_pb2.MethodDescriptorProto, - expected_ascii) - - @unittest.skipIf( - api_implementation.Type() == 'python', - 'Pure python does not raise error.') - # TODO(jieluo): Fix pure python to check with the proto type. - def testCopyToProto_TypeError(self): - file_proto = descriptor_pb2.FileDescriptorProto() - self.assertRaises(TypeError, - unittest_pb2.TestEmptyMessage.DESCRIPTOR.CopyToProto, - file_proto) - self.assertRaises(TypeError, - unittest_pb2.ForeignEnum.DESCRIPTOR.CopyToProto, - file_proto) - self.assertRaises(TypeError, - unittest_pb2.TestService.DESCRIPTOR.CopyToProto, - file_proto) - proto = descriptor_pb2.DescriptorProto() - self.assertRaises(TypeError, - unittest_import_pb2.DESCRIPTOR.CopyToProto, - proto) - - -class MakeDescriptorTest(unittest.TestCase): - - def testMakeDescriptorWithNestedFields(self): - file_descriptor_proto = descriptor_pb2.FileDescriptorProto() - file_descriptor_proto.name = 'Foo2' - message_type = file_descriptor_proto.message_type.add() - message_type.name = file_descriptor_proto.name - nested_type = message_type.nested_type.add() - nested_type.name = 'Sub' - enum_type = nested_type.enum_type.add() - enum_type.name = 'FOO' - enum_type_val = enum_type.value.add() - enum_type_val.name = 'BAR' - enum_type_val.number = 3 - field = message_type.field.add() - field.number = 1 - field.name = 'uint64_field' - field.label = descriptor.FieldDescriptor.LABEL_REQUIRED - field.type = descriptor.FieldDescriptor.TYPE_UINT64 - field = message_type.field.add() - field.number = 2 - field.name = 'nested_message_field' - field.label = descriptor.FieldDescriptor.LABEL_REQUIRED - field.type = descriptor.FieldDescriptor.TYPE_MESSAGE - field.type_name = 'Sub' - enum_field = nested_type.field.add() - enum_field.number = 2 - enum_field.name = 'bar_field' - enum_field.label = descriptor.FieldDescriptor.LABEL_REQUIRED - enum_field.type = descriptor.FieldDescriptor.TYPE_ENUM - enum_field.type_name = 'Foo2.Sub.FOO' - - result = descriptor.MakeDescriptor(message_type) - self.assertEqual(result.fields[0].cpp_type, - descriptor.FieldDescriptor.CPPTYPE_UINT64) - self.assertEqual(result.fields[1].cpp_type, - descriptor.FieldDescriptor.CPPTYPE_MESSAGE) - self.assertEqual(result.fields[1].message_type.containing_type, - result) - self.assertEqual(result.nested_types[0].fields[0].full_name, - 'Foo2.Sub.bar_field') - self.assertEqual(result.nested_types[0].fields[0].enum_type, - result.nested_types[0].enum_types[0]) - self.assertFalse(result.has_options) - self.assertFalse(result.fields[0].has_options) - if api_implementation.Type() == 'cpp': - with self.assertRaises(AttributeError): - result.fields[0].has_options = False - - def testMakeDescriptorWithUnsignedIntField(self): - file_descriptor_proto = descriptor_pb2.FileDescriptorProto() - file_descriptor_proto.name = 'Foo' - message_type = file_descriptor_proto.message_type.add() - message_type.name = file_descriptor_proto.name - enum_type = message_type.enum_type.add() - enum_type.name = 'FOO' - enum_type_val = enum_type.value.add() - enum_type_val.name = 'BAR' - enum_type_val.number = 3 - field = message_type.field.add() - field.number = 1 - field.name = 'uint64_field' - field.label = descriptor.FieldDescriptor.LABEL_REQUIRED - field.type = descriptor.FieldDescriptor.TYPE_UINT64 - enum_field = message_type.field.add() - enum_field.number = 2 - enum_field.name = 'bar_field' - enum_field.label = descriptor.FieldDescriptor.LABEL_REQUIRED - enum_field.type = descriptor.FieldDescriptor.TYPE_ENUM - enum_field.type_name = 'Foo.FOO' - - result = descriptor.MakeDescriptor(message_type) - self.assertEqual(result.fields[0].cpp_type, - descriptor.FieldDescriptor.CPPTYPE_UINT64) - - - def testMakeDescriptorWithOptions(self): - descriptor_proto = descriptor_pb2.DescriptorProto() - aggregate_message = unittest_custom_options_pb2.AggregateMessage - aggregate_message.DESCRIPTOR.CopyToProto(descriptor_proto) - reformed_descriptor = descriptor.MakeDescriptor(descriptor_proto) - - options = reformed_descriptor.GetOptions() - self.assertEqual(101, - options.Extensions[unittest_custom_options_pb2.msgopt].i) - - def testCamelcaseName(self): - descriptor_proto = descriptor_pb2.DescriptorProto() - descriptor_proto.name = 'Bar' - names = ['foo_foo', 'FooBar', 'fooBaz', 'fooFoo', 'foobar'] - camelcase_names = ['fooFoo', 'fooBar', 'fooBaz', 'fooFoo', 'foobar'] - for index in range(len(names)): - field = descriptor_proto.field.add() - field.number = index + 1 - field.name = names[index] - result = descriptor.MakeDescriptor(descriptor_proto) - for index in range(len(camelcase_names)): - self.assertEqual(result.fields[index].camelcase_name, - camelcase_names[index]) - - def testJsonName(self): - descriptor_proto = descriptor_pb2.DescriptorProto() - descriptor_proto.name = 'TestJsonName' - names = ['field_name', 'fieldName', 'FieldName', - '_field_name', 'FIELD_NAME', 'json_name'] - json_names = ['fieldName', 'fieldName', 'FieldName', - 'FieldName', 'FIELDNAME', '@type'] - for index in range(len(names)): - field = descriptor_proto.field.add() - field.number = index + 1 - field.name = names[index] - field.json_name = '@type' - result = descriptor.MakeDescriptor(descriptor_proto) - for index in range(len(json_names)): - self.assertEqual(result.fields[index].json_name, - json_names[index]) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/extension_dict.py b/ext/protobuf/Python/google/protobuf/internal/extension_dict.py index b346cf283..83c4cb5dc 100644 --- a/ext/protobuf/Python/google/protobuf/internal/extension_dict.py +++ b/ext/protobuf/Python/google/protobuf/internal/extension_dict.py @@ -89,8 +89,9 @@ def __getitem__(self, extension_handle): elif extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: message_type = extension_handle.message_type if not hasattr(message_type, '_concrete_class'): - # pylint: disable=protected-access - self._extended_message._FACTORY.GetPrototype(message_type) + # pylint: disable=g-import-not-at-top + from google.protobuf import message_factory + message_factory.GetMessageClass(message_type) assert getattr(extension_handle.message_type, '_concrete_class', None), ( 'Uninitialized concrete class found for field %r (message type %r)' % (extension_handle.full_name, diff --git a/ext/protobuf/Python/google/protobuf/internal/field_mask.py b/ext/protobuf/Python/google/protobuf/internal/field_mask.py new file mode 100644 index 000000000..489769901 --- /dev/null +++ b/ext/protobuf/Python/google/protobuf/internal/field_mask.py @@ -0,0 +1,333 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains FieldMask class.""" + +from google.protobuf.descriptor import FieldDescriptor + + +class FieldMask(object): + """Class for FieldMask message type.""" + + __slots__ = () + + def ToJsonString(self): + """Converts FieldMask to string according to proto3 JSON spec.""" + camelcase_paths = [] + for path in self.paths: + camelcase_paths.append(_SnakeCaseToCamelCase(path)) + return ','.join(camelcase_paths) + + def FromJsonString(self, value): + """Converts string to FieldMask according to proto3 JSON spec.""" + if not isinstance(value, str): + raise ValueError('FieldMask JSON value not a string: {!r}'.format(value)) + self.Clear() + if value: + for path in value.split(','): + self.paths.append(_CamelCaseToSnakeCase(path)) + + def IsValidForDescriptor(self, message_descriptor): + """Checks whether the FieldMask is valid for Message Descriptor.""" + for path in self.paths: + if not _IsValidPath(message_descriptor, path): + return False + return True + + def AllFieldsFromDescriptor(self, message_descriptor): + """Gets all direct fields of Message Descriptor to FieldMask.""" + self.Clear() + for field in message_descriptor.fields: + self.paths.append(field.name) + + def CanonicalFormFromMask(self, mask): + """Converts a FieldMask to the canonical form. + + Removes paths that are covered by another path. For example, + "foo.bar" is covered by "foo" and will be removed if "foo" + is also in the FieldMask. Then sorts all paths in alphabetical order. + + Args: + mask: The original FieldMask to be converted. + """ + tree = _FieldMaskTree(mask) + tree.ToFieldMask(self) + + def Union(self, mask1, mask2): + """Merges mask1 and mask2 into this FieldMask.""" + _CheckFieldMaskMessage(mask1) + _CheckFieldMaskMessage(mask2) + tree = _FieldMaskTree(mask1) + tree.MergeFromFieldMask(mask2) + tree.ToFieldMask(self) + + def Intersect(self, mask1, mask2): + """Intersects mask1 and mask2 into this FieldMask.""" + _CheckFieldMaskMessage(mask1) + _CheckFieldMaskMessage(mask2) + tree = _FieldMaskTree(mask1) + intersection = _FieldMaskTree() + for path in mask2.paths: + tree.IntersectPath(path, intersection) + intersection.ToFieldMask(self) + + def MergeMessage( + self, source, destination, + replace_message_field=False, replace_repeated_field=False): + """Merges fields specified in FieldMask from source to destination. + + Args: + source: Source message. + destination: The destination message to be merged into. + replace_message_field: Replace message field if True. Merge message + field if False. + replace_repeated_field: Replace repeated field if True. Append + elements of repeated field if False. + """ + tree = _FieldMaskTree(self) + tree.MergeMessage( + source, destination, replace_message_field, replace_repeated_field) + + +def _IsValidPath(message_descriptor, path): + """Checks whether the path is valid for Message Descriptor.""" + parts = path.split('.') + last = parts.pop() + for name in parts: + field = message_descriptor.fields_by_name.get(name) + if (field is None or + field.label == FieldDescriptor.LABEL_REPEATED or + field.type != FieldDescriptor.TYPE_MESSAGE): + return False + message_descriptor = field.message_type + return last in message_descriptor.fields_by_name + + +def _CheckFieldMaskMessage(message): + """Raises ValueError if message is not a FieldMask.""" + message_descriptor = message.DESCRIPTOR + if (message_descriptor.name != 'FieldMask' or + message_descriptor.file.name != 'google/protobuf/field_mask.proto'): + raise ValueError('Message {0} is not a FieldMask.'.format( + message_descriptor.full_name)) + + +def _SnakeCaseToCamelCase(path_name): + """Converts a path name from snake_case to camelCase.""" + result = [] + after_underscore = False + for c in path_name: + if c.isupper(): + raise ValueError( + 'Fail to print FieldMask to Json string: Path name ' + '{0} must not contain uppercase letters.'.format(path_name)) + if after_underscore: + if c.islower(): + result.append(c.upper()) + after_underscore = False + else: + raise ValueError( + 'Fail to print FieldMask to Json string: The ' + 'character after a "_" must be a lowercase letter ' + 'in path name {0}.'.format(path_name)) + elif c == '_': + after_underscore = True + else: + result += c + + if after_underscore: + raise ValueError('Fail to print FieldMask to Json string: Trailing "_" ' + 'in path name {0}.'.format(path_name)) + return ''.join(result) + + +def _CamelCaseToSnakeCase(path_name): + """Converts a field name from camelCase to snake_case.""" + result = [] + for c in path_name: + if c == '_': + raise ValueError('Fail to parse FieldMask: Path name ' + '{0} must not contain "_"s.'.format(path_name)) + if c.isupper(): + result += '_' + result += c.lower() + else: + result += c + return ''.join(result) + + +class _FieldMaskTree(object): + """Represents a FieldMask in a tree structure. + + For example, given a FieldMask "foo.bar,foo.baz,bar.baz", + the FieldMaskTree will be: + [_root] -+- foo -+- bar + | | + | +- baz + | + +- bar --- baz + In the tree, each leaf node represents a field path. + """ + + __slots__ = ('_root',) + + def __init__(self, field_mask=None): + """Initializes the tree by FieldMask.""" + self._root = {} + if field_mask: + self.MergeFromFieldMask(field_mask) + + def MergeFromFieldMask(self, field_mask): + """Merges a FieldMask to the tree.""" + for path in field_mask.paths: + self.AddPath(path) + + def AddPath(self, path): + """Adds a field path into the tree. + + If the field path to add is a sub-path of an existing field path + in the tree (i.e., a leaf node), it means the tree already matches + the given path so nothing will be added to the tree. If the path + matches an existing non-leaf node in the tree, that non-leaf node + will be turned into a leaf node with all its children removed because + the path matches all the node's children. Otherwise, a new path will + be added. + + Args: + path: The field path to add. + """ + node = self._root + for name in path.split('.'): + if name not in node: + node[name] = {} + elif not node[name]: + # Pre-existing empty node implies we already have this entire tree. + return + node = node[name] + # Remove any sub-trees we might have had. + node.clear() + + def ToFieldMask(self, field_mask): + """Converts the tree to a FieldMask.""" + field_mask.Clear() + _AddFieldPaths(self._root, '', field_mask) + + def IntersectPath(self, path, intersection): + """Calculates the intersection part of a field path with this tree. + + Args: + path: The field path to calculates. + intersection: The out tree to record the intersection part. + """ + node = self._root + for name in path.split('.'): + if name not in node: + return + elif not node[name]: + intersection.AddPath(path) + return + node = node[name] + intersection.AddLeafNodes(path, node) + + def AddLeafNodes(self, prefix, node): + """Adds leaf nodes begin with prefix to this tree.""" + if not node: + self.AddPath(prefix) + for name in node: + child_path = prefix + '.' + name + self.AddLeafNodes(child_path, node[name]) + + def MergeMessage( + self, source, destination, + replace_message, replace_repeated): + """Merge all fields specified by this tree from source to destination.""" + _MergeMessage( + self._root, source, destination, replace_message, replace_repeated) + + +def _StrConvert(value): + """Converts value to str if it is not.""" + # This file is imported by c extension and some methods like ClearField + # requires string for the field name. py2/py3 has different text + # type and may use unicode. + if not isinstance(value, str): + return value.encode('utf-8') + return value + + +def _MergeMessage( + node, source, destination, replace_message, replace_repeated): + """Merge all fields specified by a sub-tree from source to destination.""" + source_descriptor = source.DESCRIPTOR + for name in node: + child = node[name] + field = source_descriptor.fields_by_name[name] + if field is None: + raise ValueError('Error: Can\'t find field {0} in message {1}.'.format( + name, source_descriptor.full_name)) + if child: + # Sub-paths are only allowed for singular message fields. + if (field.label == FieldDescriptor.LABEL_REPEATED or + field.cpp_type != FieldDescriptor.CPPTYPE_MESSAGE): + raise ValueError('Error: Field {0} in message {1} is not a singular ' + 'message field and cannot have sub-fields.'.format( + name, source_descriptor.full_name)) + if source.HasField(name): + _MergeMessage( + child, getattr(source, name), getattr(destination, name), + replace_message, replace_repeated) + continue + if field.label == FieldDescriptor.LABEL_REPEATED: + if replace_repeated: + destination.ClearField(_StrConvert(name)) + repeated_source = getattr(source, name) + repeated_destination = getattr(destination, name) + repeated_destination.MergeFrom(repeated_source) + else: + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + if replace_message: + destination.ClearField(_StrConvert(name)) + if source.HasField(name): + getattr(destination, name).MergeFrom(getattr(source, name)) + else: + setattr(destination, name, getattr(source, name)) + + +def _AddFieldPaths(node, prefix, field_mask): + """Adds the field paths descended from node to field_mask.""" + if not node and prefix: + field_mask.paths.append(prefix) + return + for name in sorted(node): + if prefix: + child_path = prefix + '.' + name + else: + child_path = name + _AddFieldPaths(node[name], child_path, field_mask) diff --git a/ext/protobuf/Python/google/protobuf/internal/generator_test.py b/ext/protobuf/Python/google/protobuf/internal/generator_test.py deleted file mode 100644 index 9883fce31..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/generator_test.py +++ /dev/null @@ -1,354 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# TODO(robinson): Flesh this out considerably. We focused on reflection_test.py -# first, since it's testing the subtler code, and since it provides decent -# indirect testing of the protocol compiler output. - -"""Unittest that directly tests the output of the pure-Python protocol -compiler. See //google/protobuf/internal/reflection_test.py for a test which -further ensures that we can use Python protocol message objects as we expect. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - -import unittest - -from google.protobuf.internal import test_bad_identifiers_pb2 -from google.protobuf import unittest_custom_options_pb2 -from google.protobuf import unittest_import_pb2 -from google.protobuf import unittest_import_public_pb2 -from google.protobuf import unittest_mset_pb2 -from google.protobuf import unittest_mset_wire_format_pb2 -from google.protobuf import unittest_no_generic_services_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import service -from google.protobuf import symbol_database - -MAX_EXTENSION = 536870912 - - -class GeneratorTest(unittest.TestCase): - - def testNestedMessageDescriptor(self): - field_name = 'optional_nested_message' - proto_type = unittest_pb2.TestAllTypes - self.assertEqual( - proto_type.NestedMessage.DESCRIPTOR, - proto_type.DESCRIPTOR.fields_by_name[field_name].message_type) - - def testEnums(self): - # We test only module-level enums here. - # TODO(robinson): Examine descriptors directly to check - # enum descriptor output. - self.assertEqual(4, unittest_pb2.FOREIGN_FOO) - self.assertEqual(5, unittest_pb2.FOREIGN_BAR) - self.assertEqual(6, unittest_pb2.FOREIGN_BAZ) - - proto = unittest_pb2.TestAllTypes() - self.assertEqual(1, proto.FOO) - self.assertEqual(1, unittest_pb2.TestAllTypes.FOO) - self.assertEqual(2, proto.BAR) - self.assertEqual(2, unittest_pb2.TestAllTypes.BAR) - self.assertEqual(3, proto.BAZ) - self.assertEqual(3, unittest_pb2.TestAllTypes.BAZ) - - def testExtremeDefaultValues(self): - message = unittest_pb2.TestExtremeDefaultValues() - - # Python pre-2.6 does not have isinf() or isnan() functions, so we have - # to provide our own. - def isnan(val): - # NaN is never equal to itself. - return val != val - def isinf(val): - # Infinity times zero equals NaN. - return not isnan(val) and isnan(val * 0) - - self.assertTrue(isinf(message.inf_double)) - self.assertTrue(message.inf_double > 0) - self.assertTrue(isinf(message.neg_inf_double)) - self.assertTrue(message.neg_inf_double < 0) - self.assertTrue(isnan(message.nan_double)) - - self.assertTrue(isinf(message.inf_float)) - self.assertTrue(message.inf_float > 0) - self.assertTrue(isinf(message.neg_inf_float)) - self.assertTrue(message.neg_inf_float < 0) - self.assertTrue(isnan(message.nan_float)) - self.assertEqual("? ? ?? ?? ??? ??/ ??-", message.cpp_trigraph) - - def testHasDefaultValues(self): - desc = unittest_pb2.TestAllTypes.DESCRIPTOR - - expected_has_default_by_name = { - 'optional_int32': False, - 'repeated_int32': False, - 'optional_nested_message': False, - 'default_int32': True, - } - - has_default_by_name = dict( - [(f.name, f.has_default_value) - for f in desc.fields - if f.name in expected_has_default_by_name]) - self.assertEqual(expected_has_default_by_name, has_default_by_name) - - def testContainingTypeBehaviorForExtensions(self): - self.assertEqual(unittest_pb2.optional_int32_extension.containing_type, - unittest_pb2.TestAllExtensions.DESCRIPTOR) - self.assertEqual(unittest_pb2.TestRequired.single.containing_type, - unittest_pb2.TestAllExtensions.DESCRIPTOR) - - def testExtensionScope(self): - self.assertEqual(unittest_pb2.optional_int32_extension.extension_scope, - None) - self.assertEqual(unittest_pb2.TestRequired.single.extension_scope, - unittest_pb2.TestRequired.DESCRIPTOR) - - def testIsExtension(self): - self.assertTrue(unittest_pb2.optional_int32_extension.is_extension) - self.assertTrue(unittest_pb2.TestRequired.single.is_extension) - - message_descriptor = unittest_pb2.TestRequired.DESCRIPTOR - non_extension_descriptor = message_descriptor.fields_by_name['a'] - self.assertTrue(not non_extension_descriptor.is_extension) - - def testOptions(self): - proto = unittest_mset_wire_format_pb2.TestMessageSet() - self.assertTrue(proto.DESCRIPTOR.GetOptions().message_set_wire_format) - - def testMessageWithCustomOptions(self): - proto = unittest_custom_options_pb2.TestMessageWithCustomOptions() - enum_options = proto.DESCRIPTOR.enum_types_by_name['AnEnum'].GetOptions() - self.assertTrue(enum_options is not None) - # TODO(gps): We really should test for the presence of the enum_opt1 - # extension and for its value to be set to -789. - - def testNestedTypes(self): - self.assertEqual( - set(unittest_pb2.TestAllTypes.DESCRIPTOR.nested_types), - set([ - unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR, - unittest_pb2.TestAllTypes.OptionalGroup.DESCRIPTOR, - unittest_pb2.TestAllTypes.RepeatedGroup.DESCRIPTOR, - ])) - self.assertEqual(unittest_pb2.TestEmptyMessage.DESCRIPTOR.nested_types, []) - self.assertEqual( - unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.nested_types, []) - - def testContainingType(self): - self.assertTrue( - unittest_pb2.TestEmptyMessage.DESCRIPTOR.containing_type is None) - self.assertTrue( - unittest_pb2.TestAllTypes.DESCRIPTOR.containing_type is None) - self.assertEqual( - unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.containing_type, - unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertEqual( - unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.containing_type, - unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertEqual( - unittest_pb2.TestAllTypes.RepeatedGroup.DESCRIPTOR.containing_type, - unittest_pb2.TestAllTypes.DESCRIPTOR) - - def testContainingTypeInEnumDescriptor(self): - self.assertTrue(unittest_pb2._FOREIGNENUM.containing_type is None) - self.assertEqual(unittest_pb2._TESTALLTYPES_NESTEDENUM.containing_type, - unittest_pb2.TestAllTypes.DESCRIPTOR) - - def testPackage(self): - self.assertEqual( - unittest_pb2.TestAllTypes.DESCRIPTOR.file.package, - 'protobuf_unittest') - desc = unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR - self.assertEqual(desc.file.package, 'protobuf_unittest') - self.assertEqual( - unittest_import_pb2.ImportMessage.DESCRIPTOR.file.package, - 'protobuf_unittest_import') - - self.assertEqual( - unittest_pb2._FOREIGNENUM.file.package, 'protobuf_unittest') - self.assertEqual( - unittest_pb2._TESTALLTYPES_NESTEDENUM.file.package, - 'protobuf_unittest') - self.assertEqual( - unittest_import_pb2._IMPORTENUM.file.package, - 'protobuf_unittest_import') - - def testExtensionRange(self): - self.assertEqual( - unittest_pb2.TestAllTypes.DESCRIPTOR.extension_ranges, []) - self.assertEqual( - unittest_pb2.TestAllExtensions.DESCRIPTOR.extension_ranges, - [(1, MAX_EXTENSION)]) - self.assertEqual( - unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR.extension_ranges, - [(42, 43), (4143, 4244), (65536, MAX_EXTENSION)]) - - def testFileDescriptor(self): - self.assertEqual(unittest_pb2.DESCRIPTOR.name, - 'google/protobuf/unittest.proto') - self.assertEqual(unittest_pb2.DESCRIPTOR.package, 'protobuf_unittest') - self.assertFalse(unittest_pb2.DESCRIPTOR.serialized_pb is None) - self.assertEqual(unittest_pb2.DESCRIPTOR.dependencies, - [unittest_import_pb2.DESCRIPTOR]) - self.assertEqual(unittest_import_pb2.DESCRIPTOR.dependencies, - [unittest_import_public_pb2.DESCRIPTOR]) - self.assertEqual(unittest_import_pb2.DESCRIPTOR.public_dependencies, - [unittest_import_public_pb2.DESCRIPTOR]) - def testNoGenericServices(self): - self.assertTrue(hasattr(unittest_no_generic_services_pb2, "TestMessage")) - self.assertTrue(hasattr(unittest_no_generic_services_pb2, "FOO")) - self.assertTrue(hasattr(unittest_no_generic_services_pb2, "test_extension")) - - # Make sure unittest_no_generic_services_pb2 has no services subclassing - # Proto2 Service class. - if hasattr(unittest_no_generic_services_pb2, "TestService"): - self.assertFalse(issubclass(unittest_no_generic_services_pb2.TestService, - service.Service)) - - def testMessageTypesByName(self): - file_type = unittest_pb2.DESCRIPTOR - self.assertEqual( - unittest_pb2._TESTALLTYPES, - file_type.message_types_by_name[unittest_pb2._TESTALLTYPES.name]) - - # Nested messages shouldn't be included in the message_types_by_name - # dictionary (like in the C++ API). - self.assertFalse( - unittest_pb2._TESTALLTYPES_NESTEDMESSAGE.name in - file_type.message_types_by_name) - - def testEnumTypesByName(self): - file_type = unittest_pb2.DESCRIPTOR - self.assertEqual( - unittest_pb2._FOREIGNENUM, - file_type.enum_types_by_name[unittest_pb2._FOREIGNENUM.name]) - - def testExtensionsByName(self): - file_type = unittest_pb2.DESCRIPTOR - self.assertEqual( - unittest_pb2.my_extension_string, - file_type.extensions_by_name[unittest_pb2.my_extension_string.name]) - - def testPublicImports(self): - # Test public imports as embedded message. - all_type_proto = unittest_pb2.TestAllTypes() - self.assertEqual(0, all_type_proto.optional_public_import_message.e) - - # PublicImportMessage is actually defined in unittest_import_public_pb2 - # module, and is public imported by unittest_import_pb2 module. - public_import_proto = unittest_import_pb2.PublicImportMessage() - self.assertEqual(0, public_import_proto.e) - self.assertTrue(unittest_import_public_pb2.PublicImportMessage is - unittest_import_pb2.PublicImportMessage) - - def testBadIdentifiers(self): - # We're just testing that the code was imported without problems. - message = test_bad_identifiers_pb2.TestBadIdentifiers() - self.assertEqual(message.Extensions[test_bad_identifiers_pb2.message], - "foo") - self.assertEqual(message.Extensions[test_bad_identifiers_pb2.descriptor], - "bar") - self.assertEqual(message.Extensions[test_bad_identifiers_pb2.reflection], - "baz") - self.assertEqual(message.Extensions[test_bad_identifiers_pb2.service], - "qux") - - def testOneof(self): - desc = unittest_pb2.TestAllTypes.DESCRIPTOR - self.assertEqual(1, len(desc.oneofs)) - self.assertEqual('oneof_field', desc.oneofs[0].name) - self.assertEqual(0, desc.oneofs[0].index) - self.assertIs(desc, desc.oneofs[0].containing_type) - self.assertIs(desc.oneofs[0], desc.oneofs_by_name['oneof_field']) - nested_names = set(['oneof_uint32', 'oneof_nested_message', - 'oneof_string', 'oneof_bytes']) - self.assertEqual( - nested_names, - set([field.name for field in desc.oneofs[0].fields])) - for field_name, field_desc in desc.fields_by_name.items(): - if field_name in nested_names: - self.assertIs(desc.oneofs[0], field_desc.containing_oneof) - else: - self.assertIsNone(field_desc.containing_oneof) - - def testEnumWithDupValue(self): - self.assertEqual('FOO1', - unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.FOO1)) - self.assertEqual('FOO1', - unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.FOO2)) - self.assertEqual('BAR1', - unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.BAR1)) - self.assertEqual('BAR1', - unittest_pb2.TestEnumWithDupValue.Name(unittest_pb2.BAR2)) - - -class SymbolDatabaseRegistrationTest(unittest.TestCase): - """Checks that messages, enums and files are correctly registered.""" - - def testGetSymbol(self): - self.assertEqual( - unittest_pb2.TestAllTypes, symbol_database.Default().GetSymbol( - 'protobuf_unittest.TestAllTypes')) - self.assertEqual( - unittest_pb2.TestAllTypes.NestedMessage, - symbol_database.Default().GetSymbol( - 'protobuf_unittest.TestAllTypes.NestedMessage')) - with self.assertRaises(KeyError): - symbol_database.Default().GetSymbol('protobuf_unittest.NestedMessage') - self.assertEqual( - unittest_pb2.TestAllTypes.OptionalGroup, - symbol_database.Default().GetSymbol( - 'protobuf_unittest.TestAllTypes.OptionalGroup')) - self.assertEqual( - unittest_pb2.TestAllTypes.RepeatedGroup, - symbol_database.Default().GetSymbol( - 'protobuf_unittest.TestAllTypes.RepeatedGroup')) - - def testEnums(self): - self.assertEqual( - 'protobuf_unittest.ForeignEnum', - symbol_database.Default().pool.FindEnumTypeByName( - 'protobuf_unittest.ForeignEnum').full_name) - self.assertEqual( - 'protobuf_unittest.TestAllTypes.NestedEnum', - symbol_database.Default().pool.FindEnumTypeByName( - 'protobuf_unittest.TestAllTypes.NestedEnum').full_name) - - def testFindFileByName(self): - self.assertEqual( - 'google/protobuf/unittest.proto', - symbol_database.Default().pool.FindFileByName( - 'google/protobuf/unittest.proto').name) - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/import_test.py b/ext/protobuf/Python/google/protobuf/internal/import_test.py deleted file mode 100644 index b5c572cfc..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/import_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unittest for nested public imports.""" - -import unittest - -from google.protobuf.internal.import_test_package import outer_pb2 - - -class ImportTest(unittest.TestCase): - - def testPackageInitializationImport(self): - """Test that we can import nested import public messages.""" - - msg = outer_pb2.Outer() - self.assertEqual(58, msg.import_public_nested.value) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/import_test_package/__init__.py b/ext/protobuf/Python/google/protobuf/internal/import_test_package/__init__.py deleted file mode 100644 index 5121dd0ec..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/import_test_package/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Sample module importing a nested proto from itself.""" - -from google.protobuf.internal.import_test_package import outer_pb2 as myproto diff --git a/ext/protobuf/Python/google/protobuf/internal/json_format_test.py b/ext/protobuf/Python/google/protobuf/internal/json_format_test.py deleted file mode 100644 index d018c3f2e..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/json_format_test.py +++ /dev/null @@ -1,1285 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test for google.protobuf.json_format.""" - -__author__ = 'jieluo@google.com (Jie Luo)' - -import json -import math -import struct - -import unittest - -from google.protobuf import any_pb2 -from google.protobuf import duration_pb2 -from google.protobuf import field_mask_pb2 -from google.protobuf import struct_pb2 -from google.protobuf import timestamp_pb2 -from google.protobuf import wrappers_pb2 -from google.protobuf import any_test_pb2 -from google.protobuf import unittest_mset_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf.internal import test_proto3_optional_pb2 -from google.protobuf import descriptor_pool -from google.protobuf import json_format -from google.protobuf.util import json_format_pb2 -from google.protobuf.util import json_format_proto3_pb2 - - -class JsonFormatBase(unittest.TestCase): - - def FillAllFields(self, message): - message.int32_value = 20 - message.int64_value = -20 - message.uint32_value = 3120987654 - message.uint64_value = 12345678900 - message.float_value = float('-inf') - message.double_value = 3.1415 - message.bool_value = True - message.string_value = 'foo' - message.bytes_value = b'bar' - message.message_value.value = 10 - message.enum_value = json_format_proto3_pb2.BAR - # Repeated - message.repeated_int32_value.append(0x7FFFFFFF) - message.repeated_int32_value.append(-2147483648) - message.repeated_int64_value.append(9007199254740992) - message.repeated_int64_value.append(-9007199254740992) - message.repeated_uint32_value.append(0xFFFFFFF) - message.repeated_uint32_value.append(0x7FFFFFF) - message.repeated_uint64_value.append(9007199254740992) - message.repeated_uint64_value.append(9007199254740991) - message.repeated_float_value.append(0) - - message.repeated_double_value.append(1E-15) - message.repeated_double_value.append(float('inf')) - message.repeated_bool_value.append(True) - message.repeated_bool_value.append(False) - message.repeated_string_value.append('Few symbols!#$,;') - message.repeated_string_value.append('bar') - message.repeated_bytes_value.append(b'foo') - message.repeated_bytes_value.append(b'bar') - message.repeated_message_value.add().value = 10 - message.repeated_message_value.add().value = 11 - message.repeated_enum_value.append(json_format_proto3_pb2.FOO) - message.repeated_enum_value.append(json_format_proto3_pb2.BAR) - self.message = message - - def CheckParseBack(self, message, parsed_message): - json_format.Parse(json_format.MessageToJson(message), - parsed_message) - self.assertEqual(message, parsed_message) - - def CheckError(self, text, error_message): - message = json_format_proto3_pb2.TestMessage() - self.assertRaisesRegex(json_format.ParseError, error_message, - json_format.Parse, text, message) - - -class JsonFormatTest(JsonFormatBase): - - def testEmptyMessageToJson(self): - message = json_format_proto3_pb2.TestMessage() - self.assertEqual(json_format.MessageToJson(message), - '{}') - parsed_message = json_format_proto3_pb2.TestMessage() - self.CheckParseBack(message, parsed_message) - - def testPartialMessageToJson(self): - message = json_format_proto3_pb2.TestMessage( - string_value='test', - repeated_int32_value=[89, 4]) - self.assertEqual(json.loads(json_format.MessageToJson(message)), - json.loads('{"stringValue": "test", ' - '"repeatedInt32Value": [89, 4]}')) - parsed_message = json_format_proto3_pb2.TestMessage() - self.CheckParseBack(message, parsed_message) - - def testAllFieldsToJson(self): - message = json_format_proto3_pb2.TestMessage() - text = ('{"int32Value": 20, ' - '"int64Value": "-20", ' - '"uint32Value": 3120987654,' - '"uint64Value": "12345678900",' - '"floatValue": "-Infinity",' - '"doubleValue": 3.1415,' - '"boolValue": true,' - '"stringValue": "foo",' - '"bytesValue": "YmFy",' - '"messageValue": {"value": 10},' - '"enumValue": "BAR",' - '"repeatedInt32Value": [2147483647, -2147483648],' - '"repeatedInt64Value": ["9007199254740992", "-9007199254740992"],' - '"repeatedUint32Value": [268435455, 134217727],' - '"repeatedUint64Value": ["9007199254740992", "9007199254740991"],' - '"repeatedFloatValue": [0],' - '"repeatedDoubleValue": [1e-15, "Infinity"],' - '"repeatedBoolValue": [true, false],' - '"repeatedStringValue": ["Few symbols!#$,;", "bar"],' - '"repeatedBytesValue": ["Zm9v", "YmFy"],' - '"repeatedMessageValue": [{"value": 10}, {"value": 11}],' - '"repeatedEnumValue": ["FOO", "BAR"]' - '}') - self.FillAllFields(message) - self.assertEqual( - json.loads(json_format.MessageToJson(message)), - json.loads(text)) - parsed_message = json_format_proto3_pb2.TestMessage() - json_format.Parse(text, parsed_message) - self.assertEqual(message, parsed_message) - - def testUnknownEnumToJsonAndBack(self): - text = '{\n "enumValue": 999\n}' - message = json_format_proto3_pb2.TestMessage() - message.enum_value = 999 - self.assertEqual(json_format.MessageToJson(message), - text) - parsed_message = json_format_proto3_pb2.TestMessage() - json_format.Parse(text, parsed_message) - self.assertEqual(message, parsed_message) - - def testExtensionToJsonAndBack(self): - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - message_text = json_format.MessageToJson( - message - ) - parsed_message = unittest_mset_pb2.TestMessageSetContainer() - json_format.Parse(message_text, parsed_message) - self.assertEqual(message, parsed_message) - - def testExtensionErrors(self): - self.CheckError('{"[extensionField]": {}}', - 'Message type proto3.TestMessage does not have extensions') - - def testExtensionToDictAndBack(self): - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - message_dict = json_format.MessageToDict( - message - ) - parsed_message = unittest_mset_pb2.TestMessageSetContainer() - json_format.ParseDict(message_dict, parsed_message) - self.assertEqual(message, parsed_message) - - def testExtensionToDictAndBackWithScalar(self): - message = unittest_pb2.TestAllExtensions() - ext1 = unittest_pb2.TestNestedExtension.test - message.Extensions[ext1] = 'data' - message_dict = json_format.MessageToDict( - message - ) - parsed_message = unittest_pb2.TestAllExtensions() - json_format.ParseDict(message_dict, parsed_message) - self.assertEqual(message, parsed_message) - - def testJsonParseDictToAnyDoesNotAlterInput(self): - orig_dict = { - 'int32Value': 20, - '@type': 'type.googleapis.com/proto3.TestMessage' - } - copied_dict = json.loads(json.dumps(orig_dict)) - parsed_message = any_pb2.Any() - json_format.ParseDict(copied_dict, parsed_message) - self.assertEqual(copied_dict, orig_dict) - - def testExtensionSerializationDictMatchesProto3Spec(self): - """See go/proto3-json-spec for spec. - """ - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - message_dict = json_format.MessageToDict( - message - ) - golden_dict = { - 'messageSet': { - '[protobuf_unittest.' - 'TestMessageSetExtension1.message_set_extension]': { - 'i': 23, - }, - '[protobuf_unittest.' - 'TestMessageSetExtension2.message_set_extension]': { - 'str': u'foo', - }, - }, - } - self.assertEqual(golden_dict, message_dict) - parsed_msg = unittest_mset_pb2.TestMessageSetContainer() - json_format.ParseDict(golden_dict, parsed_msg) - self.assertEqual(message, parsed_msg) - - def testExtensionSerializationDictMatchesProto3SpecMore(self): - """See go/proto3-json-spec for spec. - """ - message = json_format_pb2.TestMessageWithExtension() - ext = json_format_pb2.TestExtension.ext - message.Extensions[ext].value = 'stuff' - message_dict = json_format.MessageToDict( - message - ) - expected_dict = { - '[protobuf_unittest.TestExtension.ext]': { - 'value': u'stuff', - }, - } - self.assertEqual(expected_dict, message_dict) - - def testExtensionSerializationJsonMatchesProto3Spec(self): - """See go/proto3-json-spec for spec. - """ - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - message_text = json_format.MessageToJson( - message - ) - ext1_text = ('protobuf_unittest.TestMessageSetExtension1.' - 'message_set_extension') - ext2_text = ('protobuf_unittest.TestMessageSetExtension2.' - 'message_set_extension') - golden_text = ('{"messageSet": {' - ' "[%s]": {' - ' "i": 23' - ' },' - ' "[%s]": {' - ' "str": "foo"' - ' }' - '}}') % (ext1_text, ext2_text) - self.assertEqual(json.loads(golden_text), json.loads(message_text)) - - def testJsonEscapeString(self): - message = json_format_proto3_pb2.TestMessage() - message.string_value = '&\n<\"\r>\b\t\f\\\001/' - message.string_value += (b'\xe2\x80\xa8\xe2\x80\xa9').decode('utf-8') - self.assertEqual( - json_format.MessageToJson(message), - '{\n "stringValue": ' - '"&\\n<\\\"\\r>\\b\\t\\f\\\\\\u0001/\\u2028\\u2029"\n}') - parsed_message = json_format_proto3_pb2.TestMessage() - self.CheckParseBack(message, parsed_message) - text = u'{"int32Value": "\u0031"}' - json_format.Parse(text, message) - self.assertEqual(message.int32_value, 1) - - def testAlwaysSeriliaze(self): - message = json_format_proto3_pb2.TestMessage( - string_value='foo') - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads('{' - '"repeatedStringValue": [],' - '"stringValue": "foo",' - '"repeatedBoolValue": [],' - '"repeatedUint32Value": [],' - '"repeatedInt32Value": [],' - '"enumValue": "FOO",' - '"int32Value": 0,' - '"floatValue": 0,' - '"int64Value": "0",' - '"uint32Value": 0,' - '"repeatedBytesValue": [],' - '"repeatedUint64Value": [],' - '"repeatedDoubleValue": [],' - '"bytesValue": "",' - '"boolValue": false,' - '"repeatedEnumValue": [],' - '"uint64Value": "0",' - '"doubleValue": 0,' - '"repeatedFloatValue": [],' - '"repeatedInt64Value": [],' - '"repeatedMessageValue": []}')) - parsed_message = json_format_proto3_pb2.TestMessage() - self.CheckParseBack(message, parsed_message) - - def testProto3Optional(self): - message = test_proto3_optional_pb2.TestProto3Optional() - self.assertEqual( - json.loads( - json_format.MessageToJson( - message, including_default_value_fields=True)), - json.loads('{}')) - message.optional_int32 = 0 - self.assertEqual( - json.loads( - json_format.MessageToJson( - message, including_default_value_fields=True)), - json.loads('{"optionalInt32": 0}')) - - def testIntegersRepresentedAsFloat(self): - message = json_format_proto3_pb2.TestMessage() - json_format.Parse('{"int32Value": -2.147483648e9}', message) - self.assertEqual(message.int32_value, -2147483648) - json_format.Parse('{"int32Value": 1e5}', message) - self.assertEqual(message.int32_value, 100000) - json_format.Parse('{"int32Value": 1.0}', message) - self.assertEqual(message.int32_value, 1) - - def testMapFields(self): - message = json_format_proto3_pb2.TestNestedMap() - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads('{' - '"boolMap": {},' - '"int32Map": {},' - '"int64Map": {},' - '"uint32Map": {},' - '"uint64Map": {},' - '"stringMap": {},' - '"mapMap": {}' - '}')) - message.bool_map[True] = 1 - message.bool_map[False] = 2 - message.int32_map[1] = 2 - message.int32_map[2] = 3 - message.int64_map[1] = 2 - message.int64_map[2] = 3 - message.uint32_map[1] = 2 - message.uint32_map[2] = 3 - message.uint64_map[1] = 2 - message.uint64_map[2] = 3 - message.string_map['1'] = 2 - message.string_map['null'] = 3 - message.map_map['1'].bool_map[True] = 3 - self.assertEqual( - json.loads(json_format.MessageToJson(message, False)), - json.loads('{' - '"boolMap": {"false": 2, "true": 1},' - '"int32Map": {"1": 2, "2": 3},' - '"int64Map": {"1": 2, "2": 3},' - '"uint32Map": {"1": 2, "2": 3},' - '"uint64Map": {"1": 2, "2": 3},' - '"stringMap": {"1": 2, "null": 3},' - '"mapMap": {"1": {"boolMap": {"true": 3}}}' - '}')) - parsed_message = json_format_proto3_pb2.TestNestedMap() - self.CheckParseBack(message, parsed_message) - - def testOneofFields(self): - message = json_format_proto3_pb2.TestOneof() - # Always print does not affect oneof fields. - self.assertEqual( - json_format.MessageToJson(message, True), - '{}') - message.oneof_int32_value = 0 - self.assertEqual( - json_format.MessageToJson(message, True), - '{\n' - ' "oneofInt32Value": 0\n' - '}') - parsed_message = json_format_proto3_pb2.TestOneof() - self.CheckParseBack(message, parsed_message) - - def testSurrogates(self): - # Test correct surrogate handling. - message = json_format_proto3_pb2.TestMessage() - json_format.Parse('{"stringValue": "\\uD83D\\uDE01"}', message) - self.assertEqual(message.string_value, - b'\xF0\x9F\x98\x81'.decode('utf-8', 'strict')) - - # Error case: unpaired high surrogate. - self.CheckError( - '{"stringValue": "\\uD83D"}', - r'Invalid \\uXXXX escape|Unpaired.*surrogate') - - # Unpaired low surrogate. - self.CheckError( - '{"stringValue": "\\uDE01"}', - r'Invalid \\uXXXX escape|Unpaired.*surrogate') - - def testTimestampMessage(self): - message = json_format_proto3_pb2.TestTimestamp() - message.value.seconds = 0 - message.value.nanos = 0 - message.repeated_value.add().seconds = 20 - message.repeated_value[0].nanos = 1 - message.repeated_value.add().seconds = 0 - message.repeated_value[1].nanos = 10000 - message.repeated_value.add().seconds = 100000000 - message.repeated_value[2].nanos = 0 - # Maximum time - message.repeated_value.add().seconds = 253402300799 - message.repeated_value[3].nanos = 999999999 - # Minimum time - message.repeated_value.add().seconds = -62135596800 - message.repeated_value[4].nanos = 0 - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads('{' - '"value": "1970-01-01T00:00:00Z",' - '"repeatedValue": [' - ' "1970-01-01T00:00:20.000000001Z",' - ' "1970-01-01T00:00:00.000010Z",' - ' "1973-03-03T09:46:40Z",' - ' "9999-12-31T23:59:59.999999999Z",' - ' "0001-01-01T00:00:00Z"' - ']' - '}')) - parsed_message = json_format_proto3_pb2.TestTimestamp() - self.CheckParseBack(message, parsed_message) - text = (r'{"value": "1970-01-01T00:00:00.01+08:00",' - r'"repeatedValue":[' - r' "1970-01-01T00:00:00.01+08:30",' - r' "1970-01-01T00:00:00.01-01:23"]}') - json_format.Parse(text, parsed_message) - self.assertEqual(parsed_message.value.seconds, -8 * 3600) - self.assertEqual(parsed_message.value.nanos, 10000000) - self.assertEqual(parsed_message.repeated_value[0].seconds, -8.5 * 3600) - self.assertEqual(parsed_message.repeated_value[1].seconds, 3600 + 23 * 60) - - def testDurationMessage(self): - message = json_format_proto3_pb2.TestDuration() - message.value.seconds = 1 - message.repeated_value.add().seconds = 0 - message.repeated_value[0].nanos = 10 - message.repeated_value.add().seconds = -1 - message.repeated_value[1].nanos = -1000 - message.repeated_value.add().seconds = 10 - message.repeated_value[2].nanos = 11000000 - message.repeated_value.add().seconds = -315576000000 - message.repeated_value.add().seconds = 315576000000 - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads('{' - '"value": "1s",' - '"repeatedValue": [' - ' "0.000000010s",' - ' "-1.000001s",' - ' "10.011s",' - ' "-315576000000s",' - ' "315576000000s"' - ']' - '}')) - parsed_message = json_format_proto3_pb2.TestDuration() - self.CheckParseBack(message, parsed_message) - - def testFieldMaskMessage(self): - message = json_format_proto3_pb2.TestFieldMask() - message.value.paths.append('foo.bar') - message.value.paths.append('bar') - self.assertEqual( - json_format.MessageToJson(message, True), - '{\n' - ' "value": "foo.bar,bar"\n' - '}') - parsed_message = json_format_proto3_pb2.TestFieldMask() - self.CheckParseBack(message, parsed_message) - - message.value.Clear() - self.assertEqual( - json_format.MessageToJson(message, True), - '{\n' - ' "value": ""\n' - '}') - self.CheckParseBack(message, parsed_message) - - def testWrapperMessage(self): - message = json_format_proto3_pb2.TestWrapper() - message.bool_value.value = False - message.int32_value.value = 0 - message.string_value.value = '' - message.bytes_value.value = b'' - message.repeated_bool_value.add().value = True - message.repeated_bool_value.add().value = False - message.repeated_int32_value.add() - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads('{\n' - ' "int32Value": 0,' - ' "boolValue": false,' - ' "stringValue": "",' - ' "bytesValue": "",' - ' "repeatedBoolValue": [true, false],' - ' "repeatedInt32Value": [0],' - ' "repeatedUint32Value": [],' - ' "repeatedFloatValue": [],' - ' "repeatedDoubleValue": [],' - ' "repeatedBytesValue": [],' - ' "repeatedInt64Value": [],' - ' "repeatedUint64Value": [],' - ' "repeatedStringValue": []' - '}')) - parsed_message = json_format_proto3_pb2.TestWrapper() - self.CheckParseBack(message, parsed_message) - - def testStructMessage(self): - message = json_format_proto3_pb2.TestStruct() - message.value['name'] = 'Jim' - message.value['age'] = 10 - message.value['attend'] = True - message.value['email'] = None - message.value.get_or_create_struct('address')['city'] = 'SFO' - message.value['address']['house_number'] = 1024 - message.value.get_or_create_struct('empty_struct') - message.value.get_or_create_list('empty_list') - struct_list = message.value.get_or_create_list('list') - struct_list.extend([6, 'seven', True, False, None]) - struct_list.add_struct()['subkey2'] = 9 - message.repeated_value.add()['age'] = 11 - message.repeated_value.add() - self.assertEqual( - json.loads(json_format.MessageToJson(message, False)), - json.loads( - '{' - ' "value": {' - ' "address": {' - ' "city": "SFO", ' - ' "house_number": 1024' - ' }, ' - ' "empty_struct": {}, ' - ' "empty_list": [], ' - ' "age": 10, ' - ' "name": "Jim", ' - ' "attend": true, ' - ' "email": null, ' - ' "list": [6, "seven", true, false, null, {"subkey2": 9}]' - ' },' - ' "repeatedValue": [{"age": 11}, {}]' - '}')) - parsed_message = json_format_proto3_pb2.TestStruct() - self.CheckParseBack(message, parsed_message) - # check for regression; this used to raise - parsed_message.value['empty_struct'] - parsed_message.value['empty_list'] - - def testValueMessage(self): - message = json_format_proto3_pb2.TestValue() - message.value.string_value = 'hello' - message.repeated_value.add().number_value = 11.1 - message.repeated_value.add().bool_value = False - message.repeated_value.add().null_value = 0 - self.assertEqual( - json.loads(json_format.MessageToJson(message, False)), - json.loads( - '{' - ' "value": "hello",' - ' "repeatedValue": [11.1, false, null]' - '}')) - parsed_message = json_format_proto3_pb2.TestValue() - self.CheckParseBack(message, parsed_message) - # Can't parse back if the Value message is not set. - message.repeated_value.add() - self.assertEqual( - json.loads(json_format.MessageToJson(message, False)), - json.loads( - '{' - ' "value": "hello",' - ' "repeatedValue": [11.1, false, null, null]' - '}')) - message.Clear() - json_format.Parse('{"value": null}', message) - self.assertEqual(message.value.WhichOneof('kind'), 'null_value') - - def testListValueMessage(self): - message = json_format_proto3_pb2.TestListValue() - message.value.values.add().number_value = 11.1 - message.value.values.add().null_value = 0 - message.value.values.add().bool_value = True - message.value.values.add().string_value = 'hello' - message.value.values.add().struct_value['name'] = 'Jim' - message.repeated_value.add().values.add().number_value = 1 - message.repeated_value.add() - self.assertEqual( - json.loads(json_format.MessageToJson(message, False)), - json.loads( - '{"value": [11.1, null, true, "hello", {"name": "Jim"}]\n,' - '"repeatedValue": [[1], []]}')) - parsed_message = json_format_proto3_pb2.TestListValue() - self.CheckParseBack(message, parsed_message) - - def testNullValue(self): - message = json_format_proto3_pb2.TestOneof() - message.oneof_null_value = 0 - self.assertEqual(json_format.MessageToJson(message), - '{\n "oneofNullValue": null\n}') - parsed_message = json_format_proto3_pb2.TestOneof() - self.CheckParseBack(message, parsed_message) - # Check old format is also accepted - new_message = json_format_proto3_pb2.TestOneof() - json_format.Parse('{\n "oneofNullValue": "NULL_VALUE"\n}', - new_message) - self.assertEqual(json_format.MessageToJson(new_message), - '{\n "oneofNullValue": null\n}') - - def testAnyMessage(self): - message = json_format_proto3_pb2.TestAny() - value1 = json_format_proto3_pb2.MessageType() - value2 = json_format_proto3_pb2.MessageType() - value1.value = 1234 - value2.value = 5678 - message.value.Pack(value1) - message.repeated_value.add().Pack(value1) - message.repeated_value.add().Pack(value2) - message.repeated_value.add() - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "repeatedValue": [ {\n' - ' "@type": "type.googleapis.com/proto3.MessageType",\n' - ' "value": 1234\n' - ' }, {\n' - ' "@type": "type.googleapis.com/proto3.MessageType",\n' - ' "value": 5678\n' - ' },\n' - ' {}],\n' - ' "value": {\n' - ' "@type": "type.googleapis.com/proto3.MessageType",\n' - ' "value": 1234\n' - ' }\n' - '}\n')) - parsed_message = json_format_proto3_pb2.TestAny() - self.CheckParseBack(message, parsed_message) - # Must print @type first - test_message = json_format_proto3_pb2.TestMessage( - bool_value=True, - int32_value=20, - int64_value=-20, - uint32_value=20, - uint64_value=20, - double_value=3.14, - string_value='foo') - message.Clear() - message.value.Pack(test_message) - self.assertEqual( - json_format.MessageToJson(message, False)[0:68], - '{\n' - ' "value": {\n' - ' "@type": "type.googleapis.com/proto3.TestMessage"') - - def testAnyMessageDescriptorPoolMissingType(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - empty_pool = descriptor_pool.DescriptorPool() - with self.assertRaises(TypeError) as cm: - json_format.MessageToJson(message, True, descriptor_pool=empty_pool) - self.assertEqual( - 'Can not find message descriptor by type_url:' - ' type.googleapis.com/protobuf_unittest.OneString', str(cm.exception)) - - def testWellKnownInAnyMessage(self): - message = any_pb2.Any() - int32_value = wrappers_pb2.Int32Value() - int32_value.value = 1234 - message.Pack(int32_value) - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "@type": \"type.googleapis.com/google.protobuf.Int32Value\",\n' - ' "value": 1234\n' - '}\n')) - parsed_message = any_pb2.Any() - self.CheckParseBack(message, parsed_message) - - timestamp = timestamp_pb2.Timestamp() - message.Pack(timestamp) - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "@type": "type.googleapis.com/google.protobuf.Timestamp",\n' - ' "value": "1970-01-01T00:00:00Z"\n' - '}\n')) - self.CheckParseBack(message, parsed_message) - - duration = duration_pb2.Duration() - duration.seconds = 1 - message.Pack(duration) - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "@type": "type.googleapis.com/google.protobuf.Duration",\n' - ' "value": "1s"\n' - '}\n')) - self.CheckParseBack(message, parsed_message) - - field_mask = field_mask_pb2.FieldMask() - field_mask.paths.append('foo.bar') - field_mask.paths.append('bar') - message.Pack(field_mask) - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "@type": "type.googleapis.com/google.protobuf.FieldMask",\n' - ' "value": "foo.bar,bar"\n' - '}\n')) - self.CheckParseBack(message, parsed_message) - - struct_message = struct_pb2.Struct() - struct_message['name'] = 'Jim' - message.Pack(struct_message) - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "@type": "type.googleapis.com/google.protobuf.Struct",\n' - ' "value": {"name": "Jim"}\n' - '}\n')) - self.CheckParseBack(message, parsed_message) - - nested_any = any_pb2.Any() - int32_value.value = 5678 - nested_any.Pack(int32_value) - message.Pack(nested_any) - self.assertEqual( - json.loads(json_format.MessageToJson(message, True)), - json.loads( - '{\n' - ' "@type": "type.googleapis.com/google.protobuf.Any",\n' - ' "value": {\n' - ' "@type": "type.googleapis.com/google.protobuf.Int32Value",\n' - ' "value": 5678\n' - ' }\n' - '}\n')) - self.CheckParseBack(message, parsed_message) - - def testParseNull(self): - message = json_format_proto3_pb2.TestMessage() - parsed_message = json_format_proto3_pb2.TestMessage() - self.FillAllFields(parsed_message) - json_format.Parse('{"int32Value": null, ' - '"int64Value": null, ' - '"uint32Value": null,' - '"uint64Value": null,' - '"floatValue": null,' - '"doubleValue": null,' - '"boolValue": null,' - '"stringValue": null,' - '"bytesValue": null,' - '"messageValue": null,' - '"enumValue": null,' - '"repeatedInt32Value": null,' - '"repeatedInt64Value": null,' - '"repeatedUint32Value": null,' - '"repeatedUint64Value": null,' - '"repeatedFloatValue": null,' - '"repeatedDoubleValue": null,' - '"repeatedBoolValue": null,' - '"repeatedStringValue": null,' - '"repeatedBytesValue": null,' - '"repeatedMessageValue": null,' - '"repeatedEnumValue": null' - '}', - parsed_message) - self.assertEqual(message, parsed_message) - # Null and {} should have different behavior for sub message. - self.assertFalse(parsed_message.HasField('message_value')) - json_format.Parse('{"messageValue": {}}', parsed_message) - self.assertTrue(parsed_message.HasField('message_value')) - # Null is not allowed to be used as an element in repeated field. - self.assertRaisesRegex( - json_format.ParseError, r'Failed to parse repeatedInt32Value field: ' - r'null is not allowed to be used as an element in a repeated field ' - r'at TestMessage.repeatedInt32Value\[1\].', json_format.Parse, - '{"repeatedInt32Value":[1, null]}', parsed_message) - self.CheckError( - '{"repeatedMessageValue":[null]}', - r'Failed to parse repeatedMessageValue field: null is not' - r' allowed to be used as an element in a repeated field ' - r'at TestMessage.repeatedMessageValue\[0\].') - - def testNanFloat(self): - message = json_format_proto3_pb2.TestMessage() - message.float_value = float('nan') - text = '{\n "floatValue": "NaN"\n}' - self.assertEqual(json_format.MessageToJson(message), text) - parsed_message = json_format_proto3_pb2.TestMessage() - json_format.Parse(text, parsed_message) - self.assertTrue(math.isnan(parsed_message.float_value)) - - def testParseDoubleToFloat(self): - message = json_format_proto3_pb2.TestMessage() - text = ('{"repeatedDoubleValue": [3.4028235e+39, 1.4028235e-39]\n}') - json_format.Parse(text, message) - self.assertEqual(message.repeated_double_value[0], 3.4028235e+39) - self.assertEqual(message.repeated_double_value[1], 1.4028235e-39) - text = ('{"repeatedFloatValue": [3.4028235e+39, 1.4028235e-39]\n}') - self.CheckError( - text, r'Failed to parse repeatedFloatValue field: ' - r'Float value too large at TestMessage.repeatedFloatValue\[0\].') - - def testFloatPrecision(self): - message = json_format_proto3_pb2.TestMessage() - message.float_value = 1.123456789 - # Set to 8 valid digits. - text = '{\n "floatValue": 1.1234568\n}' - self.assertEqual( - json_format.MessageToJson(message, float_precision=8), text) - # Set to 7 valid digits. - text = '{\n "floatValue": 1.123457\n}' - self.assertEqual( - json_format.MessageToJson(message, float_precision=7), text) - - # Default float_precision will automatic print shortest float. - message.float_value = 1.1000000011 - text = '{\n "floatValue": 1.1\n}' - self.assertEqual( - json_format.MessageToJson(message), text) - message.float_value = 1.00000075e-36 - text = '{\n "floatValue": 1.00000075e-36\n}' - self.assertEqual( - json_format.MessageToJson(message), text) - message.float_value = 12345678912345e+11 - text = '{\n "floatValue": 1.234568e+24\n}' - self.assertEqual( - json_format.MessageToJson(message), text) - - # Test a bunch of data and check json encode/decode do not - # lose precision - value_list = [0x00, 0xD8, 0x6E, 0x00] - msg2 = json_format_proto3_pb2.TestMessage() - for a in range(0, 256): - value_list[3] = a - for b in range(0, 256): - value_list[0] = b - byte_array = bytearray(value_list) - message.float_value = struct.unpack('.", - json_format.ParseDict, {'value': UnknownClass()}, message) - - def testMessageToDict(self): - message = json_format_proto3_pb2.TestMessage() - message.int32_value = 12345 - expected = {'int32Value': 12345} - self.assertEqual(expected, - json_format.MessageToDict(message)) - - def testJsonName(self): - message = json_format_proto3_pb2.TestCustomJsonName() - message.value = 12345 - self.assertEqual('{\n "@value": 12345\n}', - json_format.MessageToJson(message)) - parsed_message = json_format_proto3_pb2.TestCustomJsonName() - self.CheckParseBack(message, parsed_message) - - def testSortKeys(self): - # Testing sort_keys is not perfectly working, as by random luck we could - # get the output sorted. We just use a selection of names. - message = json_format_proto3_pb2.TestMessage(bool_value=True, - int32_value=1, - int64_value=3, - uint32_value=4, - string_value='bla') - self.assertEqual( - json_format.MessageToJson(message, sort_keys=True), - # We use json.dumps() instead of a hardcoded string due to differences - # between Python 2 and Python 3. - json.dumps({'boolValue': True, 'int32Value': 1, 'int64Value': '3', - 'uint32Value': 4, 'stringValue': 'bla'}, - indent=2, sort_keys=True)) - - def testNestedRecursiveLimit(self): - message = unittest_pb2.NestedTestAllTypes() - self.assertRaisesRegex( - json_format.ParseError, - 'Message too deep. Max recursion depth is 3', - json_format.Parse, - '{"child": {"child": {"child" : {}}}}', - message, - max_recursion_depth=3) - # The following one can pass - json_format.Parse('{"payload": {}, "child": {"child":{}}}', - message, max_recursion_depth=3) - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/keywords_test.py b/ext/protobuf/Python/google/protobuf/internal/keywords_test.py deleted file mode 100644 index 4182cf6be..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/keywords_test.py +++ /dev/null @@ -1,103 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.internal.keywords.""" - -import unittest - - -from google.protobuf.internal import more_messages_pb2 -from google.protobuf import descriptor_pool - - -class KeywordsConflictTest(unittest.TestCase): - - def setUp(self): - super(KeywordsConflictTest, self).setUp() - self.pool = descriptor_pool.Default() - - def testMessage(self): - message = getattr(more_messages_pb2, 'class')() - message.int_field = 123 - self.assertEqual(message.int_field, 123) - des = self.pool.FindMessageTypeByName('google.protobuf.internal.class') - self.assertEqual(des.name, 'class') - - def testNestedMessage(self): - message = getattr(more_messages_pb2, 'class')() - message.nested_message.field = 234 - self.assertEqual(message.nested_message.field, 234) - des = self.pool.FindMessageTypeByName('google.protobuf.internal.class.try') - self.assertEqual(des.name, 'try') - - def testField(self): - message = getattr(more_messages_pb2, 'class')() - setattr(message, 'if', 123) - setattr(message, 'as', 1) - self.assertEqual(getattr(message, 'if'), 123) - self.assertEqual(getattr(message, 'as'), 1) - - def testEnum(self): - class_ = getattr(more_messages_pb2, 'class') - message = class_() - # Normal enum value. - message.enum_field = more_messages_pb2.default - self.assertEqual(message.enum_field, more_messages_pb2.default) - # Top level enum value. - message.enum_field = getattr(more_messages_pb2, 'else') - self.assertEqual(message.enum_field, 1) - # Nested enum value - message.nested_enum_field = getattr(class_, 'True') - self.assertEqual(message.nested_enum_field, 1) - - def testExtension(self): - message = getattr(more_messages_pb2, 'class')() - # Top level extension - extension1 = getattr(more_messages_pb2, 'continue') - message.Extensions[extension1] = 456 - self.assertEqual(message.Extensions[extension1], 456) - # None top level extension - extension2 = getattr(more_messages_pb2.ExtendClass, 'return') - message.Extensions[extension2] = 789 - self.assertEqual(message.Extensions[extension2], 789) - - def testExtensionForNestedMessage(self): - message = getattr(more_messages_pb2, 'class')() - extension = getattr(more_messages_pb2, 'with') - message.nested_message.Extensions[extension] = 999 - self.assertEqual(message.nested_message.Extensions[extension], 999) - - def TestFullKeywordUsed(self): - message = more_messages_pb2.TestFullKeyword() - message.field2.int_field = 123 - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/message_factory_test.py b/ext/protobuf/Python/google/protobuf/internal/message_factory_test.py deleted file mode 100644 index efba6194c..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/message_factory_test.py +++ /dev/null @@ -1,299 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.message_factory.""" - -__author__ = 'matthewtoia@google.com (Matt Toia)' - -import unittest - -from google.protobuf import descriptor_pb2 -from google.protobuf.internal import api_implementation -from google.protobuf.internal import factory_test1_pb2 -from google.protobuf.internal import factory_test2_pb2 -from google.protobuf.internal import testing_refleaks -from google.protobuf import descriptor_database -from google.protobuf import descriptor_pool -from google.protobuf import message_factory - - -@testing_refleaks.TestCase -class MessageFactoryTest(unittest.TestCase): - - def setUp(self): - self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test1_pb2.DESCRIPTOR.serialized_pb) - self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString( - factory_test2_pb2.DESCRIPTOR.serialized_pb) - - def _ExerciseDynamicClass(self, cls): - msg = cls() - msg.mandatory = 42 - msg.nested_factory_2_enum = 0 - msg.nested_factory_2_message.value = 'nested message value' - msg.factory_1_message.factory_1_enum = 1 - msg.factory_1_message.nested_factory_1_enum = 0 - msg.factory_1_message.nested_factory_1_message.value = ( - 'nested message value') - msg.factory_1_message.scalar_value = 22 - msg.factory_1_message.list_value.extend([u'one', u'two', u'three']) - msg.factory_1_message.list_value.append(u'four') - msg.factory_1_enum = 1 - msg.nested_factory_1_enum = 0 - msg.nested_factory_1_message.value = 'nested message value' - msg.circular_message.mandatory = 1 - msg.circular_message.circular_message.mandatory = 2 - msg.circular_message.scalar_value = 'one deep' - msg.scalar_value = 'zero deep' - msg.list_value.extend([u'four', u'three', u'two']) - msg.list_value.append(u'one') - msg.grouped.add() - msg.grouped[0].part_1 = 'hello' - msg.grouped[0].part_2 = 'world' - msg.grouped.add(part_1='testing', part_2='123') - msg.loop.loop.mandatory = 2 - msg.loop.loop.loop.loop.mandatory = 4 - serialized = msg.SerializeToString() - converted = factory_test2_pb2.Factory2Message.FromString(serialized) - reserialized = converted.SerializeToString() - self.assertEqual(serialized, reserialized) - result = cls.FromString(reserialized) - self.assertEqual(msg, result) - - def testGetPrototype(self): - db = descriptor_database.DescriptorDatabase() - pool = descriptor_pool.DescriptorPool(db) - db.Add(self.factory_test1_fd) - db.Add(self.factory_test2_fd) - factory = message_factory.MessageFactory() - cls = factory.GetPrototype(pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message')) - self.assertFalse(cls is factory_test2_pb2.Factory2Message) - self._ExerciseDynamicClass(cls) - cls2 = factory.GetPrototype(pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message')) - self.assertTrue(cls is cls2) - - def testCreatePrototypeOverride(self): - class MyMessageFactory(message_factory.MessageFactory): - - def CreatePrototype(self, descriptor): - cls = super(MyMessageFactory, self).CreatePrototype(descriptor) - cls.additional_field = 'Some value' - return cls - - db = descriptor_database.DescriptorDatabase() - pool = descriptor_pool.DescriptorPool(db) - db.Add(self.factory_test1_fd) - db.Add(self.factory_test2_fd) - factory = MyMessageFactory() - cls = factory.GetPrototype(pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message')) - self.assertTrue(hasattr(cls, 'additional_field')) - - def testGetMessages(self): - # performed twice because multiple calls with the same input must be allowed - for _ in range(2): - # GetMessage should work regardless of the order the FileDescriptorProto - # are provided. In particular, the function should succeed when the files - # are not in the topological order of dependencies. - - # Assuming factory_test2_fd depends on factory_test1_fd. - self.assertIn(self.factory_test1_fd.name, - self.factory_test2_fd.dependency) - # Get messages should work when a file comes before its dependencies: - # factory_test2_fd comes before factory_test1_fd. - messages = message_factory.GetMessages([self.factory_test2_fd, - self.factory_test1_fd]) - self.assertTrue( - set(['google.protobuf.python.internal.Factory2Message', - 'google.protobuf.python.internal.Factory1Message'], - ).issubset(set(messages.keys()))) - self._ExerciseDynamicClass( - messages['google.protobuf.python.internal.Factory2Message']) - factory_msg1 = messages['google.protobuf.python.internal.Factory1Message'] - self.assertTrue(set( - ['google.protobuf.python.internal.Factory2Message.one_more_field', - 'google.protobuf.python.internal.another_field'],).issubset(set( - ext.full_name - for ext in factory_msg1.DESCRIPTOR.file.pool.FindAllExtensions( - factory_msg1.DESCRIPTOR)))) - msg1 = messages['google.protobuf.python.internal.Factory1Message']() - ext1 = msg1.Extensions._FindExtensionByName( - 'google.protobuf.python.internal.Factory2Message.one_more_field') - ext2 = msg1.Extensions._FindExtensionByName( - 'google.protobuf.python.internal.another_field') - self.assertEqual(0, len(msg1.Extensions)) - msg1.Extensions[ext1] = 'test1' - msg1.Extensions[ext2] = 'test2' - self.assertEqual('test1', msg1.Extensions[ext1]) - self.assertEqual('test2', msg1.Extensions[ext2]) - self.assertEqual(None, - msg1.Extensions._FindExtensionByNumber(12321)) - self.assertEqual(2, len(msg1.Extensions)) - if api_implementation.Type() == 'cpp': - self.assertRaises(TypeError, - msg1.Extensions._FindExtensionByName, 0) - self.assertRaises(TypeError, - msg1.Extensions._FindExtensionByNumber, '') - else: - self.assertEqual(None, - msg1.Extensions._FindExtensionByName(0)) - self.assertEqual(None, - msg1.Extensions._FindExtensionByNumber('')) - - def testDuplicateExtensionNumber(self): - pool = descriptor_pool.DescriptorPool() - factory = message_factory.MessageFactory(pool=pool) - - # Add Container message. - f = descriptor_pb2.FileDescriptorProto( - name='google/protobuf/internal/container.proto', - package='google.protobuf.python.internal') - f.message_type.add(name='Container').extension_range.add(start=1, end=10) - pool.Add(f) - msgs = factory.GetMessages([f.name]) - self.assertIn('google.protobuf.python.internal.Container', msgs) - - # Extend container. - f = descriptor_pb2.FileDescriptorProto( - name='google/protobuf/internal/extension.proto', - package='google.protobuf.python.internal', - dependency=['google/protobuf/internal/container.proto']) - msg = f.message_type.add(name='Extension') - msg.extension.add( - name='extension_field', - number=2, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL, - type_name='Extension', - extendee='Container') - pool.Add(f) - msgs = factory.GetMessages([f.name]) - self.assertIn('google.protobuf.python.internal.Extension', msgs) - - # Add Duplicate extending the same field number. - f = descriptor_pb2.FileDescriptorProto( - name='google/protobuf/internal/duplicate.proto', - package='google.protobuf.python.internal', - dependency=['google/protobuf/internal/container.proto']) - msg = f.message_type.add(name='Duplicate') - msg.extension.add( - name='extension_field', - number=2, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL, - type_name='Duplicate', - extendee='Container') - pool.Add(f) - - with self.assertRaises(Exception) as cm: - factory.GetMessages([f.name]) - - self.assertIn(str(cm.exception), - ['Extensions ' - '"google.protobuf.python.internal.Duplicate.extension_field" and' - ' "google.protobuf.python.internal.Extension.extension_field"' - ' both try to extend message type' - ' "google.protobuf.python.internal.Container"' - ' with field number 2.', - 'Double registration of Extensions']) - - def testExtensionValueInDifferentFile(self): - # Add Container message. - f1 = descriptor_pb2.FileDescriptorProto( - name='google/protobuf/internal/container.proto', - package='google.protobuf.python.internal') - f1.message_type.add(name='Container').extension_range.add(start=1, end=10) - - # Add ValueType message. - f2 = descriptor_pb2.FileDescriptorProto( - name='google/protobuf/internal/value_type.proto', - package='google.protobuf.python.internal') - f2.message_type.add(name='ValueType').field.add( - name='setting', - number=1, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL, - type=descriptor_pb2.FieldDescriptorProto.TYPE_INT32, - default_value='123') - - # Extend container with field of ValueType. - f3 = descriptor_pb2.FileDescriptorProto( - name='google/protobuf/internal/extension.proto', - package='google.protobuf.python.internal', - dependency=[f1.name, f2.name]) - f3.extension.add( - name='top_level_extension_field', - number=2, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL, - type_name='ValueType', - extendee='Container') - f3.message_type.add(name='Extension').extension.add( - name='nested_extension_field', - number=3, - label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL, - type_name='ValueType', - extendee='Container') - - class SimpleDescriptorDB: - - def __init__(self, files): - self._files = files - - def FindFileByName(self, name): - return self._files[name] - - db = SimpleDescriptorDB({f1.name: f1, f2.name: f2, f3.name: f3}) - - pool = descriptor_pool.DescriptorPool(db) - factory = message_factory.MessageFactory(pool=pool) - msgs = factory.GetMessages([f1.name, f3.name]) # Deliberately not f2. - msg = msgs['google.protobuf.python.internal.Container'] - desc = msgs['google.protobuf.python.internal.Extension'].DESCRIPTOR - ext1 = desc.file.extensions_by_name['top_level_extension_field'] - ext2 = desc.extensions_by_name['nested_extension_field'] - m = msg() - m.Extensions[ext1].setting = 234 - m.Extensions[ext2].setting = 345 - serialized = m.SerializeToString() - - pool = descriptor_pool.DescriptorPool(db) - factory = message_factory.MessageFactory(pool=pool) - msgs = factory.GetMessages([f1.name, f3.name]) # Deliberately not f2. - msg = msgs['google.protobuf.python.internal.Container'] - desc = msgs['google.protobuf.python.internal.Extension'].DESCRIPTOR - ext1 = desc.file.extensions_by_name['top_level_extension_field'] - ext2 = desc.extensions_by_name['nested_extension_field'] - m = msg.FromString(serialized) - self.assertEqual(2, len(m.ListFields())) - self.assertEqual(234, m.Extensions[ext1].setting) - self.assertEqual(345, m.Extensions[ext2].setting) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/message_test.py b/ext/protobuf/Python/google/protobuf/internal/message_test.py deleted file mode 100644 index 40abfe4a2..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/message_test.py +++ /dev/null @@ -1,2572 +0,0 @@ -# -*- coding: utf-8 -*- -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests python protocol buffers against the golden message. - -Note that the golden messages exercise every known field type, thus this -test ends up exercising and verifying nearly all of the parsing and -serialization code in the whole library. - -TODO(kenton): Merge with wire_format_test? It doesn't make a whole lot of -sense to call this a test of the "message" module, which only declares an -abstract interface. -""" - -__author__ = 'gps@google.com (Gregory P. Smith)' - -import collections -import copy -import math -import operator -import pickle -import pydoc -import sys -import unittest -import warnings - -cmp = lambda x, y: (x > y) - (x < y) - -from google.protobuf import map_proto2_unittest_pb2 -from google.protobuf import map_unittest_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import unittest_proto3_arena_pb2 -from google.protobuf import descriptor -from google.protobuf.internal import api_implementation -from google.protobuf.internal import encoder -from google.protobuf.internal import more_extensions_pb2 -from google.protobuf.internal import packed_field_test_pb2 -from google.protobuf.internal import test_util -from google.protobuf.internal import test_proto3_optional_pb2 -from google.protobuf.internal import testing_refleaks -from google.protobuf import message -from google.protobuf.internal import _parameterized - -UCS2_MAXUNICODE = 65535 - -warnings.simplefilter('error', DeprecationWarning) - - -@_parameterized.named_parameters(('_proto2', unittest_pb2), - ('_proto3', unittest_proto3_arena_pb2)) -@testing_refleaks.TestCase -class MessageTest(unittest.TestCase): - - def testBadUtf8String(self, message_module): - if api_implementation.Type() != 'python': - self.skipTest('Skipping testBadUtf8String, currently only the python ' - 'api implementation raises UnicodeDecodeError when a ' - 'string field contains bad utf-8.') - bad_utf8_data = test_util.GoldenFileData('bad_utf8_string') - with self.assertRaises(UnicodeDecodeError) as context: - message_module.TestAllTypes.FromString(bad_utf8_data) - self.assertIn('TestAllTypes.optional_string', str(context.exception)) - - def testGoldenMessage(self, message_module): - # Proto3 doesn't have the "default_foo" members or foreign enums, - # and doesn't preserve unknown fields, so for proto3 we use a golden - # message that doesn't have these fields set. - if message_module is unittest_pb2: - golden_data = test_util.GoldenFileData('golden_message_oneof_implemented') - else: - golden_data = test_util.GoldenFileData('golden_message_proto3') - - golden_message = message_module.TestAllTypes() - golden_message.ParseFromString(golden_data) - if message_module is unittest_pb2: - test_util.ExpectAllFieldsSet(self, golden_message) - self.assertEqual(golden_data, golden_message.SerializeToString()) - golden_copy = copy.deepcopy(golden_message) - self.assertEqual(golden_data, golden_copy.SerializeToString()) - - def testGoldenPackedMessage(self, message_module): - golden_data = test_util.GoldenFileData('golden_packed_fields_message') - golden_message = message_module.TestPackedTypes() - parsed_bytes = golden_message.ParseFromString(golden_data) - all_set = message_module.TestPackedTypes() - test_util.SetAllPackedFields(all_set) - self.assertEqual(parsed_bytes, len(golden_data)) - self.assertEqual(all_set, golden_message) - self.assertEqual(golden_data, all_set.SerializeToString()) - golden_copy = copy.deepcopy(golden_message) - self.assertEqual(golden_data, golden_copy.SerializeToString()) - - def testParseErrors(self, message_module): - msg = message_module.TestAllTypes() - self.assertRaises(TypeError, msg.FromString, 0) - self.assertRaises(Exception, msg.FromString, '0') - # TODO(jieluo): Fix cpp extension to raise error instead of warning. - # b/27494216 - end_tag = encoder.TagBytes(1, 4) - if (api_implementation.Type() == 'python' or - api_implementation.Type() == 'upb'): - with self.assertRaises(message.DecodeError) as context: - msg.FromString(end_tag) - if api_implementation.Type() == 'python': - # Only pure-Python has an error message this specific. - self.assertEqual('Unexpected end-group tag.', str(context.exception)) - - # Field number 0 is illegal. - self.assertRaises(message.DecodeError, msg.FromString, b'\3\4') - - def testDeterminismParameters(self, message_module): - # This message is always deterministically serialized, even if determinism - # is disabled, so we can use it to verify that all the determinism - # parameters work correctly. - golden_data = (b'\xe2\x02\nOne string' - b'\xe2\x02\nTwo string' - b'\xe2\x02\nRed string' - b'\xe2\x02\x0bBlue string') - golden_message = message_module.TestAllTypes() - golden_message.repeated_string.extend([ - 'One string', - 'Two string', - 'Red string', - 'Blue string', - ]) - self.assertEqual(golden_data, - golden_message.SerializeToString(deterministic=None)) - self.assertEqual(golden_data, - golden_message.SerializeToString(deterministic=False)) - self.assertEqual(golden_data, - golden_message.SerializeToString(deterministic=True)) - - class BadArgError(Exception): - pass - - class BadArg(object): - - def __nonzero__(self): - raise BadArgError() - - def __bool__(self): - raise BadArgError() - - with self.assertRaises(BadArgError): - golden_message.SerializeToString(deterministic=BadArg()) - - def testPickleSupport(self, message_module): - golden_data = test_util.GoldenFileData('golden_message') - golden_message = message_module.TestAllTypes() - golden_message.ParseFromString(golden_data) - pickled_message = pickle.dumps(golden_message) - - unpickled_message = pickle.loads(pickled_message) - self.assertEqual(unpickled_message, golden_message) - - def testPickleNestedMessage(self, message_module): - golden_message = message_module.TestPickleNestedMessage.NestedMessage(bb=1) - pickled_message = pickle.dumps(golden_message) - unpickled_message = pickle.loads(pickled_message) - self.assertEqual(unpickled_message, golden_message) - - def testPickleNestedNestedMessage(self, message_module): - cls = message_module.TestPickleNestedMessage.NestedMessage - golden_message = cls.NestedNestedMessage(cc=1) - pickled_message = pickle.dumps(golden_message) - unpickled_message = pickle.loads(pickled_message) - self.assertEqual(unpickled_message, golden_message) - - def testPositiveInfinity(self, message_module): - if message_module is unittest_pb2: - golden_data = (b'\x5D\x00\x00\x80\x7F' - b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F' - b'\xCD\x02\x00\x00\x80\x7F' - b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F') - else: - golden_data = (b'\x5D\x00\x00\x80\x7F' - b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F' - b'\xCA\x02\x04\x00\x00\x80\x7F' - b'\xD2\x02\x08\x00\x00\x00\x00\x00\x00\xF0\x7F') - - golden_message = message_module.TestAllTypes() - golden_message.ParseFromString(golden_data) - self.assertEqual(golden_message.optional_float, math.inf) - self.assertEqual(golden_message.optional_double, math.inf) - self.assertEqual(golden_message.repeated_float[0], math.inf) - self.assertEqual(golden_message.repeated_double[0], math.inf) - self.assertEqual(golden_data, golden_message.SerializeToString()) - - def testNegativeInfinity(self, message_module): - if message_module is unittest_pb2: - golden_data = (b'\x5D\x00\x00\x80\xFF' - b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF' - b'\xCD\x02\x00\x00\x80\xFF' - b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF') - else: - golden_data = (b'\x5D\x00\x00\x80\xFF' - b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF' - b'\xCA\x02\x04\x00\x00\x80\xFF' - b'\xD2\x02\x08\x00\x00\x00\x00\x00\x00\xF0\xFF') - - golden_message = message_module.TestAllTypes() - golden_message.ParseFromString(golden_data) - self.assertEqual(golden_message.optional_float, -math.inf) - self.assertEqual(golden_message.optional_double, -math.inf) - self.assertEqual(golden_message.repeated_float[0], -math.inf) - self.assertEqual(golden_message.repeated_double[0], -math.inf) - self.assertEqual(golden_data, golden_message.SerializeToString()) - - def testNotANumber(self, message_module): - golden_data = (b'\x5D\x00\x00\xC0\x7F' - b'\x61\x00\x00\x00\x00\x00\x00\xF8\x7F' - b'\xCD\x02\x00\x00\xC0\x7F' - b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F') - golden_message = message_module.TestAllTypes() - golden_message.ParseFromString(golden_data) - self.assertTrue(math.isnan(golden_message.optional_float)) - self.assertTrue(math.isnan(golden_message.optional_double)) - self.assertTrue(math.isnan(golden_message.repeated_float[0])) - self.assertTrue(math.isnan(golden_message.repeated_double[0])) - - # The protocol buffer may serialize to any one of multiple different - # representations of a NaN. Rather than verify a specific representation, - # verify the serialized string can be converted into a correctly - # behaving protocol buffer. - serialized = golden_message.SerializeToString() - message = message_module.TestAllTypes() - message.ParseFromString(serialized) - self.assertTrue(math.isnan(message.optional_float)) - self.assertTrue(math.isnan(message.optional_double)) - self.assertTrue(math.isnan(message.repeated_float[0])) - self.assertTrue(math.isnan(message.repeated_double[0])) - - def testPositiveInfinityPacked(self, message_module): - golden_data = (b'\xA2\x06\x04\x00\x00\x80\x7F' - b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F') - golden_message = message_module.TestPackedTypes() - golden_message.ParseFromString(golden_data) - self.assertEqual(golden_message.packed_float[0], math.inf) - self.assertEqual(golden_message.packed_double[0], math.inf) - self.assertEqual(golden_data, golden_message.SerializeToString()) - - def testNegativeInfinityPacked(self, message_module): - golden_data = (b'\xA2\x06\x04\x00\x00\x80\xFF' - b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF') - golden_message = message_module.TestPackedTypes() - golden_message.ParseFromString(golden_data) - self.assertEqual(golden_message.packed_float[0], -math.inf) - self.assertEqual(golden_message.packed_double[0], -math.inf) - self.assertEqual(golden_data, golden_message.SerializeToString()) - - def testNotANumberPacked(self, message_module): - golden_data = (b'\xA2\x06\x04\x00\x00\xC0\x7F' - b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F') - golden_message = message_module.TestPackedTypes() - golden_message.ParseFromString(golden_data) - self.assertTrue(math.isnan(golden_message.packed_float[0])) - self.assertTrue(math.isnan(golden_message.packed_double[0])) - - serialized = golden_message.SerializeToString() - message = message_module.TestPackedTypes() - message.ParseFromString(serialized) - self.assertTrue(math.isnan(message.packed_float[0])) - self.assertTrue(math.isnan(message.packed_double[0])) - - def testExtremeFloatValues(self, message_module): - message = message_module.TestAllTypes() - - # Most positive exponent, no significand bits set. - kMostPosExponentNoSigBits = math.pow(2, 127) - message.optional_float = kMostPosExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == kMostPosExponentNoSigBits) - - # Most positive exponent, one significand bit set. - kMostPosExponentOneSigBit = 1.5 * math.pow(2, 127) - message.optional_float = kMostPosExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == kMostPosExponentOneSigBit) - - # Repeat last two cases with values of same magnitude, but negative. - message.optional_float = -kMostPosExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == -kMostPosExponentNoSigBits) - - message.optional_float = -kMostPosExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == -kMostPosExponentOneSigBit) - - # Most negative exponent, no significand bits set. - kMostNegExponentNoSigBits = math.pow(2, -127) - message.optional_float = kMostNegExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == kMostNegExponentNoSigBits) - - # Most negative exponent, one significand bit set. - kMostNegExponentOneSigBit = 1.5 * math.pow(2, -127) - message.optional_float = kMostNegExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == kMostNegExponentOneSigBit) - - # Repeat last two cases with values of the same magnitude, but negative. - message.optional_float = -kMostNegExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == -kMostNegExponentNoSigBits) - - message.optional_float = -kMostNegExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_float == -kMostNegExponentOneSigBit) - - # Max 4 bytes float value - max_float = float.fromhex('0x1.fffffep+127') - message.optional_float = max_float - self.assertAlmostEqual(message.optional_float, max_float) - serialized_data = message.SerializeToString() - message.ParseFromString(serialized_data) - self.assertAlmostEqual(message.optional_float, max_float) - - # Test set double to float field. - message.optional_float = 3.4028235e+39 - self.assertEqual(message.optional_float, float('inf')) - serialized_data = message.SerializeToString() - message.ParseFromString(serialized_data) - self.assertEqual(message.optional_float, float('inf')) - - message.optional_float = -3.4028235e+39 - self.assertEqual(message.optional_float, float('-inf')) - - message.optional_float = 1.4028235e-39 - self.assertAlmostEqual(message.optional_float, 1.4028235e-39) - - def testExtremeDoubleValues(self, message_module): - message = message_module.TestAllTypes() - - # Most positive exponent, no significand bits set. - kMostPosExponentNoSigBits = math.pow(2, 1023) - message.optional_double = kMostPosExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == kMostPosExponentNoSigBits) - - # Most positive exponent, one significand bit set. - kMostPosExponentOneSigBit = 1.5 * math.pow(2, 1023) - message.optional_double = kMostPosExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == kMostPosExponentOneSigBit) - - # Repeat last two cases with values of same magnitude, but negative. - message.optional_double = -kMostPosExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == -kMostPosExponentNoSigBits) - - message.optional_double = -kMostPosExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == -kMostPosExponentOneSigBit) - - # Most negative exponent, no significand bits set. - kMostNegExponentNoSigBits = math.pow(2, -1023) - message.optional_double = kMostNegExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == kMostNegExponentNoSigBits) - - # Most negative exponent, one significand bit set. - kMostNegExponentOneSigBit = 1.5 * math.pow(2, -1023) - message.optional_double = kMostNegExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == kMostNegExponentOneSigBit) - - # Repeat last two cases with values of the same magnitude, but negative. - message.optional_double = -kMostNegExponentNoSigBits - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == -kMostNegExponentNoSigBits) - - message.optional_double = -kMostNegExponentOneSigBit - message.ParseFromString(message.SerializeToString()) - self.assertTrue(message.optional_double == -kMostNegExponentOneSigBit) - - def testFloatPrinting(self, message_module): - message = message_module.TestAllTypes() - message.optional_float = 2.0 - self.assertEqual(str(message), 'optional_float: 2.0\n') - - def testHighPrecisionFloatPrinting(self, message_module): - msg = message_module.TestAllTypes() - msg.optional_float = 0.12345678912345678 - old_float = msg.optional_float - msg.ParseFromString(msg.SerializeToString()) - self.assertEqual(old_float, msg.optional_float) - - def testHighPrecisionDoublePrinting(self, message_module): - msg = message_module.TestAllTypes() - msg.optional_double = 0.12345678912345678 - self.assertEqual(str(msg), 'optional_double: 0.12345678912345678\n') - - def testUnknownFieldPrinting(self, message_module): - populated = message_module.TestAllTypes() - test_util.SetAllNonLazyFields(populated) - empty = message_module.TestEmptyMessage() - empty.ParseFromString(populated.SerializeToString()) - self.assertEqual(str(empty), '') - - def testAppendRepeatedCompositeField(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_nested_message.append( - message_module.TestAllTypes.NestedMessage(bb=1)) - nested = message_module.TestAllTypes.NestedMessage(bb=2) - msg.repeated_nested_message.append(nested) - try: - msg.repeated_nested_message.append(1) - except TypeError: - pass - self.assertEqual(2, len(msg.repeated_nested_message)) - self.assertEqual([1, 2], [m.bb for m in msg.repeated_nested_message]) - - def testInsertRepeatedCompositeField(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_nested_message.insert( - -1, message_module.TestAllTypes.NestedMessage(bb=1)) - sub_msg = msg.repeated_nested_message[0] - msg.repeated_nested_message.insert( - 0, message_module.TestAllTypes.NestedMessage(bb=2)) - msg.repeated_nested_message.insert( - 99, message_module.TestAllTypes.NestedMessage(bb=3)) - msg.repeated_nested_message.insert( - -2, message_module.TestAllTypes.NestedMessage(bb=-1)) - msg.repeated_nested_message.insert( - -1000, message_module.TestAllTypes.NestedMessage(bb=-1000)) - try: - msg.repeated_nested_message.insert(1, 999) - except TypeError: - pass - self.assertEqual(5, len(msg.repeated_nested_message)) - self.assertEqual([-1000, 2, -1, 1, 3], - [m.bb for m in msg.repeated_nested_message]) - self.assertEqual( - str(msg), 'repeated_nested_message {\n' - ' bb: -1000\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: 2\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: -1\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: 1\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: 3\n' - '}\n') - self.assertEqual(sub_msg.bb, 1) - - def testMergeFromRepeatedField(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_int32.append(1) - msg.repeated_int32.append(3) - msg.repeated_nested_message.add(bb=1) - msg.repeated_nested_message.add(bb=2) - other_msg = message_module.TestAllTypes() - other_msg.repeated_nested_message.add(bb=3) - other_msg.repeated_nested_message.add(bb=4) - other_msg.repeated_int32.append(5) - other_msg.repeated_int32.append(7) - - msg.repeated_int32.MergeFrom(other_msg.repeated_int32) - self.assertEqual(4, len(msg.repeated_int32)) - - msg.repeated_nested_message.MergeFrom(other_msg.repeated_nested_message) - self.assertEqual([1, 2, 3, 4], [m.bb for m in msg.repeated_nested_message]) - - def testAddWrongRepeatedNestedField(self, message_module): - msg = message_module.TestAllTypes() - try: - msg.repeated_nested_message.add('wrong') - except TypeError: - pass - try: - msg.repeated_nested_message.add(value_field='wrong') - except ValueError: - pass - self.assertEqual(len(msg.repeated_nested_message), 0) - - def testRepeatedContains(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_int32.extend([1, 2, 3]) - self.assertIn(2, msg.repeated_int32) - self.assertNotIn(0, msg.repeated_int32) - - msg.repeated_nested_message.add(bb=1) - sub_msg1 = msg.repeated_nested_message[0] - sub_msg2 = message_module.TestAllTypes.NestedMessage(bb=2) - sub_msg3 = message_module.TestAllTypes.NestedMessage(bb=3) - msg.repeated_nested_message.append(sub_msg2) - msg.repeated_nested_message.insert(0, sub_msg3) - self.assertIn(sub_msg1, msg.repeated_nested_message) - self.assertIn(sub_msg2, msg.repeated_nested_message) - self.assertIn(sub_msg3, msg.repeated_nested_message) - - def testRepeatedScalarIterable(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_int32.extend([1, 2, 3]) - add = 0 - for item in msg.repeated_int32: - add += item - self.assertEqual(add, 6) - - def testRepeatedNestedFieldIteration(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_nested_message.add(bb=1) - msg.repeated_nested_message.add(bb=2) - msg.repeated_nested_message.add(bb=3) - msg.repeated_nested_message.add(bb=4) - - self.assertEqual([1, 2, 3, 4], [m.bb for m in msg.repeated_nested_message]) - self.assertEqual([4, 3, 2, 1], - [m.bb for m in reversed(msg.repeated_nested_message)]) - self.assertEqual([4, 3, 2, 1], - [m.bb for m in msg.repeated_nested_message[::-1]]) - - def testSortingRepeatedScalarFieldsDefaultComparator(self, message_module): - """Check some different types with the default comparator.""" - message = message_module.TestAllTypes() - - # TODO(mattp): would testing more scalar types strengthen test? - message.repeated_int32.append(1) - message.repeated_int32.append(3) - message.repeated_int32.append(2) - message.repeated_int32.sort() - self.assertEqual(message.repeated_int32[0], 1) - self.assertEqual(message.repeated_int32[1], 2) - self.assertEqual(message.repeated_int32[2], 3) - self.assertEqual(str(message.repeated_int32), str([1, 2, 3])) - - message.repeated_float.append(1.1) - message.repeated_float.append(1.3) - message.repeated_float.append(1.2) - message.repeated_float.sort() - self.assertAlmostEqual(message.repeated_float[0], 1.1) - self.assertAlmostEqual(message.repeated_float[1], 1.2) - self.assertAlmostEqual(message.repeated_float[2], 1.3) - - message.repeated_string.append('a') - message.repeated_string.append('c') - message.repeated_string.append('b') - message.repeated_string.sort() - self.assertEqual(message.repeated_string[0], 'a') - self.assertEqual(message.repeated_string[1], 'b') - self.assertEqual(message.repeated_string[2], 'c') - self.assertEqual(str(message.repeated_string), str([u'a', u'b', u'c'])) - - message.repeated_bytes.append(b'a') - message.repeated_bytes.append(b'c') - message.repeated_bytes.append(b'b') - message.repeated_bytes.sort() - self.assertEqual(message.repeated_bytes[0], b'a') - self.assertEqual(message.repeated_bytes[1], b'b') - self.assertEqual(message.repeated_bytes[2], b'c') - self.assertEqual(str(message.repeated_bytes), str([b'a', b'b', b'c'])) - - def testSortingRepeatedScalarFieldsCustomComparator(self, message_module): - """Check some different types with custom comparator.""" - message = message_module.TestAllTypes() - - message.repeated_int32.append(-3) - message.repeated_int32.append(-2) - message.repeated_int32.append(-1) - message.repeated_int32.sort(key=abs) - self.assertEqual(message.repeated_int32[0], -1) - self.assertEqual(message.repeated_int32[1], -2) - self.assertEqual(message.repeated_int32[2], -3) - - message.repeated_string.append('aaa') - message.repeated_string.append('bb') - message.repeated_string.append('c') - message.repeated_string.sort(key=len) - self.assertEqual(message.repeated_string[0], 'c') - self.assertEqual(message.repeated_string[1], 'bb') - self.assertEqual(message.repeated_string[2], 'aaa') - - def testSortingRepeatedCompositeFieldsCustomComparator(self, message_module): - """Check passing a custom comparator to sort a repeated composite field.""" - message = message_module.TestAllTypes() - - message.repeated_nested_message.add().bb = 1 - message.repeated_nested_message.add().bb = 3 - message.repeated_nested_message.add().bb = 2 - message.repeated_nested_message.add().bb = 6 - message.repeated_nested_message.add().bb = 5 - message.repeated_nested_message.add().bb = 4 - message.repeated_nested_message.sort(key=operator.attrgetter('bb')) - self.assertEqual(message.repeated_nested_message[0].bb, 1) - self.assertEqual(message.repeated_nested_message[1].bb, 2) - self.assertEqual(message.repeated_nested_message[2].bb, 3) - self.assertEqual(message.repeated_nested_message[3].bb, 4) - self.assertEqual(message.repeated_nested_message[4].bb, 5) - self.assertEqual(message.repeated_nested_message[5].bb, 6) - self.assertEqual( - str(message.repeated_nested_message), - '[bb: 1\n, bb: 2\n, bb: 3\n, bb: 4\n, bb: 5\n, bb: 6\n]') - - def testSortingRepeatedCompositeFieldsStable(self, message_module): - """Check passing a custom comparator to sort a repeated composite field.""" - message = message_module.TestAllTypes() - - message.repeated_nested_message.add().bb = 21 - message.repeated_nested_message.add().bb = 20 - message.repeated_nested_message.add().bb = 13 - message.repeated_nested_message.add().bb = 33 - message.repeated_nested_message.add().bb = 11 - message.repeated_nested_message.add().bb = 24 - message.repeated_nested_message.add().bb = 10 - message.repeated_nested_message.sort(key=lambda z: z.bb // 10) - self.assertEqual([13, 11, 10, 21, 20, 24, 33], - [n.bb for n in message.repeated_nested_message]) - - # Make sure that for the C++ implementation, the underlying fields - # are actually reordered. - pb = message.SerializeToString() - message.Clear() - message.MergeFromString(pb) - self.assertEqual([13, 11, 10, 21, 20, 24, 33], - [n.bb for n in message.repeated_nested_message]) - - def testRepeatedCompositeFieldSortArguments(self, message_module): - """Check sorting a repeated composite field using list.sort() arguments.""" - message = message_module.TestAllTypes() - - get_bb = operator.attrgetter('bb') - message.repeated_nested_message.add().bb = 1 - message.repeated_nested_message.add().bb = 3 - message.repeated_nested_message.add().bb = 2 - message.repeated_nested_message.add().bb = 6 - message.repeated_nested_message.add().bb = 5 - message.repeated_nested_message.add().bb = 4 - message.repeated_nested_message.sort(key=get_bb) - self.assertEqual([k.bb for k in message.repeated_nested_message], - [1, 2, 3, 4, 5, 6]) - message.repeated_nested_message.sort(key=get_bb, reverse=True) - self.assertEqual([k.bb for k in message.repeated_nested_message], - [6, 5, 4, 3, 2, 1]) - - def testRepeatedScalarFieldSortArguments(self, message_module): - """Check sorting a scalar field using list.sort() arguments.""" - message = message_module.TestAllTypes() - - message.repeated_int32.append(-3) - message.repeated_int32.append(-2) - message.repeated_int32.append(-1) - message.repeated_int32.sort(key=abs) - self.assertEqual(list(message.repeated_int32), [-1, -2, -3]) - message.repeated_int32.sort(key=abs, reverse=True) - self.assertEqual(list(message.repeated_int32), [-3, -2, -1]) - - message.repeated_string.append('aaa') - message.repeated_string.append('bb') - message.repeated_string.append('c') - message.repeated_string.sort(key=len) - self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa']) - message.repeated_string.sort(key=len, reverse=True) - self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c']) - - def testRepeatedFieldsComparable(self, message_module): - m1 = message_module.TestAllTypes() - m2 = message_module.TestAllTypes() - m1.repeated_int32.append(0) - m1.repeated_int32.append(1) - m1.repeated_int32.append(2) - m2.repeated_int32.append(0) - m2.repeated_int32.append(1) - m2.repeated_int32.append(2) - m1.repeated_nested_message.add().bb = 1 - m1.repeated_nested_message.add().bb = 2 - m1.repeated_nested_message.add().bb = 3 - m2.repeated_nested_message.add().bb = 1 - m2.repeated_nested_message.add().bb = 2 - m2.repeated_nested_message.add().bb = 3 - - def testRepeatedFieldsAreSequences(self, message_module): - m = message_module.TestAllTypes() - self.assertIsInstance(m.repeated_int32, collections.abc.MutableSequence) - self.assertIsInstance(m.repeated_nested_message, - collections.abc.MutableSequence) - - def testRepeatedFieldsNotHashable(self, message_module): - m = message_module.TestAllTypes() - with self.assertRaises(TypeError): - hash(m.repeated_int32) - with self.assertRaises(TypeError): - hash(m.repeated_nested_message) - - def testRepeatedFieldInsideNestedMessage(self, message_module): - m = message_module.NestedTestAllTypes() - m.payload.repeated_int32.extend([]) - self.assertTrue(m.HasField('payload')) - - def testMergeFrom(self, message_module): - m1 = message_module.TestAllTypes() - m2 = message_module.TestAllTypes() - # Cpp extension will lazily create a sub message which is immutable. - nested = m1.optional_nested_message - self.assertEqual(0, nested.bb) - m2.optional_nested_message.bb = 1 - # Make sure cmessage pointing to a mutable message after merge instead of - # the lazily created message. - m1.MergeFrom(m2) - self.assertEqual(1, nested.bb) - - # Test more nested sub message. - msg1 = message_module.NestedTestAllTypes() - msg2 = message_module.NestedTestAllTypes() - nested = msg1.child.payload.optional_nested_message - self.assertEqual(0, nested.bb) - msg2.child.payload.optional_nested_message.bb = 1 - msg1.MergeFrom(msg2) - self.assertEqual(1, nested.bb) - - # Test repeated field. - self.assertEqual(msg1.payload.repeated_nested_message, - msg1.payload.repeated_nested_message) - nested = msg2.payload.repeated_nested_message.add() - nested.bb = 1 - msg1.MergeFrom(msg2) - self.assertEqual(1, len(msg1.payload.repeated_nested_message)) - self.assertEqual(1, nested.bb) - - def testMergeFromString(self, message_module): - m1 = message_module.TestAllTypes() - m2 = message_module.TestAllTypes() - # Cpp extension will lazily create a sub message which is immutable. - self.assertEqual(0, m1.optional_nested_message.bb) - m2.optional_nested_message.bb = 1 - # Make sure cmessage pointing to a mutable message after merge instead of - # the lazily created message. - m1.MergeFromString(m2.SerializeToString()) - self.assertEqual(1, m1.optional_nested_message.bb) - - def testMergeFromStringUsingMemoryView(self, message_module): - m2 = message_module.TestAllTypes() - m2.optional_string = 'scalar string' - m2.repeated_string.append('repeated string') - m2.optional_bytes = b'scalar bytes' - m2.repeated_bytes.append(b'repeated bytes') - - serialized = m2.SerializeToString() - memview = memoryview(serialized) - m1 = message_module.TestAllTypes.FromString(memview) - - self.assertEqual(m1.optional_bytes, b'scalar bytes') - self.assertEqual(m1.repeated_bytes, [b'repeated bytes']) - self.assertEqual(m1.optional_string, 'scalar string') - self.assertEqual(m1.repeated_string, ['repeated string']) - # Make sure that the memoryview was correctly converted to bytes, and - # that a sub-sliced memoryview is not being used. - self.assertIsInstance(m1.optional_bytes, bytes) - self.assertIsInstance(m1.repeated_bytes[0], bytes) - self.assertIsInstance(m1.optional_string, str) - self.assertIsInstance(m1.repeated_string[0], str) - - def testMergeFromEmpty(self, message_module): - m1 = message_module.TestAllTypes() - # Cpp extension will lazily create a sub message which is immutable. - self.assertEqual(0, m1.optional_nested_message.bb) - self.assertFalse(m1.HasField('optional_nested_message')) - # Make sure the sub message is still immutable after merge from empty. - m1.MergeFromString(b'') # field state should not change - self.assertFalse(m1.HasField('optional_nested_message')) - - def ensureNestedMessageExists(self, msg, attribute): - """Make sure that a nested message object exists. - - As soon as a nested message attribute is accessed, it will be present in the - _fields dict, without being marked as actually being set. - """ - getattr(msg, attribute) - self.assertFalse(msg.HasField(attribute)) - - def testOneofGetCaseNonexistingField(self, message_module): - m = message_module.TestAllTypes() - self.assertRaises(ValueError, m.WhichOneof, 'no_such_oneof_field') - self.assertRaises(Exception, m.WhichOneof, 0) - - def testOneofDefaultValues(self, message_module): - m = message_module.TestAllTypes() - self.assertIs(None, m.WhichOneof('oneof_field')) - self.assertFalse(m.HasField('oneof_field')) - self.assertFalse(m.HasField('oneof_uint32')) - - # Oneof is set even when setting it to a default value. - m.oneof_uint32 = 0 - self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) - self.assertTrue(m.HasField('oneof_field')) - self.assertTrue(m.HasField('oneof_uint32')) - self.assertFalse(m.HasField('oneof_string')) - - m.oneof_string = '' - self.assertEqual('oneof_string', m.WhichOneof('oneof_field')) - self.assertTrue(m.HasField('oneof_string')) - self.assertFalse(m.HasField('oneof_uint32')) - - def testOneofSemantics(self, message_module): - m = message_module.TestAllTypes() - self.assertIs(None, m.WhichOneof('oneof_field')) - - m.oneof_uint32 = 11 - self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) - self.assertTrue(m.HasField('oneof_uint32')) - - m.oneof_string = u'foo' - self.assertEqual('oneof_string', m.WhichOneof('oneof_field')) - self.assertFalse(m.HasField('oneof_uint32')) - self.assertTrue(m.HasField('oneof_string')) - - # Read nested message accessor without accessing submessage. - m.oneof_nested_message - self.assertEqual('oneof_string', m.WhichOneof('oneof_field')) - self.assertTrue(m.HasField('oneof_string')) - self.assertFalse(m.HasField('oneof_nested_message')) - - # Read accessor of nested message without accessing submessage. - m.oneof_nested_message.bb - self.assertEqual('oneof_string', m.WhichOneof('oneof_field')) - self.assertTrue(m.HasField('oneof_string')) - self.assertFalse(m.HasField('oneof_nested_message')) - - m.oneof_nested_message.bb = 11 - self.assertEqual('oneof_nested_message', m.WhichOneof('oneof_field')) - self.assertFalse(m.HasField('oneof_string')) - self.assertTrue(m.HasField('oneof_nested_message')) - - m.oneof_bytes = b'bb' - self.assertEqual('oneof_bytes', m.WhichOneof('oneof_field')) - self.assertFalse(m.HasField('oneof_nested_message')) - self.assertTrue(m.HasField('oneof_bytes')) - - def testOneofCompositeFieldReadAccess(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - - self.ensureNestedMessageExists(m, 'oneof_nested_message') - self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) - self.assertEqual(11, m.oneof_uint32) - - def testOneofWhichOneof(self, message_module): - m = message_module.TestAllTypes() - self.assertIs(None, m.WhichOneof('oneof_field')) - if message_module is unittest_pb2: - self.assertFalse(m.HasField('oneof_field')) - - m.oneof_uint32 = 11 - self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) - if message_module is unittest_pb2: - self.assertTrue(m.HasField('oneof_field')) - - m.oneof_bytes = b'bb' - self.assertEqual('oneof_bytes', m.WhichOneof('oneof_field')) - - m.ClearField('oneof_bytes') - self.assertIs(None, m.WhichOneof('oneof_field')) - if message_module is unittest_pb2: - self.assertFalse(m.HasField('oneof_field')) - - def testOneofClearField(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - m.ClearField('oneof_field') - if message_module is unittest_pb2: - self.assertFalse(m.HasField('oneof_field')) - self.assertFalse(m.HasField('oneof_uint32')) - self.assertIs(None, m.WhichOneof('oneof_field')) - - def testOneofClearSetField(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - m.ClearField('oneof_uint32') - if message_module is unittest_pb2: - self.assertFalse(m.HasField('oneof_field')) - self.assertFalse(m.HasField('oneof_uint32')) - self.assertIs(None, m.WhichOneof('oneof_field')) - - def testOneofClearUnsetField(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - self.ensureNestedMessageExists(m, 'oneof_nested_message') - m.ClearField('oneof_nested_message') - self.assertEqual(11, m.oneof_uint32) - if message_module is unittest_pb2: - self.assertTrue(m.HasField('oneof_field')) - self.assertTrue(m.HasField('oneof_uint32')) - self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) - - def testOneofDeserialize(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - m2 = message_module.TestAllTypes() - m2.ParseFromString(m.SerializeToString()) - self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) - - def testOneofCopyFrom(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - m2 = message_module.TestAllTypes() - m2.CopyFrom(m) - self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) - - def testOneofNestedMergeFrom(self, message_module): - m = message_module.NestedTestAllTypes() - m.payload.oneof_uint32 = 11 - m2 = message_module.NestedTestAllTypes() - m2.payload.oneof_bytes = b'bb' - m2.child.payload.oneof_bytes = b'bb' - m2.MergeFrom(m) - self.assertEqual('oneof_uint32', m2.payload.WhichOneof('oneof_field')) - self.assertEqual('oneof_bytes', m2.child.payload.WhichOneof('oneof_field')) - - def testOneofMessageMergeFrom(self, message_module): - m = message_module.NestedTestAllTypes() - m.payload.oneof_nested_message.bb = 11 - m.child.payload.oneof_nested_message.bb = 12 - m2 = message_module.NestedTestAllTypes() - m2.payload.oneof_uint32 = 13 - m2.MergeFrom(m) - self.assertEqual('oneof_nested_message', - m2.payload.WhichOneof('oneof_field')) - self.assertEqual('oneof_nested_message', - m2.child.payload.WhichOneof('oneof_field')) - - def testOneofNestedMessageInit(self, message_module): - m = message_module.TestAllTypes( - oneof_nested_message=message_module.TestAllTypes.NestedMessage()) - self.assertEqual('oneof_nested_message', m.WhichOneof('oneof_field')) - - def testOneofClear(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - m.Clear() - self.assertIsNone(m.WhichOneof('oneof_field')) - m.oneof_bytes = b'bb' - self.assertEqual('oneof_bytes', m.WhichOneof('oneof_field')) - - def testAssignByteStringToUnicodeField(self, message_module): - """Assigning a byte string to a string field should result - - in the value being converted to a Unicode string. - """ - m = message_module.TestAllTypes() - m.optional_string = str('') - self.assertIsInstance(m.optional_string, str) - - def testLongValuedSlice(self, message_module): - """It should be possible to use int-valued indices in slices. - - This didn't used to work in the v2 C++ implementation. - """ - m = message_module.TestAllTypes() - - # Repeated scalar - m.repeated_int32.append(1) - sl = m.repeated_int32[int(0):int(len(m.repeated_int32))] - self.assertEqual(len(m.repeated_int32), len(sl)) - - # Repeated composite - m.repeated_nested_message.add().bb = 3 - sl = m.repeated_nested_message[int(0):int(len(m.repeated_nested_message))] - self.assertEqual(len(m.repeated_nested_message), len(sl)) - - def testExtendShouldNotSwallowExceptions(self, message_module): - """This didn't use to work in the v2 C++ implementation.""" - m = message_module.TestAllTypes() - with self.assertRaises(NameError) as _: - m.repeated_int32.extend(a for i in range(10)) # pylint: disable=undefined-variable - with self.assertRaises(NameError) as _: - m.repeated_nested_enum.extend(a for i in range(10)) # pylint: disable=undefined-variable - - FALSY_VALUES = [None, False, 0, 0.0, b'', u'', bytearray(), [], {}, set()] - - def testExtendInt32WithNothing(self, message_module): - """Test no-ops extending repeated int32 fields.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_int32) - - # TODO(ptucker): Deprecate this behavior. b/18413862 - for falsy_value in MessageTest.FALSY_VALUES: - m.repeated_int32.extend(falsy_value) - self.assertSequenceEqual([], m.repeated_int32) - - m.repeated_int32.extend([]) - self.assertSequenceEqual([], m.repeated_int32) - - def testExtendFloatWithNothing(self, message_module): - """Test no-ops extending repeated float fields.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_float) - - # TODO(ptucker): Deprecate this behavior. b/18413862 - for falsy_value in MessageTest.FALSY_VALUES: - m.repeated_float.extend(falsy_value) - self.assertSequenceEqual([], m.repeated_float) - - m.repeated_float.extend([]) - self.assertSequenceEqual([], m.repeated_float) - - def testExtendStringWithNothing(self, message_module): - """Test no-ops extending repeated string fields.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_string) - - # TODO(ptucker): Deprecate this behavior. b/18413862 - for falsy_value in MessageTest.FALSY_VALUES: - m.repeated_string.extend(falsy_value) - self.assertSequenceEqual([], m.repeated_string) - - m.repeated_string.extend([]) - self.assertSequenceEqual([], m.repeated_string) - - def testExtendInt32WithPythonList(self, message_module): - """Test extending repeated int32 fields with python lists.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_int32) - m.repeated_int32.extend([0]) - self.assertSequenceEqual([0], m.repeated_int32) - m.repeated_int32.extend([1, 2]) - self.assertSequenceEqual([0, 1, 2], m.repeated_int32) - m.repeated_int32.extend([3, 4]) - self.assertSequenceEqual([0, 1, 2, 3, 4], m.repeated_int32) - - def testExtendFloatWithPythonList(self, message_module): - """Test extending repeated float fields with python lists.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_float) - m.repeated_float.extend([0.0]) - self.assertSequenceEqual([0.0], m.repeated_float) - m.repeated_float.extend([1.0, 2.0]) - self.assertSequenceEqual([0.0, 1.0, 2.0], m.repeated_float) - m.repeated_float.extend([3.0, 4.0]) - self.assertSequenceEqual([0.0, 1.0, 2.0, 3.0, 4.0], m.repeated_float) - - def testExtendStringWithPythonList(self, message_module): - """Test extending repeated string fields with python lists.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_string) - m.repeated_string.extend(['']) - self.assertSequenceEqual([''], m.repeated_string) - m.repeated_string.extend(['11', '22']) - self.assertSequenceEqual(['', '11', '22'], m.repeated_string) - m.repeated_string.extend(['33', '44']) - self.assertSequenceEqual(['', '11', '22', '33', '44'], m.repeated_string) - - def testExtendStringWithString(self, message_module): - """Test extending repeated string fields with characters from a string.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_string) - m.repeated_string.extend('abc') - self.assertSequenceEqual(['a', 'b', 'c'], m.repeated_string) - - class TestIterable(object): - """This iterable object mimics the behavior of numpy.array. - - __nonzero__ fails for length > 1, and returns bool(item[0]) for length == 1. - - """ - - def __init__(self, values=None): - self._list = values or [] - - def __nonzero__(self): - size = len(self._list) - if size == 0: - return False - if size == 1: - return bool(self._list[0]) - raise ValueError('Truth value is ambiguous.') - - def __len__(self): - return len(self._list) - - def __iter__(self): - return self._list.__iter__() - - def testExtendInt32WithIterable(self, message_module): - """Test extending repeated int32 fields with iterable.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_int32) - m.repeated_int32.extend(MessageTest.TestIterable([])) - self.assertSequenceEqual([], m.repeated_int32) - m.repeated_int32.extend(MessageTest.TestIterable([0])) - self.assertSequenceEqual([0], m.repeated_int32) - m.repeated_int32.extend(MessageTest.TestIterable([1, 2])) - self.assertSequenceEqual([0, 1, 2], m.repeated_int32) - m.repeated_int32.extend(MessageTest.TestIterable([3, 4])) - self.assertSequenceEqual([0, 1, 2, 3, 4], m.repeated_int32) - - def testExtendFloatWithIterable(self, message_module): - """Test extending repeated float fields with iterable.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_float) - m.repeated_float.extend(MessageTest.TestIterable([])) - self.assertSequenceEqual([], m.repeated_float) - m.repeated_float.extend(MessageTest.TestIterable([0.0])) - self.assertSequenceEqual([0.0], m.repeated_float) - m.repeated_float.extend(MessageTest.TestIterable([1.0, 2.0])) - self.assertSequenceEqual([0.0, 1.0, 2.0], m.repeated_float) - m.repeated_float.extend(MessageTest.TestIterable([3.0, 4.0])) - self.assertSequenceEqual([0.0, 1.0, 2.0, 3.0, 4.0], m.repeated_float) - - def testExtendStringWithIterable(self, message_module): - """Test extending repeated string fields with iterable.""" - m = message_module.TestAllTypes() - self.assertSequenceEqual([], m.repeated_string) - m.repeated_string.extend(MessageTest.TestIterable([])) - self.assertSequenceEqual([], m.repeated_string) - m.repeated_string.extend(MessageTest.TestIterable([''])) - self.assertSequenceEqual([''], m.repeated_string) - m.repeated_string.extend(MessageTest.TestIterable(['1', '2'])) - self.assertSequenceEqual(['', '1', '2'], m.repeated_string) - m.repeated_string.extend(MessageTest.TestIterable(['3', '4'])) - self.assertSequenceEqual(['', '1', '2', '3', '4'], m.repeated_string) - - class TestIndex(object): - """This index object mimics the behavior of numpy.int64 and other types.""" - - def __init__(self, value=None): - self.value = value - - def __index__(self): - return self.value - - def testRepeatedIndexingWithIntIndex(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_int32.extend([1, 2, 3]) - self.assertEqual(1, msg.repeated_int32[MessageTest.TestIndex(0)]) - - def testRepeatedIndexingWithNegative1IntIndex(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_int32.extend([1, 2, 3]) - self.assertEqual(3, msg.repeated_int32[MessageTest.TestIndex(-1)]) - - def testRepeatedIndexingWithNegative1Int(self, message_module): - msg = message_module.TestAllTypes() - msg.repeated_int32.extend([1, 2, 3]) - self.assertEqual(3, msg.repeated_int32[-1]) - - def testPickleRepeatedScalarContainer(self, message_module): - # Pickle repeated scalar container is not supported. - m = message_module.TestAllTypes() - with self.assertRaises(pickle.PickleError) as _: - pickle.dumps(m.repeated_int32, pickle.HIGHEST_PROTOCOL) - - def testSortEmptyRepeatedCompositeContainer(self, message_module): - """Exercise a scenario that has led to segfaults in the past.""" - m = message_module.TestAllTypes() - m.repeated_nested_message.sort() - - def testHasFieldOnRepeatedField(self, message_module): - """Using HasField on a repeated field should raise an exception.""" - m = message_module.TestAllTypes() - with self.assertRaises(ValueError) as _: - m.HasField('repeated_int32') - - def testRepeatedScalarFieldPop(self, message_module): - m = message_module.TestAllTypes() - with self.assertRaises(IndexError) as _: - m.repeated_int32.pop() - m.repeated_int32.extend(range(5)) - self.assertEqual(4, m.repeated_int32.pop()) - self.assertEqual(0, m.repeated_int32.pop(0)) - self.assertEqual(2, m.repeated_int32.pop(1)) - self.assertEqual([1, 3], m.repeated_int32) - - def testRepeatedCompositeFieldPop(self, message_module): - m = message_module.TestAllTypes() - with self.assertRaises(IndexError) as _: - m.repeated_nested_message.pop() - with self.assertRaises(TypeError) as _: - m.repeated_nested_message.pop('0') - for i in range(5): - n = m.repeated_nested_message.add() - n.bb = i - self.assertEqual(4, m.repeated_nested_message.pop().bb) - self.assertEqual(0, m.repeated_nested_message.pop(0).bb) - self.assertEqual(2, m.repeated_nested_message.pop(1).bb) - self.assertEqual([1, 3], [n.bb for n in m.repeated_nested_message]) - - def testRepeatedCompareWithSelf(self, message_module): - m = message_module.TestAllTypes() - for i in range(5): - m.repeated_int32.insert(i, i) - n = m.repeated_nested_message.add() - n.bb = i - self.assertSequenceEqual(m.repeated_int32, m.repeated_int32) - self.assertEqual(m.repeated_nested_message, m.repeated_nested_message) - - def testReleasedNestedMessages(self, message_module): - """A case that lead to a segfault when a message detached from its parent - - container has itself a child container. - """ - m = message_module.NestedTestAllTypes() - m = m.repeated_child.add() - m = m.child - m = m.repeated_child.add() - self.assertEqual(m.payload.optional_int32, 0) - - def testSetRepeatedComposite(self, message_module): - m = message_module.TestAllTypes() - with self.assertRaises(AttributeError): - m.repeated_int32 = [] - m.repeated_int32.append(1) - with self.assertRaises(AttributeError): - m.repeated_int32 = [] - - def testReturningType(self, message_module): - m = message_module.TestAllTypes() - self.assertEqual(float, type(m.optional_float)) - self.assertEqual(float, type(m.optional_double)) - self.assertEqual(bool, type(m.optional_bool)) - m.optional_float = 1 - m.optional_double = 1 - m.optional_bool = 1 - m.repeated_float.append(1) - m.repeated_double.append(1) - m.repeated_bool.append(1) - m.ParseFromString(m.SerializeToString()) - self.assertEqual(float, type(m.optional_float)) - self.assertEqual(float, type(m.optional_double)) - self.assertEqual('1.0', str(m.optional_double)) - self.assertEqual(bool, type(m.optional_bool)) - self.assertEqual(float, type(m.repeated_float[0])) - self.assertEqual(float, type(m.repeated_double[0])) - self.assertEqual(bool, type(m.repeated_bool[0])) - self.assertEqual(True, m.repeated_bool[0]) - - -# Class to test proto2-only features (required, extensions, etc.) -@testing_refleaks.TestCase -class Proto2Test(unittest.TestCase): - - def testFieldPresence(self): - message = unittest_pb2.TestAllTypes() - - self.assertFalse(message.HasField('optional_int32')) - self.assertFalse(message.HasField('optional_bool')) - self.assertFalse(message.HasField('optional_nested_message')) - - with self.assertRaises(ValueError): - message.HasField('field_doesnt_exist') - - with self.assertRaises(ValueError): - message.HasField('repeated_int32') - with self.assertRaises(ValueError): - message.HasField('repeated_nested_message') - - self.assertEqual(0, message.optional_int32) - self.assertEqual(False, message.optional_bool) - self.assertEqual(0, message.optional_nested_message.bb) - - # Fields are set even when setting the values to default values. - message.optional_int32 = 0 - message.optional_bool = False - message.optional_nested_message.bb = 0 - self.assertTrue(message.HasField('optional_int32')) - self.assertTrue(message.HasField('optional_bool')) - self.assertTrue(message.HasField('optional_nested_message')) - - # Set the fields to non-default values. - message.optional_int32 = 5 - message.optional_bool = True - message.optional_nested_message.bb = 15 - - self.assertTrue(message.HasField(u'optional_int32')) - self.assertTrue(message.HasField('optional_bool')) - self.assertTrue(message.HasField('optional_nested_message')) - - # Clearing the fields unsets them and resets their value to default. - message.ClearField('optional_int32') - message.ClearField(u'optional_bool') - message.ClearField('optional_nested_message') - - self.assertFalse(message.HasField('optional_int32')) - self.assertFalse(message.HasField('optional_bool')) - self.assertFalse(message.HasField('optional_nested_message')) - self.assertEqual(0, message.optional_int32) - self.assertEqual(False, message.optional_bool) - self.assertEqual(0, message.optional_nested_message.bb) - - def testAssignInvalidEnum(self): - """Assigning an invalid enum number is not allowed in proto2.""" - m = unittest_pb2.TestAllTypes() - - # Proto2 can not assign unknown enum. - with self.assertRaises(ValueError) as _: - m.optional_nested_enum = 1234567 - self.assertRaises(ValueError, m.repeated_nested_enum.append, 1234567) - # Assignment is a different code path than append for the C++ impl. - m.repeated_nested_enum.append(2) - m.repeated_nested_enum[0] = 2 - with self.assertRaises(ValueError): - m.repeated_nested_enum[0] = 123456 - - # Unknown enum value can be parsed but is ignored. - m2 = unittest_proto3_arena_pb2.TestAllTypes() - m2.optional_nested_enum = 1234567 - m2.repeated_nested_enum.append(7654321) - serialized = m2.SerializeToString() - - m3 = unittest_pb2.TestAllTypes() - m3.ParseFromString(serialized) - self.assertFalse(m3.HasField('optional_nested_enum')) - # 1 is the default value for optional_nested_enum. - self.assertEqual(1, m3.optional_nested_enum) - self.assertEqual(0, len(m3.repeated_nested_enum)) - m2.Clear() - m2.ParseFromString(m3.SerializeToString()) - self.assertEqual(1234567, m2.optional_nested_enum) - self.assertEqual(7654321, m2.repeated_nested_enum[0]) - - def testUnknownEnumMap(self): - m = map_proto2_unittest_pb2.TestEnumMap() - m.known_map_field[123] = 0 - with self.assertRaises(ValueError): - m.unknown_map_field[1] = 123 - - def testExtensionsErrors(self): - msg = unittest_pb2.TestAllTypes() - self.assertRaises(AttributeError, getattr, msg, 'Extensions') - - def testMergeFromExtensions(self): - msg1 = more_extensions_pb2.TopLevelMessage() - msg2 = more_extensions_pb2.TopLevelMessage() - # Cpp extension will lazily create a sub message which is immutable. - self.assertEqual( - 0, - msg1.submessage.Extensions[more_extensions_pb2.optional_int_extension]) - self.assertFalse(msg1.HasField('submessage')) - msg2.submessage.Extensions[more_extensions_pb2.optional_int_extension] = 123 - # Make sure cmessage and extensions pointing to a mutable message - # after merge instead of the lazily created message. - msg1.MergeFrom(msg2) - self.assertEqual( - 123, - msg1.submessage.Extensions[more_extensions_pb2.optional_int_extension]) - - def testGoldenExtensions(self): - golden_data = test_util.GoldenFileData('golden_message') - golden_message = unittest_pb2.TestAllExtensions() - golden_message.ParseFromString(golden_data) - all_set = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(all_set) - self.assertEqual(all_set, golden_message) - self.assertEqual(golden_data, golden_message.SerializeToString()) - golden_copy = copy.deepcopy(golden_message) - self.assertEqual(golden_data, golden_copy.SerializeToString()) - - def testGoldenPackedExtensions(self): - golden_data = test_util.GoldenFileData('golden_packed_fields_message') - golden_message = unittest_pb2.TestPackedExtensions() - golden_message.ParseFromString(golden_data) - all_set = unittest_pb2.TestPackedExtensions() - test_util.SetAllPackedExtensions(all_set) - self.assertEqual(all_set, golden_message) - self.assertEqual(golden_data, all_set.SerializeToString()) - golden_copy = copy.deepcopy(golden_message) - self.assertEqual(golden_data, golden_copy.SerializeToString()) - - def testPickleIncompleteProto(self): - golden_message = unittest_pb2.TestRequired(a=1) - pickled_message = pickle.dumps(golden_message) - - unpickled_message = pickle.loads(pickled_message) - self.assertEqual(unpickled_message, golden_message) - self.assertEqual(unpickled_message.a, 1) - # This is still an incomplete proto - so serializing should fail - self.assertRaises(message.EncodeError, unpickled_message.SerializeToString) - - # TODO(haberman): this isn't really a proto2-specific test except that this - # message has a required field in it. Should probably be factored out so - # that we can test the other parts with proto3. - def testParsingMerge(self): - """Check the merge behavior when a required or optional field appears - - multiple times in the input. - """ - messages = [ - unittest_pb2.TestAllTypes(), - unittest_pb2.TestAllTypes(), - unittest_pb2.TestAllTypes() - ] - messages[0].optional_int32 = 1 - messages[1].optional_int64 = 2 - messages[2].optional_int32 = 3 - messages[2].optional_string = 'hello' - - merged_message = unittest_pb2.TestAllTypes() - merged_message.optional_int32 = 3 - merged_message.optional_int64 = 2 - merged_message.optional_string = 'hello' - - generator = unittest_pb2.TestParsingMerge.RepeatedFieldsGenerator() - generator.field1.extend(messages) - generator.field2.extend(messages) - generator.field3.extend(messages) - generator.ext1.extend(messages) - generator.ext2.extend(messages) - generator.group1.add().field1.MergeFrom(messages[0]) - generator.group1.add().field1.MergeFrom(messages[1]) - generator.group1.add().field1.MergeFrom(messages[2]) - generator.group2.add().field1.MergeFrom(messages[0]) - generator.group2.add().field1.MergeFrom(messages[1]) - generator.group2.add().field1.MergeFrom(messages[2]) - - data = generator.SerializeToString() - parsing_merge = unittest_pb2.TestParsingMerge() - parsing_merge.ParseFromString(data) - - # Required and optional fields should be merged. - self.assertEqual(parsing_merge.required_all_types, merged_message) - self.assertEqual(parsing_merge.optional_all_types, merged_message) - self.assertEqual(parsing_merge.optionalgroup.optional_group_all_types, - merged_message) - self.assertEqual( - parsing_merge.Extensions[unittest_pb2.TestParsingMerge.optional_ext], - merged_message) - - # Repeated fields should not be merged. - self.assertEqual(len(parsing_merge.repeated_all_types), 3) - self.assertEqual(len(parsing_merge.repeatedgroup), 3) - self.assertEqual( - len(parsing_merge.Extensions[ - unittest_pb2.TestParsingMerge.repeated_ext]), 3) - - def testPythonicInit(self): - message = unittest_pb2.TestAllTypes( - optional_int32=100, - optional_fixed32=200, - optional_float=300.5, - optional_bytes=b'x', - optionalgroup={'a': 400}, - optional_nested_message={'bb': 500}, - optional_foreign_message={}, - optional_nested_enum='BAZ', - repeatedgroup=[{ - 'a': 600 - }, { - 'a': 700 - }], - repeated_nested_enum=['FOO', unittest_pb2.TestAllTypes.BAR], - default_int32=800, - oneof_string='y') - self.assertIsInstance(message, unittest_pb2.TestAllTypes) - self.assertEqual(100, message.optional_int32) - self.assertEqual(200, message.optional_fixed32) - self.assertEqual(300.5, message.optional_float) - self.assertEqual(b'x', message.optional_bytes) - self.assertEqual(400, message.optionalgroup.a) - self.assertIsInstance(message.optional_nested_message, - unittest_pb2.TestAllTypes.NestedMessage) - self.assertEqual(500, message.optional_nested_message.bb) - self.assertTrue(message.HasField('optional_foreign_message')) - self.assertEqual(message.optional_foreign_message, - unittest_pb2.ForeignMessage()) - self.assertEqual(unittest_pb2.TestAllTypes.BAZ, - message.optional_nested_enum) - self.assertEqual(2, len(message.repeatedgroup)) - self.assertEqual(600, message.repeatedgroup[0].a) - self.assertEqual(700, message.repeatedgroup[1].a) - self.assertEqual(2, len(message.repeated_nested_enum)) - self.assertEqual(unittest_pb2.TestAllTypes.FOO, - message.repeated_nested_enum[0]) - self.assertEqual(unittest_pb2.TestAllTypes.BAR, - message.repeated_nested_enum[1]) - self.assertEqual(800, message.default_int32) - self.assertEqual('y', message.oneof_string) - self.assertFalse(message.HasField('optional_int64')) - self.assertEqual(0, len(message.repeated_float)) - self.assertEqual(42, message.default_int64) - - message = unittest_pb2.TestAllTypes(optional_nested_enum=u'BAZ') - self.assertEqual(unittest_pb2.TestAllTypes.BAZ, - message.optional_nested_enum) - - with self.assertRaises(ValueError): - unittest_pb2.TestAllTypes( - optional_nested_message={'INVALID_NESTED_FIELD': 17}) - - with self.assertRaises(TypeError): - unittest_pb2.TestAllTypes( - optional_nested_message={'bb': 'INVALID_VALUE_TYPE'}) - - with self.assertRaises(ValueError): - unittest_pb2.TestAllTypes(optional_nested_enum='INVALID_LABEL') - - with self.assertRaises(ValueError): - unittest_pb2.TestAllTypes(repeated_nested_enum='FOO') - - def testPythonicInitWithDict(self): - # Both string/unicode field name keys should work. - kwargs = { - 'optional_int32': 100, - u'optional_fixed32': 200, - } - msg = unittest_pb2.TestAllTypes(**kwargs) - self.assertEqual(100, msg.optional_int32) - self.assertEqual(200, msg.optional_fixed32) - - - def test_documentation(self): - # Also used by the interactive help() function. - doc = pydoc.html.document(unittest_pb2.TestAllTypes, 'message') - self.assertIn('class TestAllTypes', doc) - self.assertIn('SerializePartialToString', doc) - self.assertIn('repeated_float', doc) - base = unittest_pb2.TestAllTypes.__bases__[0] - self.assertRaises(AttributeError, getattr, base, '_extensions_by_name') - - -# Class to test proto3-only features/behavior (updated field presence & enums) -@testing_refleaks.TestCase -class Proto3Test(unittest.TestCase): - - # Utility method for comparing equality with a map. - def assertMapIterEquals(self, map_iter, dict_value): - # Avoid mutating caller's copy. - dict_value = dict(dict_value) - - for k, v in map_iter: - self.assertEqual(v, dict_value[k]) - del dict_value[k] - - self.assertEqual({}, dict_value) - - def testFieldPresence(self): - message = unittest_proto3_arena_pb2.TestAllTypes() - - # We can't test presence of non-repeated, non-submessage fields. - with self.assertRaises(ValueError): - message.HasField('optional_int32') - with self.assertRaises(ValueError): - message.HasField('optional_float') - with self.assertRaises(ValueError): - message.HasField('optional_string') - with self.assertRaises(ValueError): - message.HasField('optional_bool') - - # But we can still test presence of submessage fields. - self.assertFalse(message.HasField('optional_nested_message')) - - # As with proto2, we can't test presence of fields that don't exist, or - # repeated fields. - with self.assertRaises(ValueError): - message.HasField('field_doesnt_exist') - - with self.assertRaises(ValueError): - message.HasField('repeated_int32') - with self.assertRaises(ValueError): - message.HasField('repeated_nested_message') - - # Fields should default to their type-specific default. - self.assertEqual(0, message.optional_int32) - self.assertEqual(0, message.optional_float) - self.assertEqual('', message.optional_string) - self.assertEqual(False, message.optional_bool) - self.assertEqual(0, message.optional_nested_message.bb) - - # Setting a submessage should still return proper presence information. - message.optional_nested_message.bb = 0 - self.assertTrue(message.HasField('optional_nested_message')) - - # Set the fields to non-default values. - message.optional_int32 = 5 - message.optional_float = 1.1 - message.optional_string = 'abc' - message.optional_bool = True - message.optional_nested_message.bb = 15 - - # Clearing the fields unsets them and resets their value to default. - message.ClearField('optional_int32') - message.ClearField('optional_float') - message.ClearField('optional_string') - message.ClearField('optional_bool') - message.ClearField('optional_nested_message') - - self.assertEqual(0, message.optional_int32) - self.assertEqual(0, message.optional_float) - self.assertEqual('', message.optional_string) - self.assertEqual(False, message.optional_bool) - self.assertEqual(0, message.optional_nested_message.bb) - - def testProto3ParserDropDefaultScalar(self): - message_proto2 = unittest_pb2.TestAllTypes() - message_proto2.optional_int32 = 0 - message_proto2.optional_string = '' - message_proto2.optional_bytes = b'' - self.assertEqual(len(message_proto2.ListFields()), 3) - - message_proto3 = unittest_proto3_arena_pb2.TestAllTypes() - message_proto3.ParseFromString(message_proto2.SerializeToString()) - self.assertEqual(len(message_proto3.ListFields()), 0) - - def testProto3Optional(self): - msg = test_proto3_optional_pb2.TestProto3Optional() - self.assertFalse(msg.HasField('optional_int32')) - self.assertFalse(msg.HasField('optional_float')) - self.assertFalse(msg.HasField('optional_string')) - self.assertFalse(msg.HasField('optional_nested_message')) - self.assertFalse(msg.optional_nested_message.HasField('bb')) - - # Set fields. - msg.optional_int32 = 1 - msg.optional_float = 1.0 - msg.optional_string = '123' - msg.optional_nested_message.bb = 1 - self.assertTrue(msg.HasField('optional_int32')) - self.assertTrue(msg.HasField('optional_float')) - self.assertTrue(msg.HasField('optional_string')) - self.assertTrue(msg.HasField('optional_nested_message')) - self.assertTrue(msg.optional_nested_message.HasField('bb')) - # Set to default value does not clear the fields - msg.optional_int32 = 0 - msg.optional_float = 0.0 - msg.optional_string = '' - msg.optional_nested_message.bb = 0 - self.assertTrue(msg.HasField('optional_int32')) - self.assertTrue(msg.HasField('optional_float')) - self.assertTrue(msg.HasField('optional_string')) - self.assertTrue(msg.HasField('optional_nested_message')) - self.assertTrue(msg.optional_nested_message.HasField('bb')) - - # Test serialize - msg2 = test_proto3_optional_pb2.TestProto3Optional() - msg2.ParseFromString(msg.SerializeToString()) - self.assertTrue(msg2.HasField('optional_int32')) - self.assertTrue(msg2.HasField('optional_float')) - self.assertTrue(msg2.HasField('optional_string')) - self.assertTrue(msg2.HasField('optional_nested_message')) - self.assertTrue(msg2.optional_nested_message.HasField('bb')) - - self.assertEqual(msg.WhichOneof('_optional_int32'), 'optional_int32') - - # Clear these fields. - msg.ClearField('optional_int32') - msg.ClearField('optional_float') - msg.ClearField('optional_string') - msg.ClearField('optional_nested_message') - self.assertFalse(msg.HasField('optional_int32')) - self.assertFalse(msg.HasField('optional_float')) - self.assertFalse(msg.HasField('optional_string')) - self.assertFalse(msg.HasField('optional_nested_message')) - self.assertFalse(msg.optional_nested_message.HasField('bb')) - - self.assertEqual(msg.WhichOneof('_optional_int32'), None) - - # Test has presence: - for field in test_proto3_optional_pb2.TestProto3Optional.DESCRIPTOR.fields: - self.assertTrue(field.has_presence) - for field in unittest_pb2.TestAllTypes.DESCRIPTOR.fields: - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - self.assertFalse(field.has_presence) - else: - self.assertTrue(field.has_presence) - proto3_descriptor = unittest_proto3_arena_pb2.TestAllTypes.DESCRIPTOR - repeated_field = proto3_descriptor.fields_by_name['repeated_int32'] - self.assertFalse(repeated_field.has_presence) - singular_field = proto3_descriptor.fields_by_name['optional_int32'] - self.assertFalse(singular_field.has_presence) - optional_field = proto3_descriptor.fields_by_name['proto3_optional_int32'] - self.assertTrue(optional_field.has_presence) - message_field = proto3_descriptor.fields_by_name['optional_nested_message'] - self.assertTrue(message_field.has_presence) - oneof_field = proto3_descriptor.fields_by_name['oneof_uint32'] - self.assertTrue(oneof_field.has_presence) - - def testAssignUnknownEnum(self): - """Assigning an unknown enum value is allowed and preserves the value.""" - m = unittest_proto3_arena_pb2.TestAllTypes() - - # Proto3 can assign unknown enums. - m.optional_nested_enum = 1234567 - self.assertEqual(1234567, m.optional_nested_enum) - m.repeated_nested_enum.append(22334455) - self.assertEqual(22334455, m.repeated_nested_enum[0]) - # Assignment is a different code path than append for the C++ impl. - m.repeated_nested_enum[0] = 7654321 - self.assertEqual(7654321, m.repeated_nested_enum[0]) - serialized = m.SerializeToString() - - m2 = unittest_proto3_arena_pb2.TestAllTypes() - m2.ParseFromString(serialized) - self.assertEqual(1234567, m2.optional_nested_enum) - self.assertEqual(7654321, m2.repeated_nested_enum[0]) - - # Map isn't really a proto3-only feature. But there is no proto2 equivalent - # of google/protobuf/map_unittest.proto right now, so it's not easy to - # test both with the same test like we do for the other proto2/proto3 tests. - # (google/protobuf/map_proto2_unittest.proto is very different in the set - # of messages and fields it contains). - def testScalarMapDefaults(self): - msg = map_unittest_pb2.TestMap() - - # Scalars start out unset. - self.assertFalse(-123 in msg.map_int32_int32) - self.assertFalse(-2**33 in msg.map_int64_int64) - self.assertFalse(123 in msg.map_uint32_uint32) - self.assertFalse(2**33 in msg.map_uint64_uint64) - self.assertFalse(123 in msg.map_int32_double) - self.assertFalse(False in msg.map_bool_bool) - self.assertFalse('abc' in msg.map_string_string) - self.assertFalse(111 in msg.map_int32_bytes) - self.assertFalse(888 in msg.map_int32_enum) - - # Accessing an unset key returns the default. - self.assertEqual(0, msg.map_int32_int32[-123]) - self.assertEqual(0, msg.map_int64_int64[-2**33]) - self.assertEqual(0, msg.map_uint32_uint32[123]) - self.assertEqual(0, msg.map_uint64_uint64[2**33]) - self.assertEqual(0.0, msg.map_int32_double[123]) - self.assertTrue(isinstance(msg.map_int32_double[123], float)) - self.assertEqual(False, msg.map_bool_bool[False]) - self.assertTrue(isinstance(msg.map_bool_bool[False], bool)) - self.assertEqual('', msg.map_string_string['abc']) - self.assertEqual(b'', msg.map_int32_bytes[111]) - self.assertEqual(0, msg.map_int32_enum[888]) - - # It also sets the value in the map - self.assertTrue(-123 in msg.map_int32_int32) - self.assertTrue(-2**33 in msg.map_int64_int64) - self.assertTrue(123 in msg.map_uint32_uint32) - self.assertTrue(2**33 in msg.map_uint64_uint64) - self.assertTrue(123 in msg.map_int32_double) - self.assertTrue(False in msg.map_bool_bool) - self.assertTrue('abc' in msg.map_string_string) - self.assertTrue(111 in msg.map_int32_bytes) - self.assertTrue(888 in msg.map_int32_enum) - - self.assertIsInstance(msg.map_string_string['abc'], str) - - # Accessing an unset key still throws TypeError if the type of the key - # is incorrect. - with self.assertRaises(TypeError): - msg.map_string_string[123] - - with self.assertRaises(TypeError): - 123 in msg.map_string_string - - def testMapGet(self): - # Need to test that get() properly returns the default, even though the dict - # has defaultdict-like semantics. - msg = map_unittest_pb2.TestMap() - - self.assertIsNone(msg.map_int32_int32.get(5)) - self.assertEqual(10, msg.map_int32_int32.get(5, 10)) - self.assertEqual(10, msg.map_int32_int32.get(key=5, default=10)) - self.assertIsNone(msg.map_int32_int32.get(5)) - - msg.map_int32_int32[5] = 15 - self.assertEqual(15, msg.map_int32_int32.get(5)) - self.assertEqual(15, msg.map_int32_int32.get(5)) - with self.assertRaises(TypeError): - msg.map_int32_int32.get('') - - self.assertIsNone(msg.map_int32_foreign_message.get(5)) - self.assertEqual(10, msg.map_int32_foreign_message.get(5, 10)) - self.assertEqual(10, msg.map_int32_foreign_message.get(key=5, default=10)) - - submsg = msg.map_int32_foreign_message[5] - self.assertIs(submsg, msg.map_int32_foreign_message.get(5)) - with self.assertRaises(TypeError): - msg.map_int32_foreign_message.get('') - - def testScalarMap(self): - msg = map_unittest_pb2.TestMap() - - self.assertEqual(0, len(msg.map_int32_int32)) - self.assertFalse(5 in msg.map_int32_int32) - - msg.map_int32_int32[-123] = -456 - msg.map_int64_int64[-2**33] = -2**34 - msg.map_uint32_uint32[123] = 456 - msg.map_uint64_uint64[2**33] = 2**34 - msg.map_int32_float[2] = 1.2 - msg.map_int32_double[1] = 3.3 - msg.map_string_string['abc'] = '123' - msg.map_bool_bool[True] = True - msg.map_int32_enum[888] = 2 - # Unknown numeric enum is supported in proto3. - msg.map_int32_enum[123] = 456 - - self.assertEqual([], msg.FindInitializationErrors()) - - self.assertEqual(1, len(msg.map_string_string)) - - # Bad key. - with self.assertRaises(TypeError): - msg.map_string_string[123] = '123' - - # Verify that trying to assign a bad key doesn't actually add a member to - # the map. - self.assertEqual(1, len(msg.map_string_string)) - - # Bad value. - with self.assertRaises(TypeError): - msg.map_string_string['123'] = 123 - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMap() - msg2.ParseFromString(serialized) - - # Bad key. - with self.assertRaises(TypeError): - msg2.map_string_string[123] = '123' - - # Bad value. - with self.assertRaises(TypeError): - msg2.map_string_string['123'] = 123 - - self.assertEqual(-456, msg2.map_int32_int32[-123]) - self.assertEqual(-2**34, msg2.map_int64_int64[-2**33]) - self.assertEqual(456, msg2.map_uint32_uint32[123]) - self.assertEqual(2**34, msg2.map_uint64_uint64[2**33]) - self.assertAlmostEqual(1.2, msg.map_int32_float[2]) - self.assertEqual(3.3, msg.map_int32_double[1]) - self.assertEqual('123', msg2.map_string_string['abc']) - self.assertEqual(True, msg2.map_bool_bool[True]) - self.assertEqual(2, msg2.map_int32_enum[888]) - self.assertEqual(456, msg2.map_int32_enum[123]) - self.assertEqual('{-123: -456}', str(msg2.map_int32_int32)) - - def testMapEntryAlwaysSerialized(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_int32[0] = 0 - msg.map_string_string[''] = '' - self.assertEqual(msg.ByteSize(), 12) - self.assertEqual(b'\n\x04\x08\x00\x10\x00r\x04\n\x00\x12\x00', - msg.SerializeToString()) - - def testStringUnicodeConversionInMap(self): - msg = map_unittest_pb2.TestMap() - - unicode_obj = u'\u1234' - bytes_obj = unicode_obj.encode('utf8') - - msg.map_string_string[bytes_obj] = bytes_obj - - (key, value) = list(msg.map_string_string.items())[0] - - self.assertEqual(key, unicode_obj) - self.assertEqual(value, unicode_obj) - - self.assertIsInstance(key, str) - self.assertIsInstance(value, str) - - def testMessageMap(self): - msg = map_unittest_pb2.TestMap() - - self.assertEqual(0, len(msg.map_int32_foreign_message)) - self.assertFalse(5 in msg.map_int32_foreign_message) - - msg.map_int32_foreign_message[123] - # get_or_create() is an alias for getitem. - msg.map_int32_foreign_message.get_or_create(-456) - - self.assertEqual(2, len(msg.map_int32_foreign_message)) - self.assertIn(123, msg.map_int32_foreign_message) - self.assertIn(-456, msg.map_int32_foreign_message) - self.assertEqual(2, len(msg.map_int32_foreign_message)) - - # Bad key. - with self.assertRaises(TypeError): - msg.map_int32_foreign_message['123'] - - # Can't assign directly to submessage. - with self.assertRaises(ValueError): - msg.map_int32_foreign_message[999] = msg.map_int32_foreign_message[123] - - # Verify that trying to assign a bad key doesn't actually add a member to - # the map. - self.assertEqual(2, len(msg.map_int32_foreign_message)) - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMap() - msg2.ParseFromString(serialized) - - self.assertEqual(2, len(msg2.map_int32_foreign_message)) - self.assertIn(123, msg2.map_int32_foreign_message) - self.assertIn(-456, msg2.map_int32_foreign_message) - self.assertEqual(2, len(msg2.map_int32_foreign_message)) - msg2.map_int32_foreign_message[123].c = 1 - # TODO(jieluo): Fix text format for message map. - self.assertIn( - str(msg2.map_int32_foreign_message), - ('{-456: , 123: c: 1\n}', '{123: c: 1\n, -456: }')) - - def testNestedMessageMapItemDelete(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_all_types[1].optional_nested_message.bb = 1 - del msg.map_int32_all_types[1] - msg.map_int32_all_types[2].optional_nested_message.bb = 2 - self.assertEqual(1, len(msg.map_int32_all_types)) - msg.map_int32_all_types[1].optional_nested_message.bb = 1 - self.assertEqual(2, len(msg.map_int32_all_types)) - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMap() - msg2.ParseFromString(serialized) - keys = [1, 2] - # The loop triggers PyErr_Occurred() in c extension. - for key in keys: - del msg2.map_int32_all_types[key] - - def testMapByteSize(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_int32[1] = 1 - size = msg.ByteSize() - msg.map_int32_int32[1] = 128 - self.assertEqual(msg.ByteSize(), size + 1) - - msg.map_int32_foreign_message[19].c = 1 - size = msg.ByteSize() - msg.map_int32_foreign_message[19].c = 128 - self.assertEqual(msg.ByteSize(), size + 1) - - def testMergeFrom(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_int32[12] = 34 - msg.map_int32_int32[56] = 78 - msg.map_int64_int64[22] = 33 - msg.map_int32_foreign_message[111].c = 5 - msg.map_int32_foreign_message[222].c = 10 - - msg2 = map_unittest_pb2.TestMap() - msg2.map_int32_int32[12] = 55 - msg2.map_int64_int64[88] = 99 - msg2.map_int32_foreign_message[222].c = 15 - msg2.map_int32_foreign_message[222].d = 20 - old_map_value = msg2.map_int32_foreign_message[222] - - msg2.MergeFrom(msg) - # Compare with expected message instead of call - # msg2.map_int32_foreign_message[222] to make sure MergeFrom does not - # sync with repeated field and there is no duplicated keys. - expected_msg = map_unittest_pb2.TestMap() - expected_msg.CopyFrom(msg) - expected_msg.map_int64_int64[88] = 99 - self.assertEqual(msg2, expected_msg) - - self.assertEqual(34, msg2.map_int32_int32[12]) - self.assertEqual(78, msg2.map_int32_int32[56]) - self.assertEqual(33, msg2.map_int64_int64[22]) - self.assertEqual(99, msg2.map_int64_int64[88]) - self.assertEqual(5, msg2.map_int32_foreign_message[111].c) - self.assertEqual(10, msg2.map_int32_foreign_message[222].c) - self.assertFalse(msg2.map_int32_foreign_message[222].HasField('d')) - if api_implementation.Type() != 'cpp': - # During the call to MergeFrom(), the C++ implementation will have - # deallocated the underlying message, but this is very difficult to detect - # properly. The line below is likely to cause a segmentation fault. - # With the Python implementation, old_map_value is just 'detached' from - # the main message. Using it will not crash of course, but since it still - # have a reference to the parent message I'm sure we can find interesting - # ways to cause inconsistencies. - self.assertEqual(15, old_map_value.c) - - # Verify that there is only one entry per key, even though the MergeFrom - # may have internally created multiple entries for a single key in the - # list representation. - as_dict = {} - for key in msg2.map_int32_foreign_message: - self.assertFalse(key in as_dict) - as_dict[key] = msg2.map_int32_foreign_message[key].c - - self.assertEqual({111: 5, 222: 10}, as_dict) - - # Special case: test that delete of item really removes the item, even if - # there might have physically been duplicate keys due to the previous merge. - # This is only a special case for the C++ implementation which stores the - # map as an array. - del msg2.map_int32_int32[12] - self.assertFalse(12 in msg2.map_int32_int32) - - del msg2.map_int32_foreign_message[222] - self.assertFalse(222 in msg2.map_int32_foreign_message) - with self.assertRaises(TypeError): - del msg2.map_int32_foreign_message[''] - - def testMapMergeFrom(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_int32[12] = 34 - msg.map_int32_int32[56] = 78 - msg.map_int64_int64[22] = 33 - msg.map_int32_foreign_message[111].c = 5 - msg.map_int32_foreign_message[222].c = 10 - - msg2 = map_unittest_pb2.TestMap() - msg2.map_int32_int32[12] = 55 - msg2.map_int64_int64[88] = 99 - msg2.map_int32_foreign_message[222].c = 15 - msg2.map_int32_foreign_message[222].d = 20 - - msg2.map_int32_int32.MergeFrom(msg.map_int32_int32) - self.assertEqual(34, msg2.map_int32_int32[12]) - self.assertEqual(78, msg2.map_int32_int32[56]) - - msg2.map_int64_int64.MergeFrom(msg.map_int64_int64) - self.assertEqual(33, msg2.map_int64_int64[22]) - self.assertEqual(99, msg2.map_int64_int64[88]) - - msg2.map_int32_foreign_message.MergeFrom(msg.map_int32_foreign_message) - # Compare with expected message instead of call - # msg.map_int32_foreign_message[222] to make sure MergeFrom does not - # sync with repeated field and no duplicated keys. - expected_msg = map_unittest_pb2.TestMap() - expected_msg.CopyFrom(msg) - expected_msg.map_int64_int64[88] = 99 - self.assertEqual(msg2, expected_msg) - - # Test when cpp extension cache a map. - m1 = map_unittest_pb2.TestMap() - m2 = map_unittest_pb2.TestMap() - self.assertEqual(m1.map_int32_foreign_message, m1.map_int32_foreign_message) - m2.map_int32_foreign_message[123].c = 10 - m1.MergeFrom(m2) - self.assertEqual(10, m2.map_int32_foreign_message[123].c) - - # Test merge maps within different message types. - m1 = map_unittest_pb2.TestMap() - m2 = map_unittest_pb2.TestMessageMap() - m2.map_int32_message[123].optional_int32 = 10 - m1.map_int32_all_types.MergeFrom(m2.map_int32_message) - self.assertEqual(10, m1.map_int32_all_types[123].optional_int32) - - # Test overwrite message value map - msg = map_unittest_pb2.TestMap() - msg.map_int32_foreign_message[222].c = 123 - msg2 = map_unittest_pb2.TestMap() - msg2.map_int32_foreign_message[222].d = 20 - msg.MergeFromString(msg2.SerializeToString()) - self.assertEqual(msg.map_int32_foreign_message[222].d, 20) - self.assertNotEqual(msg.map_int32_foreign_message[222].c, 123) - - # Merge a dict to map field is not accepted - with self.assertRaises(AttributeError): - m1.map_int32_all_types.MergeFrom( - {1: unittest_proto3_arena_pb2.TestAllTypes()}) - - def testMergeFromBadType(self): - msg = map_unittest_pb2.TestMap() - with self.assertRaisesRegex( - TypeError, - r'Parameter to MergeFrom\(\) must be instance of same class: expected ' - r'.+TestMap got int\.'): - msg.MergeFrom(1) - - def testCopyFromBadType(self): - msg = map_unittest_pb2.TestMap() - with self.assertRaisesRegex( - TypeError, - r'Parameter to [A-Za-z]*From\(\) must be instance of same class: ' - r'expected .+TestMap got int\.'): - msg.CopyFrom(1) - - def testIntegerMapWithLongs(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_int32[int(-123)] = int(-456) - msg.map_int64_int64[int(-2**33)] = int(-2**34) - msg.map_uint32_uint32[int(123)] = int(456) - msg.map_uint64_uint64[int(2**33)] = int(2**34) - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMap() - msg2.ParseFromString(serialized) - - self.assertEqual(-456, msg2.map_int32_int32[-123]) - self.assertEqual(-2**34, msg2.map_int64_int64[-2**33]) - self.assertEqual(456, msg2.map_uint32_uint32[123]) - self.assertEqual(2**34, msg2.map_uint64_uint64[2**33]) - - def testMapAssignmentCausesPresence(self): - msg = map_unittest_pb2.TestMapSubmessage() - msg.test_map.map_int32_int32[123] = 456 - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMapSubmessage() - msg2.ParseFromString(serialized) - - self.assertEqual(msg, msg2) - - # Now test that various mutations of the map properly invalidate the - # cached size of the submessage. - msg.test_map.map_int32_int32[888] = 999 - serialized = msg.SerializeToString() - msg2.ParseFromString(serialized) - self.assertEqual(msg, msg2) - - msg.test_map.map_int32_int32.clear() - serialized = msg.SerializeToString() - msg2.ParseFromString(serialized) - self.assertEqual(msg, msg2) - - def testMapAssignmentCausesPresenceForSubmessages(self): - msg = map_unittest_pb2.TestMapSubmessage() - msg.test_map.map_int32_foreign_message[123].c = 5 - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMapSubmessage() - msg2.ParseFromString(serialized) - - self.assertEqual(msg, msg2) - - # Now test that various mutations of the map properly invalidate the - # cached size of the submessage. - msg.test_map.map_int32_foreign_message[888].c = 7 - serialized = msg.SerializeToString() - msg2.ParseFromString(serialized) - self.assertEqual(msg, msg2) - - msg.test_map.map_int32_foreign_message[888].MergeFrom( - msg.test_map.map_int32_foreign_message[123]) - serialized = msg.SerializeToString() - msg2.ParseFromString(serialized) - self.assertEqual(msg, msg2) - - msg.test_map.map_int32_foreign_message.clear() - serialized = msg.SerializeToString() - msg2.ParseFromString(serialized) - self.assertEqual(msg, msg2) - - def testModifyMapWhileIterating(self): - msg = map_unittest_pb2.TestMap() - - string_string_iter = iter(msg.map_string_string) - int32_foreign_iter = iter(msg.map_int32_foreign_message) - - msg.map_string_string['abc'] = '123' - msg.map_int32_foreign_message[5].c = 5 - - with self.assertRaises(RuntimeError): - for key in string_string_iter: - pass - - with self.assertRaises(RuntimeError): - for key in int32_foreign_iter: - pass - - def testModifyMapEntryWhileIterating(self): - msg = map_unittest_pb2.TestMap() - - msg.map_string_string['abc'] = '123' - msg.map_string_string['def'] = '456' - msg.map_string_string['ghi'] = '789' - - msg.map_int32_foreign_message[5].c = 5 - msg.map_int32_foreign_message[6].c = 6 - msg.map_int32_foreign_message[7].c = 7 - - string_string_keys = list(msg.map_string_string.keys()) - int32_foreign_keys = list(msg.map_int32_foreign_message.keys()) - - keys = [] - for key in msg.map_string_string: - keys.append(key) - msg.map_string_string[key] = '000' - self.assertEqual(keys, string_string_keys) - self.assertEqual(keys, list(msg.map_string_string.keys())) - - keys = [] - for key in msg.map_int32_foreign_message: - keys.append(key) - msg.map_int32_foreign_message[key].c = 0 - self.assertEqual(keys, int32_foreign_keys) - self.assertEqual(keys, list(msg.map_int32_foreign_message.keys())) - - def testSubmessageMap(self): - msg = map_unittest_pb2.TestMap() - - submsg = msg.map_int32_foreign_message[111] - self.assertIs(submsg, msg.map_int32_foreign_message[111]) - self.assertIsInstance(submsg, unittest_pb2.ForeignMessage) - - submsg.c = 5 - - serialized = msg.SerializeToString() - msg2 = map_unittest_pb2.TestMap() - msg2.ParseFromString(serialized) - - self.assertEqual(5, msg2.map_int32_foreign_message[111].c) - - # Doesn't allow direct submessage assignment. - with self.assertRaises(ValueError): - msg.map_int32_foreign_message[88] = unittest_pb2.ForeignMessage() - - def testMapIteration(self): - msg = map_unittest_pb2.TestMap() - - for k, v in msg.map_int32_int32.items(): - # Should not be reached. - self.assertTrue(False) - - msg.map_int32_int32[2] = 4 - msg.map_int32_int32[3] = 6 - msg.map_int32_int32[4] = 8 - self.assertEqual(3, len(msg.map_int32_int32)) - - matching_dict = {2: 4, 3: 6, 4: 8} - self.assertMapIterEquals(msg.map_int32_int32.items(), matching_dict) - - def testMapItems(self): - # Map items used to have strange behaviors when use c extension. Because - # [] may reorder the map and invalidate any existing iterators. - # TODO(jieluo): Check if [] reordering the map is a bug or intended - # behavior. - msg = map_unittest_pb2.TestMap() - msg.map_string_string['local_init_op'] = '' - msg.map_string_string['trainable_variables'] = '' - msg.map_string_string['variables'] = '' - msg.map_string_string['init_op'] = '' - msg.map_string_string['summaries'] = '' - items1 = msg.map_string_string.items() - items2 = msg.map_string_string.items() - self.assertEqual(items1, items2) - - def testMapDeterministicSerialization(self): - golden_data = (b'r\x0c\n\x07init_op\x12\x01d' - b'r\n\n\x05item1\x12\x01e' - b'r\n\n\x05item2\x12\x01f' - b'r\n\n\x05item3\x12\x01g' - b'r\x0b\n\x05item4\x12\x02QQ' - b'r\x12\n\rlocal_init_op\x12\x01a' - b'r\x0e\n\tsummaries\x12\x01e' - b'r\x18\n\x13trainable_variables\x12\x01b' - b'r\x0e\n\tvariables\x12\x01c') - msg = map_unittest_pb2.TestMap() - msg.map_string_string['local_init_op'] = 'a' - msg.map_string_string['trainable_variables'] = 'b' - msg.map_string_string['variables'] = 'c' - msg.map_string_string['init_op'] = 'd' - msg.map_string_string['summaries'] = 'e' - msg.map_string_string['item1'] = 'e' - msg.map_string_string['item2'] = 'f' - msg.map_string_string['item3'] = 'g' - msg.map_string_string['item4'] = 'QQ' - - # If deterministic serialization is not working correctly, this will be - # "flaky" depending on the exact python dict hash seed. - # - # Fortunately, there are enough items in this map that it is extremely - # unlikely to ever hit the "right" in-order combination, so the test - # itself should fail reliably. - self.assertEqual(golden_data, msg.SerializeToString(deterministic=True)) - - def testMapIterationClearMessage(self): - # Iterator needs to work even if message and map are deleted. - msg = map_unittest_pb2.TestMap() - - msg.map_int32_int32[2] = 4 - msg.map_int32_int32[3] = 6 - msg.map_int32_int32[4] = 8 - - it = msg.map_int32_int32.items() - del msg - - matching_dict = {2: 4, 3: 6, 4: 8} - self.assertMapIterEquals(it, matching_dict) - - def testMapConstruction(self): - msg = map_unittest_pb2.TestMap(map_int32_int32={1: 2, 3: 4}) - self.assertEqual(2, msg.map_int32_int32[1]) - self.assertEqual(4, msg.map_int32_int32[3]) - - msg = map_unittest_pb2.TestMap( - map_int32_foreign_message={3: unittest_pb2.ForeignMessage(c=5)}) - self.assertEqual(5, msg.map_int32_foreign_message[3].c) - - def testMapScalarFieldConstruction(self): - msg1 = map_unittest_pb2.TestMap() - msg1.map_int32_int32[1] = 42 - msg2 = map_unittest_pb2.TestMap(map_int32_int32=msg1.map_int32_int32) - self.assertEqual(42, msg2.map_int32_int32[1]) - - def testMapMessageFieldConstruction(self): - msg1 = map_unittest_pb2.TestMap() - msg1.map_string_foreign_message['test'].c = 42 - msg2 = map_unittest_pb2.TestMap( - map_string_foreign_message=msg1.map_string_foreign_message) - self.assertEqual(42, msg2.map_string_foreign_message['test'].c) - - def testMapFieldRaisesCorrectError(self): - # Should raise a TypeError when given a non-iterable. - with self.assertRaises(TypeError): - map_unittest_pb2.TestMap(map_string_foreign_message=1) - - def testMapValidAfterFieldCleared(self): - # Map needs to work even if field is cleared. - # For the C++ implementation this tests the correctness of - # MapContainer::Release() - msg = map_unittest_pb2.TestMap() - int32_map = msg.map_int32_int32 - - int32_map[2] = 4 - int32_map[3] = 6 - int32_map[4] = 8 - - msg.ClearField('map_int32_int32') - self.assertEqual(b'', msg.SerializeToString()) - matching_dict = {2: 4, 3: 6, 4: 8} - self.assertMapIterEquals(int32_map.items(), matching_dict) - - def testMessageMapValidAfterFieldCleared(self): - # Map needs to work even if field is cleared. - # For the C++ implementation this tests the correctness of - # MapContainer::Release() - msg = map_unittest_pb2.TestMap() - int32_foreign_message = msg.map_int32_foreign_message - - int32_foreign_message[2].c = 5 - - msg.ClearField('map_int32_foreign_message') - self.assertEqual(b'', msg.SerializeToString()) - self.assertTrue(2 in int32_foreign_message.keys()) - - def testMessageMapItemValidAfterTopMessageCleared(self): - # Message map item needs to work even if it is cleared. - # For the C++ implementation this tests the correctness of - # MapContainer::Release() - msg = map_unittest_pb2.TestMap() - msg.map_int32_all_types[2].optional_string = 'bar' - - if api_implementation.Type() == 'cpp': - # Need to keep the map reference because of b/27942626. - # TODO(jieluo): Remove it. - unused_map = msg.map_int32_all_types # pylint: disable=unused-variable - msg_value = msg.map_int32_all_types[2] - msg.Clear() - - # Reset to trigger sync between repeated field and map in c++. - msg.map_int32_all_types[3].optional_string = 'foo' - self.assertEqual(msg_value.optional_string, 'bar') - - def testMapIterInvalidatedByClearField(self): - # Map iterator is invalidated when field is cleared. - # But this case does need to not crash the interpreter. - # For the C++ implementation this tests the correctness of - # ScalarMapContainer::Release() - msg = map_unittest_pb2.TestMap() - - it = iter(msg.map_int32_int32) - - msg.ClearField('map_int32_int32') - with self.assertRaises(RuntimeError): - for _ in it: - pass - - it = iter(msg.map_int32_foreign_message) - msg.ClearField('map_int32_foreign_message') - with self.assertRaises(RuntimeError): - for _ in it: - pass - - def testMapDelete(self): - msg = map_unittest_pb2.TestMap() - - self.assertEqual(0, len(msg.map_int32_int32)) - - msg.map_int32_int32[4] = 6 - self.assertEqual(1, len(msg.map_int32_int32)) - - with self.assertRaises(KeyError): - del msg.map_int32_int32[88] - - del msg.map_int32_int32[4] - self.assertEqual(0, len(msg.map_int32_int32)) - - with self.assertRaises(KeyError): - del msg.map_int32_all_types[32] - - def testMapsAreMapping(self): - msg = map_unittest_pb2.TestMap() - self.assertIsInstance(msg.map_int32_int32, collections.abc.Mapping) - self.assertIsInstance(msg.map_int32_int32, collections.abc.MutableMapping) - self.assertIsInstance(msg.map_int32_foreign_message, - collections.abc.Mapping) - self.assertIsInstance(msg.map_int32_foreign_message, - collections.abc.MutableMapping) - - def testMapsCompare(self): - msg = map_unittest_pb2.TestMap() - msg.map_int32_int32[-123] = -456 - self.assertEqual(msg.map_int32_int32, msg.map_int32_int32) - self.assertEqual(msg.map_int32_foreign_message, - msg.map_int32_foreign_message) - self.assertNotEqual(msg.map_int32_int32, 0) - - def testMapFindInitializationErrorsSmokeTest(self): - msg = map_unittest_pb2.TestMap() - msg.map_string_string['abc'] = '123' - msg.map_int32_int32[35] = 64 - msg.map_string_foreign_message['foo'].c = 5 - self.assertEqual(0, len(msg.FindInitializationErrors())) - - @unittest.skipIf(sys.maxunicode == UCS2_MAXUNICODE, 'Skip for ucs2') - def testStrictUtf8Check(self): - # Test u'\ud801' is rejected at parser in both python2 and python3. - serialized = (b'r\x03\xed\xa0\x81') - msg = unittest_proto3_arena_pb2.TestAllTypes() - with self.assertRaises(Exception) as context: - msg.MergeFromString(serialized) - if api_implementation.Type() == 'python': - self.assertIn('optional_string', str(context.exception)) - else: - self.assertIn('Error parsing message', str(context.exception)) - - # Test optional_string=u'😍' is accepted. - serialized = unittest_proto3_arena_pb2.TestAllTypes( - optional_string=u'😍').SerializeToString() - msg2 = unittest_proto3_arena_pb2.TestAllTypes() - msg2.MergeFromString(serialized) - self.assertEqual(msg2.optional_string, u'😍') - - msg = unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud001') - self.assertEqual(msg.optional_string, u'\ud001') - - def testSurrogatesInPython3(self): - # Surrogates are rejected at setters in Python3. - with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801\udc01') - with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes(optional_string=b'\xed\xa0\x81') - with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801') - with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801\ud801') - - - - -@testing_refleaks.TestCase -class ValidTypeNamesTest(unittest.TestCase): - - def assertImportFromName(self, msg, base_name): - # Parse to extra 'some.name' as a string. - tp_name = str(type(msg)).split("'")[1] - valid_names = ('Repeated%sContainer' % base_name, - 'Repeated%sFieldContainer' % base_name) - self.assertTrue( - any(tp_name.endswith(v) for v in valid_names), - '%r does end with any of %r' % (tp_name, valid_names)) - - parts = tp_name.split('.') - class_name = parts[-1] - module_name = '.'.join(parts[:-1]) - __import__(module_name, fromlist=[class_name]) - - def testTypeNamesCanBeImported(self): - # If import doesn't work, pickling won't work either. - pb = unittest_pb2.TestAllTypes() - self.assertImportFromName(pb.repeated_int32, 'Scalar') - self.assertImportFromName(pb.repeated_nested_message, 'Composite') - - -@testing_refleaks.TestCase -class PackedFieldTest(unittest.TestCase): - - def setMessage(self, message): - message.repeated_int32.append(1) - message.repeated_int64.append(1) - message.repeated_uint32.append(1) - message.repeated_uint64.append(1) - message.repeated_sint32.append(1) - message.repeated_sint64.append(1) - message.repeated_fixed32.append(1) - message.repeated_fixed64.append(1) - message.repeated_sfixed32.append(1) - message.repeated_sfixed64.append(1) - message.repeated_float.append(1.0) - message.repeated_double.append(1.0) - message.repeated_bool.append(True) - message.repeated_nested_enum.append(1) - - def testPackedFields(self): - message = packed_field_test_pb2.TestPackedTypes() - self.setMessage(message) - golden_data = (b'\x0A\x01\x01' - b'\x12\x01\x01' - b'\x1A\x01\x01' - b'\x22\x01\x01' - b'\x2A\x01\x02' - b'\x32\x01\x02' - b'\x3A\x04\x01\x00\x00\x00' - b'\x42\x08\x01\x00\x00\x00\x00\x00\x00\x00' - b'\x4A\x04\x01\x00\x00\x00' - b'\x52\x08\x01\x00\x00\x00\x00\x00\x00\x00' - b'\x5A\x04\x00\x00\x80\x3f' - b'\x62\x08\x00\x00\x00\x00\x00\x00\xf0\x3f' - b'\x6A\x01\x01' - b'\x72\x01\x01') - self.assertEqual(golden_data, message.SerializeToString()) - - def testUnpackedFields(self): - message = packed_field_test_pb2.TestUnpackedTypes() - self.setMessage(message) - golden_data = (b'\x08\x01' - b'\x10\x01' - b'\x18\x01' - b'\x20\x01' - b'\x28\x02' - b'\x30\x02' - b'\x3D\x01\x00\x00\x00' - b'\x41\x01\x00\x00\x00\x00\x00\x00\x00' - b'\x4D\x01\x00\x00\x00' - b'\x51\x01\x00\x00\x00\x00\x00\x00\x00' - b'\x5D\x00\x00\x80\x3f' - b'\x61\x00\x00\x00\x00\x00\x00\xf0\x3f' - b'\x68\x01' - b'\x70\x01') - self.assertEqual(golden_data, message.SerializeToString()) - - -@unittest.skipIf(api_implementation.Type() == 'python', - 'explicit tests of the C++ implementation') -@testing_refleaks.TestCase -class OversizeProtosTest(unittest.TestCase): - - def GenerateNestedProto(self, n): - msg = unittest_pb2.TestRecursiveMessage() - sub = msg - for _ in range(n): - sub = sub.a - sub.i = 0 - return msg.SerializeToString() - - def testSucceedOkSizedProto(self): - msg = unittest_pb2.TestRecursiveMessage() - msg.ParseFromString(self.GenerateNestedProto(100)) - - def testAssertOversizeProto(self): - api_implementation._c_module.SetAllowOversizeProtos(False) - msg = unittest_pb2.TestRecursiveMessage() - with self.assertRaises(message.DecodeError) as context: - msg.ParseFromString(self.GenerateNestedProto(101)) - self.assertIn('Error parsing message', str(context.exception)) - - def testSucceedOversizeProto(self): - api_implementation._c_module.SetAllowOversizeProtos(True) - msg = unittest_pb2.TestRecursiveMessage() - msg.ParseFromString(self.GenerateNestedProto(101)) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/proto_builder_test.py b/ext/protobuf/Python/google/protobuf/internal/proto_builder_test.py deleted file mode 100644 index 48077b0a4..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/proto_builder_test.py +++ /dev/null @@ -1,106 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.proto_builder.""" - -import collections -import unittest - -from google.protobuf import descriptor_pb2 # pylint: disable=g-import-not-at-top -from google.protobuf import descriptor -from google.protobuf import descriptor_pool -from google.protobuf import proto_builder -from google.protobuf import text_format - - -class ProtoBuilderTest(unittest.TestCase): - - def setUp(self): - self.ordered_fields = collections.OrderedDict([ - ('foo', descriptor_pb2.FieldDescriptorProto.TYPE_INT64), - ('bar', descriptor_pb2.FieldDescriptorProto.TYPE_STRING), - ]) - self._fields = dict(self.ordered_fields) - - def testMakeSimpleProtoClass(self): - """Test that we can create a proto class.""" - proto_cls = proto_builder.MakeSimpleProtoClass( - self._fields, - full_name='net.proto2.python.public.proto_builder_test.Test') - proto = proto_cls() - proto.foo = 12345 - proto.bar = 'asdf' - self.assertMultiLineEqual( - 'bar: "asdf"\nfoo: 12345\n', text_format.MessageToString(proto)) - - def testOrderedFields(self): - """Test that the field order is maintained when given an OrderedDict.""" - proto_cls = proto_builder.MakeSimpleProtoClass( - self.ordered_fields, - full_name='net.proto2.python.public.proto_builder_test.OrderedTest') - proto = proto_cls() - proto.foo = 12345 - proto.bar = 'asdf' - self.assertMultiLineEqual( - 'foo: 12345\nbar: "asdf"\n', text_format.MessageToString(proto)) - - def testMakeSameProtoClassTwice(self): - """Test that the DescriptorPool is used.""" - pool = descriptor_pool.DescriptorPool() - proto_cls1 = proto_builder.MakeSimpleProtoClass( - self._fields, - full_name='net.proto2.python.public.proto_builder_test.Test', - pool=pool) - proto_cls2 = proto_builder.MakeSimpleProtoClass( - self._fields, - full_name='net.proto2.python.public.proto_builder_test.Test', - pool=pool) - self.assertIs(proto_cls1.DESCRIPTOR, proto_cls2.DESCRIPTOR) - - def testMakeLargeProtoClass(self): - """Test that large created protos don't use reserved field numbers.""" - num_fields = 123456 - fields = { - 'foo%d' % i: descriptor_pb2.FieldDescriptorProto.TYPE_INT64 - for i in range(num_fields) - } - proto_cls = proto_builder.MakeSimpleProtoClass( - fields, - full_name='net.proto2.python.public.proto_builder_test.LargeProtoTest') - - reserved_field_numbers = set( - range(descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER, - descriptor.FieldDescriptor.LAST_RESERVED_FIELD_NUMBER + 1)) - proto_field_numbers = set(proto_cls.DESCRIPTOR.fields_by_number) - self.assertFalse(reserved_field_numbers.intersection(proto_field_numbers)) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/python_message.py b/ext/protobuf/Python/google/protobuf/internal/python_message.py index 5550b425c..bf9acefd2 100644 --- a/ext/protobuf/Python/google/protobuf/internal/python_message.py +++ b/ext/protobuf/Python/google/protobuf/internal/python_message.py @@ -283,20 +283,8 @@ def _IsMessageMapField(field): def _AttachFieldHelpers(cls, field_descriptor): is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED) - is_packable = (is_repeated and - wire_format.IsTypePackable(field_descriptor.type)) - is_proto3 = field_descriptor.containing_type.syntax == 'proto3' - if not is_packable: - is_packed = False - elif field_descriptor.containing_type.syntax == 'proto2': - is_packed = (field_descriptor.has_options and - field_descriptor.GetOptions().packed) - else: - has_packed_false = (field_descriptor.has_options and - field_descriptor.GetOptions().HasField('packed') and - field_descriptor.GetOptions().packed == False) - is_packed = not has_packed_false is_map_entry = _IsMapField(field_descriptor) + is_packed = field_descriptor.is_packed if is_map_entry: field_encoder = encoder.MapEncoder(field_descriptor) @@ -320,16 +308,12 @@ def AddDecoder(wiretype, is_packed): tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype) decode_type = field_descriptor.type if (decode_type == _FieldDescriptor.TYPE_ENUM and - type_checkers.SupportsOpenEnums(field_descriptor)): + not field_descriptor.enum_type.is_closed): decode_type = _FieldDescriptor.TYPE_INT32 oneof_descriptor = None - clear_if_default = False if field_descriptor.containing_oneof is not None: oneof_descriptor = field_descriptor - elif (is_proto3 and not is_repeated and - field_descriptor.cpp_type != _FieldDescriptor.CPPTYPE_MESSAGE): - clear_if_default = True if is_map_entry: is_message_map = _IsMessageMapField(field_descriptor) @@ -341,7 +325,7 @@ def AddDecoder(wiretype, is_packed): field_decoder = decoder.StringDecoder( field_descriptor.number, is_repeated, is_packed, field_descriptor, field_descriptor._default_constructor, - clear_if_default) + not field_descriptor.has_presence) elif field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: field_decoder = type_checkers.TYPE_TO_DECODER[decode_type]( field_descriptor.number, is_repeated, is_packed, @@ -351,7 +335,7 @@ def AddDecoder(wiretype, is_packed): field_descriptor.number, is_repeated, is_packed, # pylint: disable=protected-access field_descriptor, field_descriptor._default_constructor, - clear_if_default) + not field_descriptor.has_presence) cls._decoders_by_tag[tag_bytes] = (field_decoder, oneof_descriptor) @@ -683,7 +667,6 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls): property_name = _PropertyName(proto_field_name) type_checker = type_checkers.GetTypeChecker(field) default_value = field.default_value - is_proto3 = field.containing_type.syntax == 'proto3' def getter(self): # TODO(protobuf-team): This may be broken since there may not be @@ -692,8 +675,6 @@ def getter(self): getter.__module__ = None getter.__doc__ = 'Getter for %s.' % proto_field_name - clear_when_set_to_default = is_proto3 and not field.containing_oneof - def field_setter(self, new_value): # pylint: disable=protected-access # Testing the value for truthiness captures all of the proto3 defaults @@ -703,7 +684,7 @@ def field_setter(self, new_value): except TypeError as e: raise TypeError( 'Cannot set %s to %.1024r: %s' % (field.full_name, new_value, e)) - if clear_when_set_to_default and not new_value: + if not field.has_presence and not new_value: self._fields.pop(field, None) else: self._fields[field] = new_value @@ -788,12 +769,12 @@ def _AddPropertiesForExtensions(descriptor, cls): def _AddStaticMethods(cls): # TODO(robinson): This probably needs to be thread-safe(?) - def RegisterExtension(extension_handle): - extension_handle.containing_type = cls.DESCRIPTOR + def RegisterExtension(field_descriptor): + field_descriptor.containing_type = cls.DESCRIPTOR # TODO(amauryfa): Use cls.MESSAGE_FACTORY.pool when available. # pylint: disable=protected-access - cls.DESCRIPTOR.file.pool._AddExtensionDescriptor(extension_handle) - _AttachFieldHelpers(cls, extension_handle) + cls.DESCRIPTOR.file.pool._AddExtensionDescriptor(field_descriptor) + _AttachFieldHelpers(cls, field_descriptor) cls.RegisterExtension = staticmethod(RegisterExtension) def FromString(s): @@ -825,24 +806,16 @@ def ListFields(self): cls.ListFields = ListFields -_PROTO3_ERROR_TEMPLATE = \ - ('Protocol message %s has no non-repeated submessage field "%s" ' - 'nor marked as optional') -_PROTO2_ERROR_TEMPLATE = 'Protocol message %s has no non-repeated field "%s"' def _AddHasFieldMethod(message_descriptor, cls): """Helper for _AddMessageMethods().""" - is_proto3 = (message_descriptor.syntax == "proto3") - error_msg = _PROTO3_ERROR_TEMPLATE if is_proto3 else _PROTO2_ERROR_TEMPLATE - hassable_fields = {} for field in message_descriptor.fields: if field.label == _FieldDescriptor.LABEL_REPEATED: continue # For proto3, only submessages and fields inside a oneof have presence. - if (is_proto3 and field.cpp_type != _FieldDescriptor.CPPTYPE_MESSAGE and - not field.containing_oneof): + if not field.has_presence: continue hassable_fields[field.name] = field @@ -853,8 +826,10 @@ def _AddHasFieldMethod(message_descriptor, cls): def HasField(self, field_name): try: field = hassable_fields[field_name] - except KeyError: - raise ValueError(error_msg % (message_descriptor.full_name, field_name)) + except KeyError as exc: + raise ValueError('Protocol message %s has no non-repeated field "%s" ' + 'nor has presence is not available for this field.' % ( + message_descriptor.full_name, field_name)) from exc if isinstance(field, descriptor_mod.OneofDescriptor): try: @@ -911,28 +886,28 @@ def ClearField(self, field_name): def _AddClearExtensionMethod(cls): """Helper for _AddMessageMethods().""" - def ClearExtension(self, extension_handle): - extension_dict._VerifyExtensionHandle(self, extension_handle) + def ClearExtension(self, field_descriptor): + extension_dict._VerifyExtensionHandle(self, field_descriptor) # Similar to ClearField(), above. - if extension_handle in self._fields: - del self._fields[extension_handle] + if field_descriptor in self._fields: + del self._fields[field_descriptor] self._Modified() cls.ClearExtension = ClearExtension def _AddHasExtensionMethod(cls): """Helper for _AddMessageMethods().""" - def HasExtension(self, extension_handle): - extension_dict._VerifyExtensionHandle(self, extension_handle) - if extension_handle.label == _FieldDescriptor.LABEL_REPEATED: - raise KeyError('"%s" is repeated.' % extension_handle.full_name) + def HasExtension(self, field_descriptor): + extension_dict._VerifyExtensionHandle(self, field_descriptor) + if field_descriptor.label == _FieldDescriptor.LABEL_REPEATED: + raise KeyError('"%s" is repeated.' % field_descriptor.full_name) - if extension_handle.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - value = self._fields.get(extension_handle) + if field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + value = self._fields.get(field_descriptor) return value is not None and value._is_present_in_parent else: - return extension_handle in self._fields + return field_descriptor in self._fields cls.HasExtension = HasExtension def _InternalUnpackAny(msg): diff --git a/ext/protobuf/Python/google/protobuf/internal/reflection_test.py b/ext/protobuf/Python/google/protobuf/internal/reflection_test.py deleted file mode 100644 index 62957d3fd..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/reflection_test.py +++ /dev/null @@ -1,3381 +0,0 @@ -# -*- coding: utf-8 -*- -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unittest for reflection.py, which also indirectly tests the output of the -pure-Python protocol compiler. -""" - -import copy -import gc -import operator -import struct -import sys -import warnings -import unittest - -from google.protobuf import unittest_import_pb2 -from google.protobuf import unittest_mset_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import unittest_proto3_arena_pb2 -from google.protobuf import descriptor_pb2 -from google.protobuf import descriptor -from google.protobuf import message -from google.protobuf import reflection -from google.protobuf import text_format -from google.protobuf.internal import api_implementation -from google.protobuf.internal import more_extensions_pb2 -from google.protobuf.internal import more_messages_pb2 -from google.protobuf.internal import message_set_extensions_pb2 -from google.protobuf.internal import wire_format -from google.protobuf.internal import test_util -from google.protobuf.internal import testing_refleaks -from google.protobuf.internal import decoder -from google.protobuf.internal import _parameterized - - -warnings.simplefilter('error', DeprecationWarning) - - -class _MiniDecoder(object): - """Decodes a stream of values from a string. - - Once upon a time we actually had a class called decoder.Decoder. Then we - got rid of it during a redesign that made decoding much, much faster overall. - But a couple tests in this file used it to check that the serialized form of - a message was correct. So, this class implements just the methods that were - used by said tests, so that we don't have to rewrite the tests. - """ - - def __init__(self, bytes): - self._bytes = bytes - self._pos = 0 - - def ReadVarint(self): - result, self._pos = decoder._DecodeVarint(self._bytes, self._pos) - return result - - ReadInt32 = ReadVarint - ReadInt64 = ReadVarint - ReadUInt32 = ReadVarint - ReadUInt64 = ReadVarint - - def ReadSInt64(self): - return wire_format.ZigZagDecode(self.ReadVarint()) - - ReadSInt32 = ReadSInt64 - - def ReadFieldNumberAndWireType(self): - return wire_format.UnpackTag(self.ReadVarint()) - - def ReadFloat(self): - result = struct.unpack('= (3, 10)): - self.assertRaises(TypeError, setattr, proto, 'optional_bool', 1.1) - else: - proto.optional_bool = 1.1 - - def assertIntegerTypes(self, integer_fn, message_module): - """Verifies setting of scalar integers. - - Args: - integer_fn: A function to wrap the integers that will be assigned. - message_module: unittest_pb2 or unittest_proto3_arena_pb2 - """ - def TestGetAndDeserialize(field_name, value, expected_type): - proto = message_module.TestAllTypes() - value = integer_fn(value) - setattr(proto, field_name, value) - self.assertIsInstance(getattr(proto, field_name), expected_type) - proto2 = message_module.TestAllTypes() - proto2.ParseFromString(proto.SerializeToString()) - self.assertIsInstance(getattr(proto2, field_name), expected_type) - - TestGetAndDeserialize('optional_int32', 1, int) - TestGetAndDeserialize('optional_int32', 1 << 30, int) - TestGetAndDeserialize('optional_uint32', 1 << 30, int) - integer_64 = int - if struct.calcsize('L') == 4: - # Python only has signed ints, so 32-bit python can't fit an uint32 - # in an int. - TestGetAndDeserialize('optional_uint32', 1 << 31, integer_64) - else: - # 64-bit python can fit uint32 inside an int - TestGetAndDeserialize('optional_uint32', 1 << 31, int) - TestGetAndDeserialize('optional_int64', 1 << 30, integer_64) - TestGetAndDeserialize('optional_int64', 1 << 60, integer_64) - TestGetAndDeserialize('optional_uint64', 1 << 30, integer_64) - TestGetAndDeserialize('optional_uint64', 1 << 60, integer_64) - - def testIntegerTypes(self, message_module): - self.assertIntegerTypes(lambda x: x, message_module) - - def testNonStandardIntegerTypes(self, message_module): - self.assertIntegerTypes(test_util.NonStandardInteger, message_module) - - def testIllegalValuesForIntegers(self, message_module): - pb = message_module.TestAllTypes() - - # Strings are illegal, even when the represent an integer. - with self.assertRaises(TypeError): - pb.optional_uint64 = '2' - - # The exact error should propagate with a poorly written custom integer. - with self.assertRaisesRegex(RuntimeError, 'my_error'): - pb.optional_uint64 = test_util.NonStandardInteger(5, 'my_error') - - def assetIntegerBoundsChecking(self, integer_fn, message_module): - """Verifies bounds checking for scalar integer fields. - - Args: - integer_fn: A function to wrap the integers that will be assigned. - message_module: unittest_pb2 or unittest_proto3_arena_pb2 - """ - def TestMinAndMaxIntegers(field_name, expected_min, expected_max): - pb = message_module.TestAllTypes() - expected_min = integer_fn(expected_min) - expected_max = integer_fn(expected_max) - setattr(pb, field_name, expected_min) - self.assertEqual(expected_min, getattr(pb, field_name)) - setattr(pb, field_name, expected_max) - self.assertEqual(expected_max, getattr(pb, field_name)) - self.assertRaises((ValueError, TypeError), setattr, pb, field_name, - expected_min - 1) - self.assertRaises((ValueError, TypeError), setattr, pb, field_name, - expected_max + 1) - - TestMinAndMaxIntegers('optional_int32', -(1 << 31), (1 << 31) - 1) - TestMinAndMaxIntegers('optional_uint32', 0, 0xffffffff) - TestMinAndMaxIntegers('optional_int64', -(1 << 63), (1 << 63) - 1) - TestMinAndMaxIntegers('optional_uint64', 0, 0xffffffffffffffff) - # A bit of white-box testing since -1 is an int and not a long in C++ and - # so goes down a different path. - pb = message_module.TestAllTypes() - with self.assertRaises((ValueError, TypeError)): - pb.optional_uint64 = integer_fn(-(1 << 63)) - - pb = message_module.TestAllTypes() - pb.optional_nested_enum = integer_fn(1) - self.assertEqual(1, pb.optional_nested_enum) - - def testSingleScalarBoundsChecking(self, message_module): - self.assetIntegerBoundsChecking(lambda x: x, message_module) - - def testNonStandardSingleScalarBoundsChecking(self, message_module): - self.assetIntegerBoundsChecking( - test_util.NonStandardInteger, message_module) - - def testRepeatedScalarTypeSafety(self, message_module): - proto = message_module.TestAllTypes() - self.assertRaises(TypeError, proto.repeated_int32.append, 1.1) - self.assertRaises(TypeError, proto.repeated_int32.append, 'foo') - self.assertRaises(TypeError, proto.repeated_string, 10) - self.assertRaises(TypeError, proto.repeated_bytes, 10) - - proto.repeated_int32.append(10) - proto.repeated_int32[0] = 23 - self.assertRaises(IndexError, proto.repeated_int32.__setitem__, 500, 23) - self.assertRaises(TypeError, proto.repeated_int32.__setitem__, 0, 'abc') - self.assertRaises(TypeError, proto.repeated_int32.__setitem__, 0, []) - self.assertRaises(TypeError, proto.repeated_int32.__setitem__, - 'index', 23) - - proto.repeated_string.append('2') - self.assertRaises(TypeError, proto.repeated_string.__setitem__, 0, 10) - - # Repeated enums tests. - #proto.repeated_nested_enum.append(0) - - def testSingleScalarGettersAndSetters(self, message_module): - proto = message_module.TestAllTypes() - self.assertEqual(0, proto.optional_int32) - proto.optional_int32 = 1 - self.assertEqual(1, proto.optional_int32) - - proto.optional_uint64 = 0xffffffffffff - self.assertEqual(0xffffffffffff, proto.optional_uint64) - proto.optional_uint64 = 0xffffffffffffffff - self.assertEqual(0xffffffffffffffff, proto.optional_uint64) - # TODO(robinson): Test all other scalar field types. - - def testEnums(self, message_module): - proto = message_module.TestAllTypes() - self.assertEqual(1, proto.FOO) - self.assertEqual(1, message_module.TestAllTypes.FOO) - self.assertEqual(2, proto.BAR) - self.assertEqual(2, message_module.TestAllTypes.BAR) - self.assertEqual(3, proto.BAZ) - self.assertEqual(3, message_module.TestAllTypes.BAZ) - - def testEnum_Name(self, message_module): - self.assertEqual( - 'FOREIGN_FOO', - message_module.ForeignEnum.Name(message_module.FOREIGN_FOO)) - self.assertEqual( - 'FOREIGN_BAR', - message_module.ForeignEnum.Name(message_module.FOREIGN_BAR)) - self.assertEqual( - 'FOREIGN_BAZ', - message_module.ForeignEnum.Name(message_module.FOREIGN_BAZ)) - self.assertRaises(ValueError, - message_module.ForeignEnum.Name, 11312) - - proto = message_module.TestAllTypes() - self.assertEqual('FOO', - proto.NestedEnum.Name(proto.FOO)) - self.assertEqual('FOO', - message_module.TestAllTypes.NestedEnum.Name(proto.FOO)) - self.assertEqual('BAR', - proto.NestedEnum.Name(proto.BAR)) - self.assertEqual('BAR', - message_module.TestAllTypes.NestedEnum.Name(proto.BAR)) - self.assertEqual('BAZ', - proto.NestedEnum.Name(proto.BAZ)) - self.assertEqual('BAZ', - message_module.TestAllTypes.NestedEnum.Name(proto.BAZ)) - self.assertRaises(ValueError, - proto.NestedEnum.Name, 11312) - self.assertRaises(ValueError, - message_module.TestAllTypes.NestedEnum.Name, 11312) - - # Check some coercion cases. - self.assertRaises(TypeError, message_module.TestAllTypes.NestedEnum.Name, - 11312.0) - self.assertRaises(TypeError, message_module.TestAllTypes.NestedEnum.Name, - None) - self.assertEqual('FOO', message_module.TestAllTypes.NestedEnum.Name(True)) - - def testEnum_Value(self, message_module): - self.assertEqual(message_module.FOREIGN_FOO, - message_module.ForeignEnum.Value('FOREIGN_FOO')) - self.assertEqual(message_module.FOREIGN_FOO, - message_module.ForeignEnum.FOREIGN_FOO) - - self.assertEqual(message_module.FOREIGN_BAR, - message_module.ForeignEnum.Value('FOREIGN_BAR')) - self.assertEqual(message_module.FOREIGN_BAR, - message_module.ForeignEnum.FOREIGN_BAR) - - self.assertEqual(message_module.FOREIGN_BAZ, - message_module.ForeignEnum.Value('FOREIGN_BAZ')) - self.assertEqual(message_module.FOREIGN_BAZ, - message_module.ForeignEnum.FOREIGN_BAZ) - - self.assertRaises(ValueError, - message_module.ForeignEnum.Value, 'FO') - with self.assertRaises(AttributeError): - message_module.ForeignEnum.FO - - proto = message_module.TestAllTypes() - self.assertEqual(proto.FOO, - proto.NestedEnum.Value('FOO')) - self.assertEqual(proto.FOO, - proto.NestedEnum.FOO) - - self.assertEqual(proto.FOO, - message_module.TestAllTypes.NestedEnum.Value('FOO')) - self.assertEqual(proto.FOO, - message_module.TestAllTypes.NestedEnum.FOO) - - self.assertEqual(proto.BAR, - proto.NestedEnum.Value('BAR')) - self.assertEqual(proto.BAR, - proto.NestedEnum.BAR) - - self.assertEqual(proto.BAR, - message_module.TestAllTypes.NestedEnum.Value('BAR')) - self.assertEqual(proto.BAR, - message_module.TestAllTypes.NestedEnum.BAR) - - self.assertEqual(proto.BAZ, - proto.NestedEnum.Value('BAZ')) - self.assertEqual(proto.BAZ, - proto.NestedEnum.BAZ) - - self.assertEqual(proto.BAZ, - message_module.TestAllTypes.NestedEnum.Value('BAZ')) - self.assertEqual(proto.BAZ, - message_module.TestAllTypes.NestedEnum.BAZ) - - self.assertRaises(ValueError, - proto.NestedEnum.Value, 'Foo') - with self.assertRaises(AttributeError): - proto.NestedEnum.Value.Foo - - self.assertRaises(ValueError, - message_module.TestAllTypes.NestedEnum.Value, 'Foo') - with self.assertRaises(AttributeError): - message_module.TestAllTypes.NestedEnum.Value.Foo - - def testEnum_KeysAndValues(self, message_module): - if message_module == unittest_pb2: - keys = ['FOREIGN_FOO', 'FOREIGN_BAR', 'FOREIGN_BAZ'] - values = [4, 5, 6] - items = [('FOREIGN_FOO', 4), ('FOREIGN_BAR', 5), ('FOREIGN_BAZ', 6)] - else: - keys = ['FOREIGN_ZERO', 'FOREIGN_FOO', 'FOREIGN_BAR', 'FOREIGN_BAZ'] - values = [0, 4, 5, 6] - items = [('FOREIGN_ZERO', 0), ('FOREIGN_FOO', 4), - ('FOREIGN_BAR', 5), ('FOREIGN_BAZ', 6)] - self.assertEqual(keys, - list(message_module.ForeignEnum.keys())) - self.assertEqual(values, - list(message_module.ForeignEnum.values())) - self.assertEqual(items, - list(message_module.ForeignEnum.items())) - - proto = message_module.TestAllTypes() - if message_module == unittest_pb2: - keys = ['FOO', 'BAR', 'BAZ', 'NEG'] - values = [1, 2, 3, -1] - items = [('FOO', 1), ('BAR', 2), ('BAZ', 3), ('NEG', -1)] - else: - keys = ['ZERO', 'FOO', 'BAR', 'BAZ', 'NEG'] - values = [0, 1, 2, 3, -1] - items = [('ZERO', 0), ('FOO', 1), ('BAR', 2), ('BAZ', 3), ('NEG', -1)] - self.assertEqual(keys, list(proto.NestedEnum.keys())) - self.assertEqual(values, list(proto.NestedEnum.values())) - self.assertEqual(items, - list(proto.NestedEnum.items())) - - def testStaticParseFrom(self, message_module): - proto1 = message_module.TestAllTypes() - test_util.SetAllFields(proto1) - - string1 = proto1.SerializeToString() - proto2 = message_module.TestAllTypes.FromString(string1) - - # Messages should be equal. - self.assertEqual(proto2, proto1) - - def testMergeFromSingularField(self, message_module): - # Test merge with just a singular field. - proto1 = message_module.TestAllTypes() - proto1.optional_int32 = 1 - - proto2 = message_module.TestAllTypes() - # This shouldn't get overwritten. - proto2.optional_string = 'value' - - proto2.MergeFrom(proto1) - self.assertEqual(1, proto2.optional_int32) - self.assertEqual('value', proto2.optional_string) - - def testMergeFromRepeatedField(self, message_module): - # Test merge with just a repeated field. - proto1 = message_module.TestAllTypes() - proto1.repeated_int32.append(1) - proto1.repeated_int32.append(2) - - proto2 = message_module.TestAllTypes() - proto2.repeated_int32.append(0) - proto2.MergeFrom(proto1) - - self.assertEqual(0, proto2.repeated_int32[0]) - self.assertEqual(1, proto2.repeated_int32[1]) - self.assertEqual(2, proto2.repeated_int32[2]) - - def testMergeFromRepeatedNestedMessage(self, message_module): - # Test merge with a repeated nested message. - proto1 = message_module.TestAllTypes() - m = proto1.repeated_nested_message.add() - m.bb = 123 - m = proto1.repeated_nested_message.add() - m.bb = 321 - - proto2 = message_module.TestAllTypes() - m = proto2.repeated_nested_message.add() - m.bb = 999 - proto2.MergeFrom(proto1) - self.assertEqual(999, proto2.repeated_nested_message[0].bb) - self.assertEqual(123, proto2.repeated_nested_message[1].bb) - self.assertEqual(321, proto2.repeated_nested_message[2].bb) - - proto3 = message_module.TestAllTypes() - proto3.repeated_nested_message.MergeFrom(proto2.repeated_nested_message) - self.assertEqual(999, proto3.repeated_nested_message[0].bb) - self.assertEqual(123, proto3.repeated_nested_message[1].bb) - self.assertEqual(321, proto3.repeated_nested_message[2].bb) - - def testMergeFromAllFields(self, message_module): - # With all fields set. - proto1 = message_module.TestAllTypes() - test_util.SetAllFields(proto1) - proto2 = message_module.TestAllTypes() - proto2.MergeFrom(proto1) - - # Messages should be equal. - self.assertEqual(proto2, proto1) - - # Serialized string should be equal too. - string1 = proto1.SerializeToString() - string2 = proto2.SerializeToString() - self.assertEqual(string1, string2) - - def testMergeFromBug(self, message_module): - message1 = message_module.TestAllTypes() - message2 = message_module.TestAllTypes() - - # Cause optional_nested_message to be instantiated within message1, even - # though it is not considered to be "present". - message1.optional_nested_message - self.assertFalse(message1.HasField('optional_nested_message')) - - # Merge into message2. This should not instantiate the field is message2. - message2.MergeFrom(message1) - self.assertFalse(message2.HasField('optional_nested_message')) - - def testCopyFromSingularField(self, message_module): - # Test copy with just a singular field. - proto1 = message_module.TestAllTypes() - proto1.optional_int32 = 1 - proto1.optional_string = 'important-text' - - proto2 = message_module.TestAllTypes() - proto2.optional_string = 'value' - - proto2.CopyFrom(proto1) - self.assertEqual(1, proto2.optional_int32) - self.assertEqual('important-text', proto2.optional_string) - - def testCopyFromRepeatedField(self, message_module): - # Test copy with a repeated field. - proto1 = message_module.TestAllTypes() - proto1.repeated_int32.append(1) - proto1.repeated_int32.append(2) - - proto2 = message_module.TestAllTypes() - proto2.repeated_int32.append(0) - proto2.CopyFrom(proto1) - - self.assertEqual(1, proto2.repeated_int32[0]) - self.assertEqual(2, proto2.repeated_int32[1]) - - def testCopyFromAllFields(self, message_module): - # With all fields set. - proto1 = message_module.TestAllTypes() - test_util.SetAllFields(proto1) - proto2 = message_module.TestAllTypes() - proto2.CopyFrom(proto1) - - # Messages should be equal. - self.assertEqual(proto2, proto1) - - # Serialized string should be equal too. - string1 = proto1.SerializeToString() - string2 = proto2.SerializeToString() - self.assertEqual(string1, string2) - - def testCopyFromSelf(self, message_module): - proto1 = message_module.TestAllTypes() - proto1.repeated_int32.append(1) - proto1.optional_int32 = 2 - proto1.optional_string = 'important-text' - - proto1.CopyFrom(proto1) - self.assertEqual(1, proto1.repeated_int32[0]) - self.assertEqual(2, proto1.optional_int32) - self.assertEqual('important-text', proto1.optional_string) - - def testDeepCopy(self, message_module): - proto1 = message_module.TestAllTypes() - proto1.optional_int32 = 1 - proto2 = copy.deepcopy(proto1) - self.assertEqual(1, proto2.optional_int32) - - proto1.repeated_int32.append(2) - proto1.repeated_int32.append(3) - container = copy.deepcopy(proto1.repeated_int32) - self.assertEqual([2, 3], container) - container.remove(container[0]) - self.assertEqual([3], container) - - message1 = proto1.repeated_nested_message.add() - message1.bb = 1 - messages = copy.deepcopy(proto1.repeated_nested_message) - self.assertEqual(proto1.repeated_nested_message, messages) - message1.bb = 2 - self.assertNotEqual(proto1.repeated_nested_message, messages) - messages.remove(messages[0]) - self.assertEqual(len(messages), 0) - - # TODO(anuraag): Implement deepcopy for extension dict - - def testDisconnectingBeforeClear(self, message_module): - proto = message_module.TestAllTypes() - nested = proto.optional_nested_message - proto.Clear() - self.assertIsNot(nested, proto.optional_nested_message) - nested.bb = 23 - self.assertFalse(proto.HasField('optional_nested_message')) - self.assertEqual(0, proto.optional_nested_message.bb) - - proto = message_module.TestAllTypes() - nested = proto.optional_nested_message - nested.bb = 5 - foreign = proto.optional_foreign_message - foreign.c = 6 - proto.Clear() - self.assertIsNot(nested, proto.optional_nested_message) - self.assertIsNot(foreign, proto.optional_foreign_message) - self.assertEqual(5, nested.bb) - self.assertEqual(6, foreign.c) - nested.bb = 15 - foreign.c = 16 - self.assertFalse(proto.HasField('optional_nested_message')) - self.assertEqual(0, proto.optional_nested_message.bb) - self.assertFalse(proto.HasField('optional_foreign_message')) - self.assertEqual(0, proto.optional_foreign_message.c) - - def testStringUTF8Encoding(self, message_module): - proto = message_module.TestAllTypes() - - # Assignment of a unicode object to a field of type 'bytes' is not allowed. - self.assertRaises(TypeError, - setattr, proto, 'optional_bytes', u'unicode object') - - # Check that the default value is of python's 'unicode' type. - self.assertEqual(type(proto.optional_string), str) - - proto.optional_string = str('Testing') - self.assertEqual(proto.optional_string, str('Testing')) - - # Assign a value of type 'str' which can be encoded in UTF-8. - proto.optional_string = str('Testing') - self.assertEqual(proto.optional_string, str('Testing')) - - # Try to assign a 'bytes' object which contains non-UTF-8. - self.assertRaises(ValueError, - setattr, proto, 'optional_string', b'a\x80a') - # No exception: Assign already encoded UTF-8 bytes to a string field. - utf8_bytes = u'Тест'.encode('utf-8') - proto.optional_string = utf8_bytes - # No exception: Assign the a non-ascii unicode object. - proto.optional_string = u'Тест' - # No exception thrown (normal str assignment containing ASCII). - proto.optional_string = 'abc' - - def testBytesInTextFormat(self, message_module): - proto = message_module.TestAllTypes(optional_bytes=b'\x00\x7f\x80\xff') - self.assertEqual(u'optional_bytes: "\\000\\177\\200\\377"\n', str(proto)) - - def testEmptyNestedMessage(self, message_module): - proto = message_module.TestAllTypes() - proto.optional_nested_message.MergeFrom( - message_module.TestAllTypes.NestedMessage()) - self.assertTrue(proto.HasField('optional_nested_message')) - - proto = message_module.TestAllTypes() - proto.optional_nested_message.CopyFrom( - message_module.TestAllTypes.NestedMessage()) - self.assertTrue(proto.HasField('optional_nested_message')) - - proto = message_module.TestAllTypes() - bytes_read = proto.optional_nested_message.MergeFromString(b'') - self.assertEqual(0, bytes_read) - self.assertTrue(proto.HasField('optional_nested_message')) - - proto = message_module.TestAllTypes() - proto.optional_nested_message.ParseFromString(b'') - self.assertTrue(proto.HasField('optional_nested_message')) - - serialized = proto.SerializeToString() - proto2 = message_module.TestAllTypes() - self.assertEqual( - len(serialized), - proto2.MergeFromString(serialized)) - self.assertTrue(proto2.HasField('optional_nested_message')) - - -# Class to test proto2-only features (required, extensions, etc.) -@testing_refleaks.TestCase -class Proto2ReflectionTest(unittest.TestCase): - - def testRepeatedCompositeConstructor(self): - # Constructor with only repeated composite types should succeed. - proto = unittest_pb2.TestAllTypes( - repeated_nested_message=[ - unittest_pb2.TestAllTypes.NestedMessage( - bb=unittest_pb2.TestAllTypes.FOO), - unittest_pb2.TestAllTypes.NestedMessage( - bb=unittest_pb2.TestAllTypes.BAR)], - repeated_foreign_message=[ - unittest_pb2.ForeignMessage(c=-43), - unittest_pb2.ForeignMessage(c=45324), - unittest_pb2.ForeignMessage(c=12)], - repeatedgroup=[ - unittest_pb2.TestAllTypes.RepeatedGroup(), - unittest_pb2.TestAllTypes.RepeatedGroup(a=1), - unittest_pb2.TestAllTypes.RepeatedGroup(a=2)]) - - self.assertEqual( - [unittest_pb2.TestAllTypes.NestedMessage( - bb=unittest_pb2.TestAllTypes.FOO), - unittest_pb2.TestAllTypes.NestedMessage( - bb=unittest_pb2.TestAllTypes.BAR)], - list(proto.repeated_nested_message)) - self.assertEqual( - [unittest_pb2.ForeignMessage(c=-43), - unittest_pb2.ForeignMessage(c=45324), - unittest_pb2.ForeignMessage(c=12)], - list(proto.repeated_foreign_message)) - self.assertEqual( - [unittest_pb2.TestAllTypes.RepeatedGroup(), - unittest_pb2.TestAllTypes.RepeatedGroup(a=1), - unittest_pb2.TestAllTypes.RepeatedGroup(a=2)], - list(proto.repeatedgroup)) - - def assertListsEqual(self, values, others): - self.assertEqual(len(values), len(others)) - for i in range(len(values)): - self.assertEqual(values[i], others[i]) - - def testSimpleHasBits(self): - # Test a scalar. - proto = unittest_pb2.TestAllTypes() - self.assertFalse(proto.HasField('optional_int32')) - self.assertEqual(0, proto.optional_int32) - # HasField() shouldn't be true if all we've done is - # read the default value. - self.assertFalse(proto.HasField('optional_int32')) - proto.optional_int32 = 1 - # Setting a value however *should* set the "has" bit. - self.assertTrue(proto.HasField('optional_int32')) - proto.ClearField('optional_int32') - # And clearing that value should unset the "has" bit. - self.assertFalse(proto.HasField('optional_int32')) - - def testHasBitsWithSinglyNestedScalar(self): - # Helper used to test foreign messages and groups. - # - # composite_field_name should be the name of a non-repeated - # composite (i.e., foreign or group) field in TestAllTypes, - # and scalar_field_name should be the name of an integer-valued - # scalar field within that composite. - # - # I never thought I'd miss C++ macros and templates so much. :( - # This helper is semantically just: - # - # assert proto.composite_field.scalar_field == 0 - # assert not proto.composite_field.HasField('scalar_field') - # assert not proto.HasField('composite_field') - # - # proto.composite_field.scalar_field = 10 - # old_composite_field = proto.composite_field - # - # assert proto.composite_field.scalar_field == 10 - # assert proto.composite_field.HasField('scalar_field') - # assert proto.HasField('composite_field') - # - # proto.ClearField('composite_field') - # - # assert not proto.composite_field.HasField('scalar_field') - # assert not proto.HasField('composite_field') - # assert proto.composite_field.scalar_field == 0 - # - # # Now ensure that ClearField('composite_field') disconnected - # # the old field object from the object tree... - # assert old_composite_field is not proto.composite_field - # old_composite_field.scalar_field = 20 - # assert not proto.composite_field.HasField('scalar_field') - # assert not proto.HasField('composite_field') - def TestCompositeHasBits(composite_field_name, scalar_field_name): - proto = unittest_pb2.TestAllTypes() - # First, check that we can get the scalar value, and see that it's the - # default (0), but that proto.HasField('omposite') and - # proto.composite.HasField('scalar') will still return False. - composite_field = getattr(proto, composite_field_name) - original_scalar_value = getattr(composite_field, scalar_field_name) - self.assertEqual(0, original_scalar_value) - # Assert that the composite object does not "have" the scalar. - self.assertFalse(composite_field.HasField(scalar_field_name)) - # Assert that proto does not "have" the composite field. - self.assertFalse(proto.HasField(composite_field_name)) - - # Now set the scalar within the composite field. Ensure that the setting - # is reflected, and that proto.HasField('composite') and - # proto.composite.HasField('scalar') now both return True. - new_val = 20 - setattr(composite_field, scalar_field_name, new_val) - self.assertEqual(new_val, getattr(composite_field, scalar_field_name)) - # Hold on to a reference to the current composite_field object. - old_composite_field = composite_field - # Assert that the has methods now return true. - self.assertTrue(composite_field.HasField(scalar_field_name)) - self.assertTrue(proto.HasField(composite_field_name)) - - # Now call the clear method... - proto.ClearField(composite_field_name) - - # ...and ensure that the "has" bits are all back to False... - composite_field = getattr(proto, composite_field_name) - self.assertFalse(composite_field.HasField(scalar_field_name)) - self.assertFalse(proto.HasField(composite_field_name)) - # ...and ensure that the scalar field has returned to its default. - self.assertEqual(0, getattr(composite_field, scalar_field_name)) - - self.assertIsNot(old_composite_field, composite_field) - setattr(old_composite_field, scalar_field_name, new_val) - self.assertFalse(composite_field.HasField(scalar_field_name)) - self.assertFalse(proto.HasField(composite_field_name)) - self.assertEqual(0, getattr(composite_field, scalar_field_name)) - - # Test simple, single-level nesting when we set a scalar. - TestCompositeHasBits('optionalgroup', 'a') - TestCompositeHasBits('optional_nested_message', 'bb') - TestCompositeHasBits('optional_foreign_message', 'c') - TestCompositeHasBits('optional_import_message', 'd') - - def testHasBitsWhenModifyingRepeatedFields(self): - # Test nesting when we add an element to a repeated field in a submessage. - proto = unittest_pb2.TestNestedMessageHasBits() - proto.optional_nested_message.nestedmessage_repeated_int32.append(5) - self.assertEqual( - [5], proto.optional_nested_message.nestedmessage_repeated_int32) - self.assertTrue(proto.HasField('optional_nested_message')) - - # Do the same test, but with a repeated composite field within the - # submessage. - proto.ClearField('optional_nested_message') - self.assertFalse(proto.HasField('optional_nested_message')) - proto.optional_nested_message.nestedmessage_repeated_foreignmessage.add() - self.assertTrue(proto.HasField('optional_nested_message')) - - def testHasBitsForManyLevelsOfNesting(self): - # Test nesting many levels deep. - recursive_proto = unittest_pb2.TestMutualRecursionA() - self.assertFalse(recursive_proto.HasField('bb')) - self.assertEqual(0, recursive_proto.bb.a.bb.a.bb.optional_int32) - self.assertFalse(recursive_proto.HasField('bb')) - recursive_proto.bb.a.bb.a.bb.optional_int32 = 5 - self.assertEqual(5, recursive_proto.bb.a.bb.a.bb.optional_int32) - self.assertTrue(recursive_proto.HasField('bb')) - self.assertTrue(recursive_proto.bb.HasField('a')) - self.assertTrue(recursive_proto.bb.a.HasField('bb')) - self.assertTrue(recursive_proto.bb.a.bb.HasField('a')) - self.assertTrue(recursive_proto.bb.a.bb.a.HasField('bb')) - self.assertFalse(recursive_proto.bb.a.bb.a.bb.HasField('a')) - self.assertTrue(recursive_proto.bb.a.bb.a.bb.HasField('optional_int32')) - - def testSingularListExtensions(self): - proto = unittest_pb2.TestAllExtensions() - proto.Extensions[unittest_pb2.optional_fixed32_extension] = 1 - proto.Extensions[unittest_pb2.optional_int32_extension ] = 5 - proto.Extensions[unittest_pb2.optional_string_extension ] = 'foo' - self.assertEqual( - [ (unittest_pb2.optional_int32_extension , 5), - (unittest_pb2.optional_fixed32_extension, 1), - (unittest_pb2.optional_string_extension , 'foo') ], - proto.ListFields()) - del proto.Extensions[unittest_pb2.optional_fixed32_extension] - self.assertEqual( - [(unittest_pb2.optional_int32_extension, 5), - (unittest_pb2.optional_string_extension, 'foo')], - proto.ListFields()) - - def testRepeatedListExtensions(self): - proto = unittest_pb2.TestAllExtensions() - proto.Extensions[unittest_pb2.repeated_fixed32_extension].append(1) - proto.Extensions[unittest_pb2.repeated_int32_extension ].append(5) - proto.Extensions[unittest_pb2.repeated_int32_extension ].append(11) - proto.Extensions[unittest_pb2.repeated_string_extension ].append('foo') - proto.Extensions[unittest_pb2.repeated_string_extension ].append('bar') - proto.Extensions[unittest_pb2.repeated_string_extension ].append('baz') - proto.Extensions[unittest_pb2.optional_int32_extension ] = 21 - self.assertEqual( - [ (unittest_pb2.optional_int32_extension , 21), - (unittest_pb2.repeated_int32_extension , [5, 11]), - (unittest_pb2.repeated_fixed32_extension, [1]), - (unittest_pb2.repeated_string_extension , ['foo', 'bar', 'baz']) ], - proto.ListFields()) - del proto.Extensions[unittest_pb2.repeated_int32_extension] - del proto.Extensions[unittest_pb2.repeated_string_extension] - self.assertEqual( - [(unittest_pb2.optional_int32_extension, 21), - (unittest_pb2.repeated_fixed32_extension, [1])], - proto.ListFields()) - - def testListFieldsAndExtensions(self): - proto = unittest_pb2.TestFieldOrderings() - test_util.SetAllFieldsAndExtensions(proto) - unittest_pb2.my_extension_int - self.assertEqual( - [ (proto.DESCRIPTOR.fields_by_name['my_int' ], 1), - (unittest_pb2.my_extension_int , 23), - (proto.DESCRIPTOR.fields_by_name['my_string'], 'foo'), - (unittest_pb2.my_extension_string , 'bar'), - (proto.DESCRIPTOR.fields_by_name['my_float' ], 1.0) ], - proto.ListFields()) - - def testDefaultValues(self): - proto = unittest_pb2.TestAllTypes() - self.assertEqual(0, proto.optional_int32) - self.assertEqual(0, proto.optional_int64) - self.assertEqual(0, proto.optional_uint32) - self.assertEqual(0, proto.optional_uint64) - self.assertEqual(0, proto.optional_sint32) - self.assertEqual(0, proto.optional_sint64) - self.assertEqual(0, proto.optional_fixed32) - self.assertEqual(0, proto.optional_fixed64) - self.assertEqual(0, proto.optional_sfixed32) - self.assertEqual(0, proto.optional_sfixed64) - self.assertEqual(0.0, proto.optional_float) - self.assertEqual(0.0, proto.optional_double) - self.assertEqual(False, proto.optional_bool) - self.assertEqual('', proto.optional_string) - self.assertEqual(b'', proto.optional_bytes) - - self.assertEqual(41, proto.default_int32) - self.assertEqual(42, proto.default_int64) - self.assertEqual(43, proto.default_uint32) - self.assertEqual(44, proto.default_uint64) - self.assertEqual(-45, proto.default_sint32) - self.assertEqual(46, proto.default_sint64) - self.assertEqual(47, proto.default_fixed32) - self.assertEqual(48, proto.default_fixed64) - self.assertEqual(49, proto.default_sfixed32) - self.assertEqual(-50, proto.default_sfixed64) - self.assertEqual(51.5, proto.default_float) - self.assertEqual(52e3, proto.default_double) - self.assertEqual(True, proto.default_bool) - self.assertEqual('hello', proto.default_string) - self.assertEqual(b'world', proto.default_bytes) - self.assertEqual(unittest_pb2.TestAllTypes.BAR, proto.default_nested_enum) - self.assertEqual(unittest_pb2.FOREIGN_BAR, proto.default_foreign_enum) - self.assertEqual(unittest_import_pb2.IMPORT_BAR, - proto.default_import_enum) - - proto = unittest_pb2.TestExtremeDefaultValues() - self.assertEqual(u'\u1234', proto.utf8_string) - - def testHasFieldWithUnknownFieldName(self): - proto = unittest_pb2.TestAllTypes() - self.assertRaises(ValueError, proto.HasField, 'nonexistent_field') - - def testClearRemovesChildren(self): - # Make sure there aren't any implementation bugs that are only partially - # clearing the message (which can happen in the more complex C++ - # implementation which has parallel message lists). - proto = unittest_pb2.TestRequiredForeign() - for i in range(10): - proto.repeated_message.add() - proto2 = unittest_pb2.TestRequiredForeign() - proto.CopyFrom(proto2) - self.assertRaises(IndexError, lambda: proto.repeated_message[5]) - - def testSingleScalarClearField(self): - proto = unittest_pb2.TestAllTypes() - # Should be allowed to clear something that's not there (a no-op). - proto.ClearField('optional_int32') - proto.optional_int32 = 1 - self.assertTrue(proto.HasField('optional_int32')) - proto.ClearField('optional_int32') - self.assertEqual(0, proto.optional_int32) - self.assertFalse(proto.HasField('optional_int32')) - # TODO(robinson): Test all other scalar field types. - - def testRepeatedScalars(self): - proto = unittest_pb2.TestAllTypes() - - self.assertFalse(proto.repeated_int32) - self.assertEqual(0, len(proto.repeated_int32)) - proto.repeated_int32.append(5) - proto.repeated_int32.append(10) - proto.repeated_int32.append(15) - self.assertTrue(proto.repeated_int32) - self.assertEqual(3, len(proto.repeated_int32)) - - self.assertEqual([5, 10, 15], proto.repeated_int32) - - # Test single retrieval. - self.assertEqual(5, proto.repeated_int32[0]) - self.assertEqual(15, proto.repeated_int32[-1]) - # Test out-of-bounds indices. - self.assertRaises(IndexError, proto.repeated_int32.__getitem__, 1234) - self.assertRaises(IndexError, proto.repeated_int32.__getitem__, -1234) - # Test incorrect types passed to __getitem__. - self.assertRaises(TypeError, proto.repeated_int32.__getitem__, 'foo') - self.assertRaises(TypeError, proto.repeated_int32.__getitem__, None) - - # Test single assignment. - proto.repeated_int32[1] = 20 - self.assertEqual([5, 20, 15], proto.repeated_int32) - - # Test insertion. - proto.repeated_int32.insert(1, 25) - self.assertEqual([5, 25, 20, 15], proto.repeated_int32) - - # Test slice retrieval. - proto.repeated_int32.append(30) - self.assertEqual([25, 20, 15], proto.repeated_int32[1:4]) - self.assertEqual([5, 25, 20, 15, 30], proto.repeated_int32[:]) - - # Test slice assignment with an iterator - proto.repeated_int32[1:4] = (i for i in range(3)) - self.assertEqual([5, 0, 1, 2, 30], proto.repeated_int32) - - # Test slice assignment. - proto.repeated_int32[1:4] = [35, 40, 45] - self.assertEqual([5, 35, 40, 45, 30], proto.repeated_int32) - - # Test that we can use the field as an iterator. - result = [] - for i in proto.repeated_int32: - result.append(i) - self.assertEqual([5, 35, 40, 45, 30], result) - - # Test single deletion. - del proto.repeated_int32[2] - self.assertEqual([5, 35, 45, 30], proto.repeated_int32) - - # Test slice deletion. - del proto.repeated_int32[2:] - self.assertEqual([5, 35], proto.repeated_int32) - - # Test extending. - proto.repeated_int32.extend([3, 13]) - self.assertEqual([5, 35, 3, 13], proto.repeated_int32) - - # Test clearing. - proto.ClearField('repeated_int32') - self.assertFalse(proto.repeated_int32) - self.assertEqual(0, len(proto.repeated_int32)) - - proto.repeated_int32.append(1) - self.assertEqual(1, proto.repeated_int32[-1]) - # Test assignment to a negative index. - proto.repeated_int32[-1] = 2 - self.assertEqual(2, proto.repeated_int32[-1]) - - # Test deletion at negative indices. - proto.repeated_int32[:] = [0, 1, 2, 3] - del proto.repeated_int32[-1] - self.assertEqual([0, 1, 2], proto.repeated_int32) - - del proto.repeated_int32[-2] - self.assertEqual([0, 2], proto.repeated_int32) - - self.assertRaises(IndexError, proto.repeated_int32.__delitem__, -3) - self.assertRaises(IndexError, proto.repeated_int32.__delitem__, 300) - - del proto.repeated_int32[-2:-1] - self.assertEqual([2], proto.repeated_int32) - - del proto.repeated_int32[100:10000] - self.assertEqual([2], proto.repeated_int32) - - def testRepeatedScalarsRemove(self): - proto = unittest_pb2.TestAllTypes() - - self.assertFalse(proto.repeated_int32) - self.assertEqual(0, len(proto.repeated_int32)) - proto.repeated_int32.append(5) - proto.repeated_int32.append(10) - proto.repeated_int32.append(5) - proto.repeated_int32.append(5) - - self.assertEqual(4, len(proto.repeated_int32)) - proto.repeated_int32.remove(5) - self.assertEqual(3, len(proto.repeated_int32)) - self.assertEqual(10, proto.repeated_int32[0]) - self.assertEqual(5, proto.repeated_int32[1]) - self.assertEqual(5, proto.repeated_int32[2]) - - proto.repeated_int32.remove(5) - self.assertEqual(2, len(proto.repeated_int32)) - self.assertEqual(10, proto.repeated_int32[0]) - self.assertEqual(5, proto.repeated_int32[1]) - - proto.repeated_int32.remove(10) - self.assertEqual(1, len(proto.repeated_int32)) - self.assertEqual(5, proto.repeated_int32[0]) - - # Remove a non-existent element. - self.assertRaises(ValueError, proto.repeated_int32.remove, 123) - - def testRepeatedScalarsReverse_Empty(self): - proto = unittest_pb2.TestAllTypes() - - self.assertFalse(proto.repeated_int32) - self.assertEqual(0, len(proto.repeated_int32)) - - self.assertIsNone(proto.repeated_int32.reverse()) - - self.assertFalse(proto.repeated_int32) - self.assertEqual(0, len(proto.repeated_int32)) - - def testRepeatedScalarsReverse_NonEmpty(self): - proto = unittest_pb2.TestAllTypes() - - self.assertFalse(proto.repeated_int32) - self.assertEqual(0, len(proto.repeated_int32)) - - proto.repeated_int32.append(1) - proto.repeated_int32.append(2) - proto.repeated_int32.append(3) - proto.repeated_int32.append(4) - - self.assertEqual(4, len(proto.repeated_int32)) - - self.assertIsNone(proto.repeated_int32.reverse()) - - self.assertEqual(4, len(proto.repeated_int32)) - self.assertEqual(4, proto.repeated_int32[0]) - self.assertEqual(3, proto.repeated_int32[1]) - self.assertEqual(2, proto.repeated_int32[2]) - self.assertEqual(1, proto.repeated_int32[3]) - - def testRepeatedComposites(self): - proto = unittest_pb2.TestAllTypes() - self.assertFalse(proto.repeated_nested_message) - self.assertEqual(0, len(proto.repeated_nested_message)) - m0 = proto.repeated_nested_message.add() - m1 = proto.repeated_nested_message.add() - self.assertTrue(proto.repeated_nested_message) - self.assertEqual(2, len(proto.repeated_nested_message)) - self.assertListsEqual([m0, m1], proto.repeated_nested_message) - self.assertIsInstance(m0, unittest_pb2.TestAllTypes.NestedMessage) - - # Test out-of-bounds indices. - self.assertRaises(IndexError, proto.repeated_nested_message.__getitem__, - 1234) - self.assertRaises(IndexError, proto.repeated_nested_message.__getitem__, - -1234) - - # Test incorrect types passed to __getitem__. - self.assertRaises(TypeError, proto.repeated_nested_message.__getitem__, - 'foo') - self.assertRaises(TypeError, proto.repeated_nested_message.__getitem__, - None) - - # Test slice retrieval. - m2 = proto.repeated_nested_message.add() - m3 = proto.repeated_nested_message.add() - m4 = proto.repeated_nested_message.add() - self.assertListsEqual( - [m1, m2, m3], proto.repeated_nested_message[1:4]) - self.assertListsEqual( - [m0, m1, m2, m3, m4], proto.repeated_nested_message[:]) - self.assertListsEqual( - [m0, m1], proto.repeated_nested_message[:2]) - self.assertListsEqual( - [m2, m3, m4], proto.repeated_nested_message[2:]) - self.assertEqual( - m0, proto.repeated_nested_message[0]) - self.assertListsEqual( - [m0], proto.repeated_nested_message[:1]) - - # Test that we can use the field as an iterator. - result = [] - for i in proto.repeated_nested_message: - result.append(i) - self.assertListsEqual([m0, m1, m2, m3, m4], result) - - # Test single deletion. - del proto.repeated_nested_message[2] - self.assertListsEqual([m0, m1, m3, m4], proto.repeated_nested_message) - - # Test slice deletion. - del proto.repeated_nested_message[2:] - self.assertListsEqual([m0, m1], proto.repeated_nested_message) - - # Test extending. - n1 = unittest_pb2.TestAllTypes.NestedMessage(bb=1) - n2 = unittest_pb2.TestAllTypes.NestedMessage(bb=2) - proto.repeated_nested_message.extend([n1,n2]) - self.assertEqual(4, len(proto.repeated_nested_message)) - self.assertEqual(n1, proto.repeated_nested_message[2]) - self.assertEqual(n2, proto.repeated_nested_message[3]) - self.assertRaises(TypeError, - proto.repeated_nested_message.extend, n1) - self.assertRaises(TypeError, - proto.repeated_nested_message.extend, [0]) - wrong_message_type = unittest_pb2.TestAllTypes() - self.assertRaises(TypeError, - proto.repeated_nested_message.extend, - [wrong_message_type]) - - # Test clearing. - proto.ClearField('repeated_nested_message') - self.assertFalse(proto.repeated_nested_message) - self.assertEqual(0, len(proto.repeated_nested_message)) - - # Test constructing an element while adding it. - proto.repeated_nested_message.add(bb=23) - self.assertEqual(1, len(proto.repeated_nested_message)) - self.assertEqual(23, proto.repeated_nested_message[0].bb) - self.assertRaises(TypeError, proto.repeated_nested_message.add, 23) - with self.assertRaises(Exception): - proto.repeated_nested_message[0] = 23 - - def testRepeatedCompositeRemove(self): - proto = unittest_pb2.TestAllTypes() - - self.assertEqual(0, len(proto.repeated_nested_message)) - m0 = proto.repeated_nested_message.add() - # Need to set some differentiating variable so m0 != m1 != m2: - m0.bb = len(proto.repeated_nested_message) - m1 = proto.repeated_nested_message.add() - m1.bb = len(proto.repeated_nested_message) - self.assertTrue(m0 != m1) - m2 = proto.repeated_nested_message.add() - m2.bb = len(proto.repeated_nested_message) - self.assertListsEqual([m0, m1, m2], proto.repeated_nested_message) - - self.assertEqual(3, len(proto.repeated_nested_message)) - proto.repeated_nested_message.remove(m0) - self.assertEqual(2, len(proto.repeated_nested_message)) - self.assertEqual(m1, proto.repeated_nested_message[0]) - self.assertEqual(m2, proto.repeated_nested_message[1]) - - # Removing m0 again or removing None should raise error - self.assertRaises(ValueError, proto.repeated_nested_message.remove, m0) - self.assertRaises(ValueError, proto.repeated_nested_message.remove, None) - self.assertEqual(2, len(proto.repeated_nested_message)) - - proto.repeated_nested_message.remove(m2) - self.assertEqual(1, len(proto.repeated_nested_message)) - self.assertEqual(m1, proto.repeated_nested_message[0]) - - def testRepeatedCompositeReverse_Empty(self): - proto = unittest_pb2.TestAllTypes() - - self.assertFalse(proto.repeated_nested_message) - self.assertEqual(0, len(proto.repeated_nested_message)) - - self.assertIsNone(proto.repeated_nested_message.reverse()) - - self.assertFalse(proto.repeated_nested_message) - self.assertEqual(0, len(proto.repeated_nested_message)) - - def testRepeatedCompositeReverse_NonEmpty(self): - proto = unittest_pb2.TestAllTypes() - - self.assertFalse(proto.repeated_nested_message) - self.assertEqual(0, len(proto.repeated_nested_message)) - - m0 = proto.repeated_nested_message.add() - m0.bb = len(proto.repeated_nested_message) - m1 = proto.repeated_nested_message.add() - m1.bb = len(proto.repeated_nested_message) - m2 = proto.repeated_nested_message.add() - m2.bb = len(proto.repeated_nested_message) - self.assertListsEqual([m0, m1, m2], proto.repeated_nested_message) - - self.assertIsNone(proto.repeated_nested_message.reverse()) - - self.assertListsEqual([m2, m1, m0], proto.repeated_nested_message) - - def testHandWrittenReflection(self): - # Hand written extensions are only supported by the pure-Python - # implementation of the API. - if api_implementation.Type() != 'python': - return - - FieldDescriptor = descriptor.FieldDescriptor - foo_field_descriptor = FieldDescriptor( - name='foo_field', full_name='MyProto.foo_field', - index=0, number=1, type=FieldDescriptor.TYPE_INT64, - cpp_type=FieldDescriptor.CPPTYPE_INT64, - label=FieldDescriptor.LABEL_OPTIONAL, default_value=0, - containing_type=None, message_type=None, enum_type=None, - is_extension=False, extension_scope=None, - options=descriptor_pb2.FieldOptions(), - # pylint: disable=protected-access - create_key=descriptor._internal_create_key) - mydescriptor = descriptor.Descriptor( - name='MyProto', full_name='MyProto', filename='ignored', - containing_type=None, nested_types=[], enum_types=[], - fields=[foo_field_descriptor], extensions=[], - options=descriptor_pb2.MessageOptions(), - # pylint: disable=protected-access - create_key=descriptor._internal_create_key) - - class MyProtoClass( - message.Message, metaclass=reflection.GeneratedProtocolMessageType): - DESCRIPTOR = mydescriptor - myproto_instance = MyProtoClass() - self.assertEqual(0, myproto_instance.foo_field) - self.assertFalse(myproto_instance.HasField('foo_field')) - myproto_instance.foo_field = 23 - self.assertEqual(23, myproto_instance.foo_field) - self.assertTrue(myproto_instance.HasField('foo_field')) - - @testing_refleaks.SkipReferenceLeakChecker('MakeDescriptor is not repeatable') - def testDescriptorProtoSupport(self): - # Hand written descriptors/reflection are only supported by the pure-Python - # implementation of the API. - if api_implementation.Type() != 'python': - return - - def AddDescriptorField(proto, field_name, field_type): - AddDescriptorField.field_index += 1 - new_field = proto.field.add() - new_field.name = field_name - new_field.type = field_type - new_field.number = AddDescriptorField.field_index - new_field.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL - - AddDescriptorField.field_index = 0 - - desc_proto = descriptor_pb2.DescriptorProto() - desc_proto.name = 'Car' - fdp = descriptor_pb2.FieldDescriptorProto - AddDescriptorField(desc_proto, 'name', fdp.TYPE_STRING) - AddDescriptorField(desc_proto, 'year', fdp.TYPE_INT64) - AddDescriptorField(desc_proto, 'automatic', fdp.TYPE_BOOL) - AddDescriptorField(desc_proto, 'price', fdp.TYPE_DOUBLE) - # Add a repeated field - AddDescriptorField.field_index += 1 - new_field = desc_proto.field.add() - new_field.name = 'owners' - new_field.type = fdp.TYPE_STRING - new_field.number = AddDescriptorField.field_index - new_field.label = descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED - - desc = descriptor.MakeDescriptor(desc_proto) - self.assertTrue('name' in desc.fields_by_name) - self.assertTrue('year' in desc.fields_by_name) - self.assertTrue('automatic' in desc.fields_by_name) - self.assertTrue('price' in desc.fields_by_name) - self.assertTrue('owners' in desc.fields_by_name) - - class CarMessage( - message.Message, metaclass=reflection.GeneratedProtocolMessageType): - DESCRIPTOR = desc - - prius = CarMessage() - prius.name = 'prius' - prius.year = 2010 - prius.automatic = True - prius.price = 25134.75 - prius.owners.extend(['bob', 'susan']) - - serialized_prius = prius.SerializeToString() - new_prius = reflection.ParseMessage(desc, serialized_prius) - self.assertIsNot(new_prius, prius) - self.assertEqual(prius, new_prius) - - # these are unnecessary assuming message equality works as advertised but - # explicitly check to be safe since we're mucking about in metaclass foo - self.assertEqual(prius.name, new_prius.name) - self.assertEqual(prius.year, new_prius.year) - self.assertEqual(prius.automatic, new_prius.automatic) - self.assertEqual(prius.price, new_prius.price) - self.assertEqual(prius.owners, new_prius.owners) - - def testExtensionDelete(self): - extendee_proto = more_extensions_pb2.ExtendedMessage() - - extension_int32 = more_extensions_pb2.optional_int_extension - extendee_proto.Extensions[extension_int32] = 23 - - extension_repeated = more_extensions_pb2.repeated_int_extension - extendee_proto.Extensions[extension_repeated].append(11) - - extension_msg = more_extensions_pb2.optional_message_extension - extendee_proto.Extensions[extension_msg].foreign_message_int = 56 - - self.assertEqual(len(extendee_proto.Extensions), 3) - del extendee_proto.Extensions[extension_msg] - self.assertEqual(len(extendee_proto.Extensions), 2) - del extendee_proto.Extensions[extension_repeated] - self.assertEqual(len(extendee_proto.Extensions), 1) - # Delete a none exist extension. It is OK to "del m.Extensions[ext]" - # even if the extension is not present in the message; we don't - # raise KeyError. This is consistent with "m.Extensions[ext]" - # returning a default value even if we did not set anything. - del extendee_proto.Extensions[extension_repeated] - self.assertEqual(len(extendee_proto.Extensions), 1) - del extendee_proto.Extensions[extension_int32] - self.assertEqual(len(extendee_proto.Extensions), 0) - - def testExtensionIter(self): - extendee_proto = more_extensions_pb2.ExtendedMessage() - - extension_int32 = more_extensions_pb2.optional_int_extension - extendee_proto.Extensions[extension_int32] = 23 - - extension_repeated = more_extensions_pb2.repeated_int_extension - extendee_proto.Extensions[extension_repeated].append(11) - - extension_msg = more_extensions_pb2.optional_message_extension - extendee_proto.Extensions[extension_msg].foreign_message_int = 56 - - # Set some normal fields. - extendee_proto.optional_int32 = 1 - extendee_proto.repeated_string.append('hi') - - expected = (extension_int32, extension_msg, extension_repeated) - count = 0 - for item in extendee_proto.Extensions: - self.assertEqual(item.name, expected[count].name) - self.assertIn(item, extendee_proto.Extensions) - count += 1 - self.assertEqual(count, 3) - - def testExtensionContainsError(self): - extendee_proto = more_extensions_pb2.ExtendedMessage() - self.assertRaises(KeyError, extendee_proto.Extensions.__contains__, 0) - - field = more_extensions_pb2.ExtendedMessage.DESCRIPTOR.fields_by_name[ - 'optional_int32'] - self.assertRaises(KeyError, extendee_proto.Extensions.__contains__, field) - - def testTopLevelExtensionsForOptionalScalar(self): - extendee_proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.optional_int32_extension - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - self.assertEqual(0, extendee_proto.Extensions[extension]) - # As with normal scalar fields, just doing a read doesn't actually set the - # "has" bit. - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - # Actually set the thing. - extendee_proto.Extensions[extension] = 23 - self.assertEqual(23, extendee_proto.Extensions[extension]) - self.assertTrue(extendee_proto.HasExtension(extension)) - self.assertIn(extension, extendee_proto.Extensions) - # Ensure that clearing works as well. - extendee_proto.ClearExtension(extension) - self.assertEqual(0, extendee_proto.Extensions[extension]) - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - - def testTopLevelExtensionsForRepeatedScalar(self): - extendee_proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.repeated_string_extension - self.assertEqual(0, len(extendee_proto.Extensions[extension])) - self.assertNotIn(extension, extendee_proto.Extensions) - extendee_proto.Extensions[extension].append('foo') - self.assertEqual(['foo'], extendee_proto.Extensions[extension]) - self.assertIn(extension, extendee_proto.Extensions) - string_list = extendee_proto.Extensions[extension] - extendee_proto.ClearExtension(extension) - self.assertEqual(0, len(extendee_proto.Extensions[extension])) - self.assertNotIn(extension, extendee_proto.Extensions) - self.assertIsNot(string_list, extendee_proto.Extensions[extension]) - # Shouldn't be allowed to do Extensions[extension] = 'a' - self.assertRaises(TypeError, operator.setitem, extendee_proto.Extensions, - extension, 'a') - - def testTopLevelExtensionsForOptionalMessage(self): - extendee_proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.optional_foreign_message_extension - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - self.assertEqual(0, extendee_proto.Extensions[extension].c) - # As with normal (non-extension) fields, merely reading from the - # thing shouldn't set the "has" bit. - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - extendee_proto.Extensions[extension].c = 23 - self.assertEqual(23, extendee_proto.Extensions[extension].c) - self.assertTrue(extendee_proto.HasExtension(extension)) - self.assertIn(extension, extendee_proto.Extensions) - # Save a reference here. - foreign_message = extendee_proto.Extensions[extension] - extendee_proto.ClearExtension(extension) - self.assertIsNot(foreign_message, extendee_proto.Extensions[extension]) - # Setting a field on foreign_message now shouldn't set - # any "has" bits on extendee_proto. - foreign_message.c = 42 - self.assertEqual(42, foreign_message.c) - self.assertTrue(foreign_message.HasField('c')) - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - # Shouldn't be allowed to do Extensions[extension] = 'a' - self.assertRaises(TypeError, operator.setitem, extendee_proto.Extensions, - extension, 'a') - - def testTopLevelExtensionsForRepeatedMessage(self): - extendee_proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.repeatedgroup_extension - self.assertEqual(0, len(extendee_proto.Extensions[extension])) - group = extendee_proto.Extensions[extension].add() - group.a = 23 - self.assertEqual(23, extendee_proto.Extensions[extension][0].a) - group.a = 42 - self.assertEqual(42, extendee_proto.Extensions[extension][0].a) - group_list = extendee_proto.Extensions[extension] - extendee_proto.ClearExtension(extension) - self.assertEqual(0, len(extendee_proto.Extensions[extension])) - self.assertIsNot(group_list, extendee_proto.Extensions[extension]) - # Shouldn't be allowed to do Extensions[extension] = 'a' - self.assertRaises(TypeError, operator.setitem, extendee_proto.Extensions, - extension, 'a') - - def testNestedExtensions(self): - extendee_proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.TestRequired.single - - # We just test the non-repeated case. - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - required = extendee_proto.Extensions[extension] - self.assertEqual(0, required.a) - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - required.a = 23 - self.assertEqual(23, extendee_proto.Extensions[extension].a) - self.assertTrue(extendee_proto.HasExtension(extension)) - self.assertIn(extension, extendee_proto.Extensions) - extendee_proto.ClearExtension(extension) - self.assertIsNot(required, extendee_proto.Extensions[extension]) - self.assertFalse(extendee_proto.HasExtension(extension)) - self.assertNotIn(extension, extendee_proto.Extensions) - - def testRegisteredExtensions(self): - pool = unittest_pb2.DESCRIPTOR.pool - self.assertTrue( - pool.FindExtensionByNumber( - unittest_pb2.TestAllExtensions.DESCRIPTOR, 1)) - self.assertIs( - pool.FindExtensionByName( - 'protobuf_unittest.optional_int32_extension').containing_type, - unittest_pb2.TestAllExtensions.DESCRIPTOR) - # Make sure extensions haven't been registered into types that shouldn't - # have any. - self.assertEqual(0, len( - pool.FindAllExtensions(unittest_pb2.TestAllTypes.DESCRIPTOR))) - - # If message A directly contains message B, and - # a.HasField('b') is currently False, then mutating any - # extension in B should change a.HasField('b') to True - # (and so on up the object tree). - def testHasBitsForAncestorsOfExtendedMessage(self): - # Optional scalar extension. - toplevel = more_extensions_pb2.TopLevelMessage() - self.assertFalse(toplevel.HasField('submessage')) - self.assertEqual(0, toplevel.submessage.Extensions[ - more_extensions_pb2.optional_int_extension]) - self.assertFalse(toplevel.HasField('submessage')) - toplevel.submessage.Extensions[ - more_extensions_pb2.optional_int_extension] = 23 - self.assertEqual(23, toplevel.submessage.Extensions[ - more_extensions_pb2.optional_int_extension]) - self.assertTrue(toplevel.HasField('submessage')) - - # Repeated scalar extension. - toplevel = more_extensions_pb2.TopLevelMessage() - self.assertFalse(toplevel.HasField('submessage')) - self.assertEqual([], toplevel.submessage.Extensions[ - more_extensions_pb2.repeated_int_extension]) - self.assertFalse(toplevel.HasField('submessage')) - toplevel.submessage.Extensions[ - more_extensions_pb2.repeated_int_extension].append(23) - self.assertEqual([23], toplevel.submessage.Extensions[ - more_extensions_pb2.repeated_int_extension]) - self.assertTrue(toplevel.HasField('submessage')) - - # Optional message extension. - toplevel = more_extensions_pb2.TopLevelMessage() - self.assertFalse(toplevel.HasField('submessage')) - self.assertEqual(0, toplevel.submessage.Extensions[ - more_extensions_pb2.optional_message_extension].foreign_message_int) - self.assertFalse(toplevel.HasField('submessage')) - toplevel.submessage.Extensions[ - more_extensions_pb2.optional_message_extension].foreign_message_int = 23 - self.assertEqual(23, toplevel.submessage.Extensions[ - more_extensions_pb2.optional_message_extension].foreign_message_int) - self.assertTrue(toplevel.HasField('submessage')) - - # Repeated message extension. - toplevel = more_extensions_pb2.TopLevelMessage() - self.assertFalse(toplevel.HasField('submessage')) - self.assertEqual(0, len(toplevel.submessage.Extensions[ - more_extensions_pb2.repeated_message_extension])) - self.assertFalse(toplevel.HasField('submessage')) - foreign = toplevel.submessage.Extensions[ - more_extensions_pb2.repeated_message_extension].add() - self.assertEqual(foreign, toplevel.submessage.Extensions[ - more_extensions_pb2.repeated_message_extension][0]) - self.assertTrue(toplevel.HasField('submessage')) - - def testDisconnectionAfterClearingEmptyMessage(self): - toplevel = more_extensions_pb2.TopLevelMessage() - extendee_proto = toplevel.submessage - extension = more_extensions_pb2.optional_message_extension - extension_proto = extendee_proto.Extensions[extension] - extendee_proto.ClearExtension(extension) - extension_proto.foreign_message_int = 23 - - self.assertIsNot(extension_proto, extendee_proto.Extensions[extension]) - - def testExtensionFailureModes(self): - extendee_proto = unittest_pb2.TestAllExtensions() - - # Try non-extension-handle arguments to HasExtension, - # ClearExtension(), and Extensions[]... - self.assertRaises(KeyError, extendee_proto.HasExtension, 1234) - self.assertRaises(KeyError, extendee_proto.ClearExtension, 1234) - self.assertRaises(KeyError, extendee_proto.Extensions.__getitem__, 1234) - self.assertRaises(KeyError, extendee_proto.Extensions.__setitem__, 1234, 5) - - # Try something that *is* an extension handle, just not for - # this message... - for unknown_handle in (more_extensions_pb2.optional_int_extension, - more_extensions_pb2.optional_message_extension, - more_extensions_pb2.repeated_int_extension, - more_extensions_pb2.repeated_message_extension): - self.assertRaises(KeyError, extendee_proto.HasExtension, - unknown_handle) - self.assertRaises(KeyError, extendee_proto.ClearExtension, - unknown_handle) - self.assertRaises(KeyError, extendee_proto.Extensions.__getitem__, - unknown_handle) - self.assertRaises(KeyError, extendee_proto.Extensions.__setitem__, - unknown_handle, 5) - - # Try call HasExtension() with a valid handle, but for a - # *repeated* field. (Just as with non-extension repeated - # fields, Has*() isn't supported for extension repeated fields). - self.assertRaises(KeyError, extendee_proto.HasExtension, - unittest_pb2.repeated_string_extension) - - def testMergeFromOptionalGroup(self): - # Test merge with an optional group. - proto1 = unittest_pb2.TestAllTypes() - proto1.optionalgroup.a = 12 - proto2 = unittest_pb2.TestAllTypes() - proto2.MergeFrom(proto1) - self.assertEqual(12, proto2.optionalgroup.a) - - def testMergeFromExtensionsSingular(self): - proto1 = unittest_pb2.TestAllExtensions() - proto1.Extensions[unittest_pb2.optional_int32_extension] = 1 - - proto2 = unittest_pb2.TestAllExtensions() - proto2.MergeFrom(proto1) - self.assertEqual( - 1, proto2.Extensions[unittest_pb2.optional_int32_extension]) - - def testMergeFromExtensionsRepeated(self): - proto1 = unittest_pb2.TestAllExtensions() - proto1.Extensions[unittest_pb2.repeated_int32_extension].append(1) - proto1.Extensions[unittest_pb2.repeated_int32_extension].append(2) - - proto2 = unittest_pb2.TestAllExtensions() - proto2.Extensions[unittest_pb2.repeated_int32_extension].append(0) - proto2.MergeFrom(proto1) - self.assertEqual( - 3, len(proto2.Extensions[unittest_pb2.repeated_int32_extension])) - self.assertEqual( - 0, proto2.Extensions[unittest_pb2.repeated_int32_extension][0]) - self.assertEqual( - 1, proto2.Extensions[unittest_pb2.repeated_int32_extension][1]) - self.assertEqual( - 2, proto2.Extensions[unittest_pb2.repeated_int32_extension][2]) - - def testMergeFromExtensionsNestedMessage(self): - proto1 = unittest_pb2.TestAllExtensions() - ext1 = proto1.Extensions[ - unittest_pb2.repeated_nested_message_extension] - m = ext1.add() - m.bb = 222 - m = ext1.add() - m.bb = 333 - - proto2 = unittest_pb2.TestAllExtensions() - ext2 = proto2.Extensions[ - unittest_pb2.repeated_nested_message_extension] - m = ext2.add() - m.bb = 111 - - proto2.MergeFrom(proto1) - ext2 = proto2.Extensions[ - unittest_pb2.repeated_nested_message_extension] - self.assertEqual(3, len(ext2)) - self.assertEqual(111, ext2[0].bb) - self.assertEqual(222, ext2[1].bb) - self.assertEqual(333, ext2[2].bb) - - def testCopyFromBadType(self): - # The python implementation doesn't raise an exception in this - # case. In theory it should. - if api_implementation.Type() == 'python': - return - proto1 = unittest_pb2.TestAllTypes() - proto2 = unittest_pb2.TestAllExtensions() - self.assertRaises(TypeError, proto1.CopyFrom, proto2) - - def testClear(self): - proto = unittest_pb2.TestAllTypes() - # C++ implementation does not support lazy fields right now so leave it - # out for now. - if api_implementation.Type() == 'python': - test_util.SetAllFields(proto) - else: - test_util.SetAllNonLazyFields(proto) - # Clear the message. - proto.Clear() - self.assertEqual(proto.ByteSize(), 0) - empty_proto = unittest_pb2.TestAllTypes() - self.assertEqual(proto, empty_proto) - - # Test if extensions which were set are cleared. - proto = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(proto) - # Clear the message. - proto.Clear() - self.assertEqual(proto.ByteSize(), 0) - empty_proto = unittest_pb2.TestAllExtensions() - self.assertEqual(proto, empty_proto) - - def testDisconnectingInOneof(self): - m = unittest_pb2.TestOneof2() # This message has two messages in a oneof. - m.foo_message.moo_int = 5 - sub_message = m.foo_message - # Accessing another message's field does not clear the first one - self.assertEqual(m.foo_lazy_message.moo_int, 0) - self.assertEqual(m.foo_message.moo_int, 5) - # But mutating another message in the oneof detaches the first one. - m.foo_lazy_message.moo_int = 6 - self.assertEqual(m.foo_message.moo_int, 0) - # The reference we got above was detached and is still valid. - self.assertEqual(sub_message.moo_int, 5) - sub_message.moo_int = 7 - - def assertInitialized(self, proto): - self.assertTrue(proto.IsInitialized()) - # Neither method should raise an exception. - proto.SerializeToString() - proto.SerializePartialToString() - - def assertNotInitialized(self, proto, error_size=None): - errors = [] - self.assertFalse(proto.IsInitialized()) - self.assertFalse(proto.IsInitialized(errors)) - self.assertEqual(error_size, len(errors)) - self.assertRaises(message.EncodeError, proto.SerializeToString) - # "Partial" serialization doesn't care if message is uninitialized. - proto.SerializePartialToString() - - def testIsInitialized(self): - # Trivial cases - all optional fields and extensions. - proto = unittest_pb2.TestAllTypes() - self.assertInitialized(proto) - proto = unittest_pb2.TestAllExtensions() - self.assertInitialized(proto) - - # The case of uninitialized required fields. - proto = unittest_pb2.TestRequired() - self.assertNotInitialized(proto, 3) - proto.a = proto.b = proto.c = 2 - self.assertInitialized(proto) - - # The case of uninitialized submessage. - proto = unittest_pb2.TestRequiredForeign() - self.assertInitialized(proto) - proto.optional_message.a = 1 - self.assertNotInitialized(proto, 2) - proto.optional_message.b = 0 - proto.optional_message.c = 0 - self.assertInitialized(proto) - - # Uninitialized repeated submessage. - message1 = proto.repeated_message.add() - self.assertNotInitialized(proto, 3) - message1.a = message1.b = message1.c = 0 - self.assertInitialized(proto) - - # Uninitialized repeated group in an extension. - proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.TestRequired.multi - message1 = proto.Extensions[extension].add() - message2 = proto.Extensions[extension].add() - self.assertNotInitialized(proto, 6) - message1.a = 1 - message1.b = 1 - message1.c = 1 - self.assertNotInitialized(proto, 3) - message2.a = 2 - message2.b = 2 - message2.c = 2 - self.assertInitialized(proto) - - # Uninitialized nonrepeated message in an extension. - proto = unittest_pb2.TestAllExtensions() - extension = unittest_pb2.TestRequired.single - proto.Extensions[extension].a = 1 - self.assertNotInitialized(proto, 2) - proto.Extensions[extension].b = 2 - proto.Extensions[extension].c = 3 - self.assertInitialized(proto) - - # Try passing an errors list. - errors = [] - proto = unittest_pb2.TestRequired() - self.assertFalse(proto.IsInitialized(errors)) - self.assertEqual(errors, ['a', 'b', 'c']) - self.assertRaises(TypeError, proto.IsInitialized, 1, 2, 3) - - @unittest.skipIf( - api_implementation.Type() == 'python', - 'Errors are only available from the most recent C++ implementation.') - def testFileDescriptorErrors(self): - file_name = 'test_file_descriptor_errors.proto' - package_name = 'test_file_descriptor_errors.proto' - file_descriptor_proto = descriptor_pb2.FileDescriptorProto() - file_descriptor_proto.name = file_name - file_descriptor_proto.package = package_name - m1 = file_descriptor_proto.message_type.add() - m1.name = 'msg1' - # Compiles the proto into the C++ descriptor pool - descriptor.FileDescriptor( - file_name, - package_name, - serialized_pb=file_descriptor_proto.SerializeToString()) - # Add a FileDescriptorProto that has duplicate symbols - another_file_name = 'another_test_file_descriptor_errors.proto' - file_descriptor_proto.name = another_file_name - m2 = file_descriptor_proto.message_type.add() - m2.name = 'msg2' - with self.assertRaises(TypeError) as cm: - descriptor.FileDescriptor( - another_file_name, - package_name, - serialized_pb=file_descriptor_proto.SerializeToString()) - self.assertTrue(hasattr(cm, 'exception'), '%s not raised' % - getattr(cm.expected, '__name__', cm.expected)) - self.assertIn('test_file_descriptor_errors.proto', str(cm.exception)) - # Error message will say something about this definition being a - # duplicate, though we don't check the message exactly to avoid a - # dependency on the C++ logging code. - self.assertIn('test_file_descriptor_errors.msg1', str(cm.exception)) - - def testStringUTF8Serialization(self): - proto = message_set_extensions_pb2.TestMessageSet() - extension_message = message_set_extensions_pb2.TestMessageSetExtension2 - extension = extension_message.message_set_extension - - test_utf8 = u'Тест' - test_utf8_bytes = test_utf8.encode('utf-8') - - # 'Test' in another language, using UTF-8 charset. - proto.Extensions[extension].str = test_utf8 - - # Serialize using the MessageSet wire format (this is specified in the - # .proto file). - serialized = proto.SerializeToString() - - # Check byte size. - self.assertEqual(proto.ByteSize(), len(serialized)) - - raw = unittest_mset_pb2.RawMessageSet() - bytes_read = raw.MergeFromString(serialized) - self.assertEqual(len(serialized), bytes_read) - - message2 = message_set_extensions_pb2.TestMessageSetExtension2() - - self.assertEqual(1, len(raw.item)) - # Check that the type_id is the same as the tag ID in the .proto file. - self.assertEqual(raw.item[0].type_id, 98418634) - - # Check the actual bytes on the wire. - self.assertTrue(raw.item[0].message.endswith(test_utf8_bytes)) - bytes_read = message2.MergeFromString(raw.item[0].message) - self.assertEqual(len(raw.item[0].message), bytes_read) - - self.assertEqual(type(message2.str), str) - self.assertEqual(message2.str, test_utf8) - - # The pure Python API throws an exception on MergeFromString(), - # if any of the string fields of the message can't be UTF-8 decoded. - # The C++ implementation of the API has no way to check that on - # MergeFromString and thus has no way to throw the exception. - # - # The pure Python API always returns objects of type 'unicode' (UTF-8 - # encoded), or 'bytes' (in 7 bit ASCII). - badbytes = raw.item[0].message.replace( - test_utf8_bytes, len(test_utf8_bytes) * b'\xff') - - unicode_decode_failed = False - try: - message2.MergeFromString(badbytes) - except UnicodeDecodeError: - unicode_decode_failed = True - string_field = message2.str - self.assertTrue(unicode_decode_failed or type(string_field) is bytes) - - def testSetInParent(self): - proto = unittest_pb2.TestAllTypes() - self.assertFalse(proto.HasField('optionalgroup')) - proto.optionalgroup.SetInParent() - self.assertTrue(proto.HasField('optionalgroup')) - - def testPackageInitializationImport(self): - """Test that we can import nested messages from their __init__.py. - - Such setup is not trivial since at the time of processing of __init__.py one - can't refer to its submodules by name in code, so expressions like - google.protobuf.internal.import_test_package.inner_pb2 - don't work. They do work in imports, so we have assign an alias at import - and then use that alias in generated code. - """ - # We import here since it's the import that used to fail, and we want - # the failure to have the right context. - # pylint: disable=g-import-not-at-top - from google.protobuf.internal import import_test_package - # pylint: enable=g-import-not-at-top - msg = import_test_package.myproto.Outer() - # Just check the default value. - self.assertEqual(57, msg.inner.value) - -# Since we had so many tests for protocol buffer equality, we broke these out -# into separate TestCase classes. - - -@testing_refleaks.TestCase -class TestAllTypesEqualityTest(unittest.TestCase): - - def setUp(self): - self.first_proto = unittest_pb2.TestAllTypes() - self.second_proto = unittest_pb2.TestAllTypes() - - def testNotHashable(self): - self.assertRaises(TypeError, hash, self.first_proto) - - def testSelfEquality(self): - self.assertEqual(self.first_proto, self.first_proto) - - def testEmptyProtosEqual(self): - self.assertEqual(self.first_proto, self.second_proto) - - -@testing_refleaks.TestCase -class FullProtosEqualityTest(unittest.TestCase): - - """Equality tests using completely-full protos as a starting point.""" - - def setUp(self): - self.first_proto = unittest_pb2.TestAllTypes() - self.second_proto = unittest_pb2.TestAllTypes() - test_util.SetAllFields(self.first_proto) - test_util.SetAllFields(self.second_proto) - - def testNotHashable(self): - self.assertRaises(TypeError, hash, self.first_proto) - - def testNoneNotEqual(self): - self.assertNotEqual(self.first_proto, None) - self.assertNotEqual(None, self.second_proto) - - def testNotEqualToOtherMessage(self): - third_proto = unittest_pb2.TestRequired() - self.assertNotEqual(self.first_proto, third_proto) - self.assertNotEqual(third_proto, self.second_proto) - - def testAllFieldsFilledEquality(self): - self.assertEqual(self.first_proto, self.second_proto) - - def testNonRepeatedScalar(self): - # Nonrepeated scalar field change should cause inequality. - self.first_proto.optional_int32 += 1 - self.assertNotEqual(self.first_proto, self.second_proto) - # ...as should clearing a field. - self.first_proto.ClearField('optional_int32') - self.assertNotEqual(self.first_proto, self.second_proto) - - def testNonRepeatedComposite(self): - # Change a nonrepeated composite field. - self.first_proto.optional_nested_message.bb += 1 - self.assertNotEqual(self.first_proto, self.second_proto) - self.first_proto.optional_nested_message.bb -= 1 - self.assertEqual(self.first_proto, self.second_proto) - # Clear a field in the nested message. - self.first_proto.optional_nested_message.ClearField('bb') - self.assertNotEqual(self.first_proto, self.second_proto) - self.first_proto.optional_nested_message.bb = ( - self.second_proto.optional_nested_message.bb) - self.assertEqual(self.first_proto, self.second_proto) - # Remove the nested message entirely. - self.first_proto.ClearField('optional_nested_message') - self.assertNotEqual(self.first_proto, self.second_proto) - - def testRepeatedScalar(self): - # Change a repeated scalar field. - self.first_proto.repeated_int32.append(5) - self.assertNotEqual(self.first_proto, self.second_proto) - self.first_proto.ClearField('repeated_int32') - self.assertNotEqual(self.first_proto, self.second_proto) - - def testRepeatedComposite(self): - # Change value within a repeated composite field. - self.first_proto.repeated_nested_message[0].bb += 1 - self.assertNotEqual(self.first_proto, self.second_proto) - self.first_proto.repeated_nested_message[0].bb -= 1 - self.assertEqual(self.first_proto, self.second_proto) - # Add a value to a repeated composite field. - self.first_proto.repeated_nested_message.add() - self.assertNotEqual(self.first_proto, self.second_proto) - self.second_proto.repeated_nested_message.add() - self.assertEqual(self.first_proto, self.second_proto) - - def testNonRepeatedScalarHasBits(self): - # Ensure that we test "has" bits as well as value for - # nonrepeated scalar field. - self.first_proto.ClearField('optional_int32') - self.second_proto.optional_int32 = 0 - self.assertNotEqual(self.first_proto, self.second_proto) - - def testNonRepeatedCompositeHasBits(self): - # Ensure that we test "has" bits as well as value for - # nonrepeated composite field. - self.first_proto.ClearField('optional_nested_message') - self.second_proto.optional_nested_message.ClearField('bb') - self.assertNotEqual(self.first_proto, self.second_proto) - self.first_proto.optional_nested_message.bb = 0 - self.first_proto.optional_nested_message.ClearField('bb') - self.assertEqual(self.first_proto, self.second_proto) - - -@testing_refleaks.TestCase -class ExtensionEqualityTest(unittest.TestCase): - - def testExtensionEquality(self): - first_proto = unittest_pb2.TestAllExtensions() - second_proto = unittest_pb2.TestAllExtensions() - self.assertEqual(first_proto, second_proto) - test_util.SetAllExtensions(first_proto) - self.assertNotEqual(first_proto, second_proto) - test_util.SetAllExtensions(second_proto) - self.assertEqual(first_proto, second_proto) - - # Ensure that we check value equality. - first_proto.Extensions[unittest_pb2.optional_int32_extension] += 1 - self.assertNotEqual(first_proto, second_proto) - first_proto.Extensions[unittest_pb2.optional_int32_extension] -= 1 - self.assertEqual(first_proto, second_proto) - - # Ensure that we also look at "has" bits. - first_proto.ClearExtension(unittest_pb2.optional_int32_extension) - second_proto.Extensions[unittest_pb2.optional_int32_extension] = 0 - self.assertNotEqual(first_proto, second_proto) - first_proto.Extensions[unittest_pb2.optional_int32_extension] = 0 - self.assertEqual(first_proto, second_proto) - - # Ensure that differences in cached values - # don't matter if "has" bits are both false. - first_proto = unittest_pb2.TestAllExtensions() - second_proto = unittest_pb2.TestAllExtensions() - self.assertEqual( - 0, first_proto.Extensions[unittest_pb2.optional_int32_extension]) - self.assertEqual(first_proto, second_proto) - - -@testing_refleaks.TestCase -class MutualRecursionEqualityTest(unittest.TestCase): - - def testEqualityWithMutualRecursion(self): - first_proto = unittest_pb2.TestMutualRecursionA() - second_proto = unittest_pb2.TestMutualRecursionA() - self.assertEqual(first_proto, second_proto) - first_proto.bb.a.bb.optional_int32 = 23 - self.assertNotEqual(first_proto, second_proto) - second_proto.bb.a.bb.optional_int32 = 23 - self.assertEqual(first_proto, second_proto) - - -@testing_refleaks.TestCase -class ByteSizeTest(unittest.TestCase): - - def setUp(self): - self.proto = unittest_pb2.TestAllTypes() - self.extended_proto = more_extensions_pb2.ExtendedMessage() - self.packed_proto = unittest_pb2.TestPackedTypes() - self.packed_extended_proto = unittest_pb2.TestPackedExtensions() - - def Size(self): - return self.proto.ByteSize() - - def testEmptyMessage(self): - self.assertEqual(0, self.proto.ByteSize()) - - def testSizedOnKwargs(self): - # Use a separate message to ensure testing right after creation. - proto = unittest_pb2.TestAllTypes() - self.assertEqual(0, proto.ByteSize()) - proto_kwargs = unittest_pb2.TestAllTypes(optional_int64 = 1) - # One byte for the tag, one to encode varint 1. - self.assertEqual(2, proto_kwargs.ByteSize()) - - def testVarints(self): - def Test(i, expected_varint_size): - self.proto.Clear() - self.proto.optional_int64 = i - # Add one to the varint size for the tag info - # for tag 1. - self.assertEqual(expected_varint_size + 1, self.Size()) - Test(0, 1) - Test(1, 1) - for i, num_bytes in zip(range(7, 63, 7), range(1, 10000)): - Test((1 << i) - 1, num_bytes) - Test(-1, 10) - Test(-2, 10) - Test(-(1 << 63), 10) - - def testStrings(self): - self.proto.optional_string = '' - # Need one byte for tag info (tag #14), and one byte for length. - self.assertEqual(2, self.Size()) - - self.proto.optional_string = 'abc' - # Need one byte for tag info (tag #14), and one byte for length. - self.assertEqual(2 + len(self.proto.optional_string), self.Size()) - - self.proto.optional_string = 'x' * 128 - # Need one byte for tag info (tag #14), and TWO bytes for length. - self.assertEqual(3 + len(self.proto.optional_string), self.Size()) - - def testOtherNumerics(self): - self.proto.optional_fixed32 = 1234 - # One byte for tag and 4 bytes for fixed32. - self.assertEqual(5, self.Size()) - self.proto = unittest_pb2.TestAllTypes() - - self.proto.optional_fixed64 = 1234 - # One byte for tag and 8 bytes for fixed64. - self.assertEqual(9, self.Size()) - self.proto = unittest_pb2.TestAllTypes() - - self.proto.optional_float = 1.234 - # One byte for tag and 4 bytes for float. - self.assertEqual(5, self.Size()) - self.proto = unittest_pb2.TestAllTypes() - - self.proto.optional_double = 1.234 - # One byte for tag and 8 bytes for float. - self.assertEqual(9, self.Size()) - self.proto = unittest_pb2.TestAllTypes() - - self.proto.optional_sint32 = 64 - # One byte for tag and 2 bytes for zig-zag-encoded 64. - self.assertEqual(3, self.Size()) - self.proto = unittest_pb2.TestAllTypes() - - def testComposites(self): - # 3 bytes. - self.proto.optional_nested_message.bb = (1 << 14) - # Plus one byte for bb tag. - # Plus 1 byte for optional_nested_message serialized size. - # Plus two bytes for optional_nested_message tag. - self.assertEqual(3 + 1 + 1 + 2, self.Size()) - - def testGroups(self): - # 4 bytes. - self.proto.optionalgroup.a = (1 << 21) - # Plus two bytes for |a| tag. - # Plus 2 * two bytes for START_GROUP and END_GROUP tags. - self.assertEqual(4 + 2 + 2*2, self.Size()) - - def testRepeatedScalars(self): - self.proto.repeated_int32.append(10) # 1 byte. - self.proto.repeated_int32.append(128) # 2 bytes. - # Also need 2 bytes for each entry for tag. - self.assertEqual(1 + 2 + 2*2, self.Size()) - - def testRepeatedScalarsExtend(self): - self.proto.repeated_int32.extend([10, 128]) # 3 bytes. - # Also need 2 bytes for each entry for tag. - self.assertEqual(1 + 2 + 2*2, self.Size()) - - def testRepeatedScalarsRemove(self): - self.proto.repeated_int32.append(10) # 1 byte. - self.proto.repeated_int32.append(128) # 2 bytes. - # Also need 2 bytes for each entry for tag. - self.assertEqual(1 + 2 + 2*2, self.Size()) - self.proto.repeated_int32.remove(128) - self.assertEqual(1 + 2, self.Size()) - - def testRepeatedComposites(self): - # Empty message. 2 bytes tag plus 1 byte length. - foreign_message_0 = self.proto.repeated_nested_message.add() - # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. - foreign_message_1 = self.proto.repeated_nested_message.add() - foreign_message_1.bb = 7 - self.assertEqual(2 + 1 + 2 + 1 + 1 + 1, self.Size()) - - def testRepeatedCompositesDelete(self): - # Empty message. 2 bytes tag plus 1 byte length. - foreign_message_0 = self.proto.repeated_nested_message.add() - # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. - foreign_message_1 = self.proto.repeated_nested_message.add() - foreign_message_1.bb = 9 - self.assertEqual(2 + 1 + 2 + 1 + 1 + 1, self.Size()) - repeated_nested_message = copy.deepcopy( - self.proto.repeated_nested_message) - - # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. - del self.proto.repeated_nested_message[0] - self.assertEqual(2 + 1 + 1 + 1, self.Size()) - - # Now add a new message. - foreign_message_2 = self.proto.repeated_nested_message.add() - foreign_message_2.bb = 12 - - # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. - # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. - self.assertEqual(2 + 1 + 1 + 1 + 2 + 1 + 1 + 1, self.Size()) - - # 2 bytes tag plus 1 byte length plus 1 byte bb tag 1 byte int. - del self.proto.repeated_nested_message[1] - self.assertEqual(2 + 1 + 1 + 1, self.Size()) - - del self.proto.repeated_nested_message[0] - self.assertEqual(0, self.Size()) - - self.assertEqual(2, len(repeated_nested_message)) - del repeated_nested_message[0:1] - # TODO(jieluo): Fix cpp extension bug when delete repeated message. - if api_implementation.Type() == 'python': - self.assertEqual(1, len(repeated_nested_message)) - del repeated_nested_message[-1] - # TODO(jieluo): Fix cpp extension bug when delete repeated message. - if api_implementation.Type() == 'python': - self.assertEqual(0, len(repeated_nested_message)) - - def testRepeatedGroups(self): - # 2-byte START_GROUP plus 2-byte END_GROUP. - group_0 = self.proto.repeatedgroup.add() - # 2-byte START_GROUP plus 2-byte |a| tag + 1-byte |a| - # plus 2-byte END_GROUP. - group_1 = self.proto.repeatedgroup.add() - group_1.a = 7 - self.assertEqual(2 + 2 + 2 + 2 + 1 + 2, self.Size()) - - def testExtensions(self): - proto = unittest_pb2.TestAllExtensions() - self.assertEqual(0, proto.ByteSize()) - extension = unittest_pb2.optional_int32_extension # Field #1, 1 byte. - proto.Extensions[extension] = 23 - # 1 byte for tag, 1 byte for value. - self.assertEqual(2, proto.ByteSize()) - field = unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name[ - 'optional_int32'] - with self.assertRaises(KeyError): - proto.Extensions[field] = 23 - - def testCacheInvalidationForNonrepeatedScalar(self): - # Test non-extension. - self.proto.optional_int32 = 1 - self.assertEqual(2, self.proto.ByteSize()) - self.proto.optional_int32 = 128 - self.assertEqual(3, self.proto.ByteSize()) - self.proto.ClearField('optional_int32') - self.assertEqual(0, self.proto.ByteSize()) - - # Test within extension. - extension = more_extensions_pb2.optional_int_extension - self.extended_proto.Extensions[extension] = 1 - self.assertEqual(2, self.extended_proto.ByteSize()) - self.extended_proto.Extensions[extension] = 128 - self.assertEqual(3, self.extended_proto.ByteSize()) - self.extended_proto.ClearExtension(extension) - self.assertEqual(0, self.extended_proto.ByteSize()) - - def testCacheInvalidationForRepeatedScalar(self): - # Test non-extension. - self.proto.repeated_int32.append(1) - self.assertEqual(3, self.proto.ByteSize()) - self.proto.repeated_int32.append(1) - self.assertEqual(6, self.proto.ByteSize()) - self.proto.repeated_int32[1] = 128 - self.assertEqual(7, self.proto.ByteSize()) - self.proto.ClearField('repeated_int32') - self.assertEqual(0, self.proto.ByteSize()) - - # Test within extension. - extension = more_extensions_pb2.repeated_int_extension - repeated = self.extended_proto.Extensions[extension] - repeated.append(1) - self.assertEqual(2, self.extended_proto.ByteSize()) - repeated.append(1) - self.assertEqual(4, self.extended_proto.ByteSize()) - repeated[1] = 128 - self.assertEqual(5, self.extended_proto.ByteSize()) - self.extended_proto.ClearExtension(extension) - self.assertEqual(0, self.extended_proto.ByteSize()) - - def testCacheInvalidationForNonrepeatedMessage(self): - # Test non-extension. - self.proto.optional_foreign_message.c = 1 - self.assertEqual(5, self.proto.ByteSize()) - self.proto.optional_foreign_message.c = 128 - self.assertEqual(6, self.proto.ByteSize()) - self.proto.optional_foreign_message.ClearField('c') - self.assertEqual(3, self.proto.ByteSize()) - self.proto.ClearField('optional_foreign_message') - self.assertEqual(0, self.proto.ByteSize()) - - if api_implementation.Type() == 'python': - # This is only possible in pure-Python implementation of the API. - child = self.proto.optional_foreign_message - self.proto.ClearField('optional_foreign_message') - child.c = 128 - self.assertEqual(0, self.proto.ByteSize()) - - # Test within extension. - extension = more_extensions_pb2.optional_message_extension - child = self.extended_proto.Extensions[extension] - self.assertEqual(0, self.extended_proto.ByteSize()) - child.foreign_message_int = 1 - self.assertEqual(4, self.extended_proto.ByteSize()) - child.foreign_message_int = 128 - self.assertEqual(5, self.extended_proto.ByteSize()) - self.extended_proto.ClearExtension(extension) - self.assertEqual(0, self.extended_proto.ByteSize()) - - def testCacheInvalidationForRepeatedMessage(self): - # Test non-extension. - child0 = self.proto.repeated_foreign_message.add() - self.assertEqual(3, self.proto.ByteSize()) - self.proto.repeated_foreign_message.add() - self.assertEqual(6, self.proto.ByteSize()) - child0.c = 1 - self.assertEqual(8, self.proto.ByteSize()) - self.proto.ClearField('repeated_foreign_message') - self.assertEqual(0, self.proto.ByteSize()) - - # Test within extension. - extension = more_extensions_pb2.repeated_message_extension - child_list = self.extended_proto.Extensions[extension] - child0 = child_list.add() - self.assertEqual(2, self.extended_proto.ByteSize()) - child_list.add() - self.assertEqual(4, self.extended_proto.ByteSize()) - child0.foreign_message_int = 1 - self.assertEqual(6, self.extended_proto.ByteSize()) - child0.ClearField('foreign_message_int') - self.assertEqual(4, self.extended_proto.ByteSize()) - self.extended_proto.ClearExtension(extension) - self.assertEqual(0, self.extended_proto.ByteSize()) - - def testPackedRepeatedScalars(self): - self.assertEqual(0, self.packed_proto.ByteSize()) - - self.packed_proto.packed_int32.append(10) # 1 byte. - self.packed_proto.packed_int32.append(128) # 2 bytes. - # The tag is 2 bytes (the field number is 90), and the varint - # storing the length is 1 byte. - int_size = 1 + 2 + 3 - self.assertEqual(int_size, self.packed_proto.ByteSize()) - - self.packed_proto.packed_double.append(4.2) # 8 bytes - self.packed_proto.packed_double.append(3.25) # 8 bytes - # 2 more tag bytes, 1 more length byte. - double_size = 8 + 8 + 3 - self.assertEqual(int_size+double_size, self.packed_proto.ByteSize()) - - self.packed_proto.ClearField('packed_int32') - self.assertEqual(double_size, self.packed_proto.ByteSize()) - - def testPackedExtensions(self): - self.assertEqual(0, self.packed_extended_proto.ByteSize()) - extension = self.packed_extended_proto.Extensions[ - unittest_pb2.packed_fixed32_extension] - extension.extend([1, 2, 3, 4]) # 16 bytes - # Tag is 3 bytes. - self.assertEqual(19, self.packed_extended_proto.ByteSize()) - - -# Issues to be sure to cover include: -# * Handling of unrecognized tags ("uninterpreted_bytes"). -# * Handling of MessageSets. -# * Consistent ordering of tags in the wire format, -# including ordering between extensions and non-extension -# fields. -# * Consistent serialization of negative numbers, especially -# negative int32s. -# * Handling of empty submessages (with and without "has" -# bits set). - -@testing_refleaks.TestCase -class SerializationTest(unittest.TestCase): - - def testSerializeEmtpyMessage(self): - first_proto = unittest_pb2.TestAllTypes() - second_proto = unittest_pb2.TestAllTypes() - serialized = first_proto.SerializeToString() - self.assertEqual(first_proto.ByteSize(), len(serialized)) - self.assertEqual( - len(serialized), - second_proto.MergeFromString(serialized)) - self.assertEqual(first_proto, second_proto) - - def testSerializeAllFields(self): - first_proto = unittest_pb2.TestAllTypes() - second_proto = unittest_pb2.TestAllTypes() - test_util.SetAllFields(first_proto) - serialized = first_proto.SerializeToString() - self.assertEqual(first_proto.ByteSize(), len(serialized)) - self.assertEqual( - len(serialized), - second_proto.MergeFromString(serialized)) - self.assertEqual(first_proto, second_proto) - - def testSerializeAllExtensions(self): - first_proto = unittest_pb2.TestAllExtensions() - second_proto = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(first_proto) - serialized = first_proto.SerializeToString() - self.assertEqual( - len(serialized), - second_proto.MergeFromString(serialized)) - self.assertEqual(first_proto, second_proto) - - def testSerializeWithOptionalGroup(self): - first_proto = unittest_pb2.TestAllTypes() - second_proto = unittest_pb2.TestAllTypes() - first_proto.optionalgroup.a = 242 - serialized = first_proto.SerializeToString() - self.assertEqual( - len(serialized), - second_proto.MergeFromString(serialized)) - self.assertEqual(first_proto, second_proto) - - def testSerializeNegativeValues(self): - first_proto = unittest_pb2.TestAllTypes() - - first_proto.optional_int32 = -1 - first_proto.optional_int64 = -(2 << 40) - first_proto.optional_sint32 = -3 - first_proto.optional_sint64 = -(4 << 40) - first_proto.optional_sfixed32 = -5 - first_proto.optional_sfixed64 = -(6 << 40) - - second_proto = unittest_pb2.TestAllTypes.FromString( - first_proto.SerializeToString()) - - self.assertEqual(first_proto, second_proto) - - def testParseTruncated(self): - # This test is only applicable for the Python implementation of the API. - if api_implementation.Type() != 'python': - return - - first_proto = unittest_pb2.TestAllTypes() - test_util.SetAllFields(first_proto) - serialized = memoryview(first_proto.SerializeToString()) - - for truncation_point in range(len(serialized) + 1): - try: - second_proto = unittest_pb2.TestAllTypes() - unknown_fields = unittest_pb2.TestEmptyMessage() - pos = second_proto._InternalParse(serialized, 0, truncation_point) - # If we didn't raise an error then we read exactly the amount expected. - self.assertEqual(truncation_point, pos) - - # Parsing to unknown fields should not throw if parsing to known fields - # did not. - try: - pos2 = unknown_fields._InternalParse(serialized, 0, truncation_point) - self.assertEqual(truncation_point, pos2) - except message.DecodeError: - self.fail('Parsing unknown fields failed when parsing known fields ' - 'did not.') - except message.DecodeError: - # Parsing unknown fields should also fail. - self.assertRaises(message.DecodeError, unknown_fields._InternalParse, - serialized, 0, truncation_point) - - def testCanonicalSerializationOrder(self): - proto = more_messages_pb2.OutOfOrderFields() - # These are also their tag numbers. Even though we're setting these in - # reverse-tag order AND they're listed in reverse tag-order in the .proto - # file, they should nonetheless be serialized in tag order. - proto.optional_sint32 = 5 - proto.Extensions[more_messages_pb2.optional_uint64] = 4 - proto.optional_uint32 = 3 - proto.Extensions[more_messages_pb2.optional_int64] = 2 - proto.optional_int32 = 1 - serialized = proto.SerializeToString() - self.assertEqual(proto.ByteSize(), len(serialized)) - d = _MiniDecoder(serialized) - ReadTag = d.ReadFieldNumberAndWireType - self.assertEqual((1, wire_format.WIRETYPE_VARINT), ReadTag()) - self.assertEqual(1, d.ReadInt32()) - self.assertEqual((2, wire_format.WIRETYPE_VARINT), ReadTag()) - self.assertEqual(2, d.ReadInt64()) - self.assertEqual((3, wire_format.WIRETYPE_VARINT), ReadTag()) - self.assertEqual(3, d.ReadUInt32()) - self.assertEqual((4, wire_format.WIRETYPE_VARINT), ReadTag()) - self.assertEqual(4, d.ReadUInt64()) - self.assertEqual((5, wire_format.WIRETYPE_VARINT), ReadTag()) - self.assertEqual(5, d.ReadSInt32()) - - def testCanonicalSerializationOrderSameAsCpp(self): - # Copy of the same test we use for C++. - proto = unittest_pb2.TestFieldOrderings() - test_util.SetAllFieldsAndExtensions(proto) - serialized = proto.SerializeToString() - test_util.ExpectAllFieldsAndExtensionsInOrder(serialized) - - def testMergeFromStringWhenFieldsAlreadySet(self): - first_proto = unittest_pb2.TestAllTypes() - first_proto.repeated_string.append('foobar') - first_proto.optional_int32 = 23 - first_proto.optional_nested_message.bb = 42 - serialized = first_proto.SerializeToString() - - second_proto = unittest_pb2.TestAllTypes() - second_proto.repeated_string.append('baz') - second_proto.optional_int32 = 100 - second_proto.optional_nested_message.bb = 999 - - bytes_parsed = second_proto.MergeFromString(serialized) - self.assertEqual(len(serialized), bytes_parsed) - - # Ensure that we append to repeated fields. - self.assertEqual(['baz', 'foobar'], list(second_proto.repeated_string)) - # Ensure that we overwrite nonrepeatd scalars. - self.assertEqual(23, second_proto.optional_int32) - # Ensure that we recursively call MergeFromString() on - # submessages. - self.assertEqual(42, second_proto.optional_nested_message.bb) - - def testMessageSetWireFormat(self): - proto = message_set_extensions_pb2.TestMessageSet() - extension_message1 = message_set_extensions_pb2.TestMessageSetExtension1 - extension_message2 = message_set_extensions_pb2.TestMessageSetExtension2 - extension1 = extension_message1.message_set_extension - extension2 = extension_message2.message_set_extension - extension3 = message_set_extensions_pb2.message_set_extension3 - proto.Extensions[extension1].i = 123 - proto.Extensions[extension2].str = 'foo' - proto.Extensions[extension3].text = 'bar' - - # Serialize using the MessageSet wire format (this is specified in the - # .proto file). - serialized = proto.SerializeToString() - - raw = unittest_mset_pb2.RawMessageSet() - self.assertEqual(False, - raw.DESCRIPTOR.GetOptions().message_set_wire_format) - self.assertEqual( - len(serialized), - raw.MergeFromString(serialized)) - self.assertEqual(3, len(raw.item)) - - message1 = message_set_extensions_pb2.TestMessageSetExtension1() - self.assertEqual( - len(raw.item[0].message), - message1.MergeFromString(raw.item[0].message)) - self.assertEqual(123, message1.i) - - message2 = message_set_extensions_pb2.TestMessageSetExtension2() - self.assertEqual( - len(raw.item[1].message), - message2.MergeFromString(raw.item[1].message)) - self.assertEqual('foo', message2.str) - - message3 = message_set_extensions_pb2.TestMessageSetExtension3() - self.assertEqual( - len(raw.item[2].message), - message3.MergeFromString(raw.item[2].message)) - self.assertEqual('bar', message3.text) - - # Deserialize using the MessageSet wire format. - proto2 = message_set_extensions_pb2.TestMessageSet() - self.assertEqual( - len(serialized), - proto2.MergeFromString(serialized)) - self.assertEqual(123, proto2.Extensions[extension1].i) - self.assertEqual('foo', proto2.Extensions[extension2].str) - self.assertEqual('bar', proto2.Extensions[extension3].text) - - # Check byte size. - self.assertEqual(proto2.ByteSize(), len(serialized)) - self.assertEqual(proto.ByteSize(), len(serialized)) - - def testMessageSetWireFormatUnknownExtension(self): - # Create a message using the message set wire format with an unknown - # message. - raw = unittest_mset_pb2.RawMessageSet() - - # Add an item. - item = raw.item.add() - item.type_id = 98418603 - extension_message1 = message_set_extensions_pb2.TestMessageSetExtension1 - message1 = message_set_extensions_pb2.TestMessageSetExtension1() - message1.i = 12345 - item.message = message1.SerializeToString() - - # Add a second, unknown extension. - item = raw.item.add() - item.type_id = 98418604 - extension_message1 = message_set_extensions_pb2.TestMessageSetExtension1 - message1 = message_set_extensions_pb2.TestMessageSetExtension1() - message1.i = 12346 - item.message = message1.SerializeToString() - - # Add another unknown extension. - item = raw.item.add() - item.type_id = 98418605 - message1 = message_set_extensions_pb2.TestMessageSetExtension2() - message1.str = 'foo' - item.message = message1.SerializeToString() - - serialized = raw.SerializeToString() - - # Parse message using the message set wire format. - proto = message_set_extensions_pb2.TestMessageSet() - self.assertEqual( - len(serialized), - proto.MergeFromString(serialized)) - - # Check that the message parsed well. - extension_message1 = message_set_extensions_pb2.TestMessageSetExtension1 - extension1 = extension_message1.message_set_extension - self.assertEqual(12345, proto.Extensions[extension1].i) - - def testUnknownFields(self): - proto = unittest_pb2.TestAllTypes() - test_util.SetAllFields(proto) - - serialized = proto.SerializeToString() - - # The empty message should be parsable with all of the fields - # unknown. - proto2 = unittest_pb2.TestEmptyMessage() - - # Parsing this message should succeed. - self.assertEqual( - len(serialized), - proto2.MergeFromString(serialized)) - - # Now test with a int64 field set. - proto = unittest_pb2.TestAllTypes() - proto.optional_int64 = 0x0fffffffffffffff - serialized = proto.SerializeToString() - # The empty message should be parsable with all of the fields - # unknown. - proto2 = unittest_pb2.TestEmptyMessage() - # Parsing this message should succeed. - self.assertEqual( - len(serialized), - proto2.MergeFromString(serialized)) - - def _CheckRaises(self, exc_class, callable_obj, exception): - """This method checks if the exception type and message are as expected.""" - try: - callable_obj() - except exc_class as ex: - # Check if the exception message is the right one. - self.assertEqual(exception, str(ex)) - return - else: - raise self.failureException('%s not raised' % str(exc_class)) - - def testSerializeUninitialized(self): - proto = unittest_pb2.TestRequired() - self._CheckRaises( - message.EncodeError, - proto.SerializeToString, - 'Message protobuf_unittest.TestRequired is missing required fields: ' - 'a,b,c') - # Shouldn't raise exceptions. - partial = proto.SerializePartialToString() - - proto2 = unittest_pb2.TestRequired() - self.assertFalse(proto2.HasField('a')) - # proto2 ParseFromString does not check that required fields are set. - proto2.ParseFromString(partial) - self.assertFalse(proto2.HasField('a')) - - proto.a = 1 - self._CheckRaises( - message.EncodeError, - proto.SerializeToString, - 'Message protobuf_unittest.TestRequired is missing required fields: b,c') - # Shouldn't raise exceptions. - partial = proto.SerializePartialToString() - - proto.b = 2 - self._CheckRaises( - message.EncodeError, - proto.SerializeToString, - 'Message protobuf_unittest.TestRequired is missing required fields: c') - # Shouldn't raise exceptions. - partial = proto.SerializePartialToString() - - proto.c = 3 - serialized = proto.SerializeToString() - # Shouldn't raise exceptions. - partial = proto.SerializePartialToString() - - proto2 = unittest_pb2.TestRequired() - self.assertEqual( - len(serialized), - proto2.MergeFromString(serialized)) - self.assertEqual(1, proto2.a) - self.assertEqual(2, proto2.b) - self.assertEqual(3, proto2.c) - self.assertEqual( - len(partial), - proto2.MergeFromString(partial)) - self.assertEqual(1, proto2.a) - self.assertEqual(2, proto2.b) - self.assertEqual(3, proto2.c) - - def testSerializeUninitializedSubMessage(self): - proto = unittest_pb2.TestRequiredForeign() - - # Sub-message doesn't exist yet, so this succeeds. - proto.SerializeToString() - - proto.optional_message.a = 1 - self._CheckRaises( - message.EncodeError, - proto.SerializeToString, - 'Message protobuf_unittest.TestRequiredForeign ' - 'is missing required fields: ' - 'optional_message.b,optional_message.c') - - proto.optional_message.b = 2 - proto.optional_message.c = 3 - proto.SerializeToString() - - proto.repeated_message.add().a = 1 - proto.repeated_message.add().b = 2 - self._CheckRaises( - message.EncodeError, - proto.SerializeToString, - 'Message protobuf_unittest.TestRequiredForeign is missing required fields: ' - 'repeated_message[0].b,repeated_message[0].c,' - 'repeated_message[1].a,repeated_message[1].c') - - proto.repeated_message[0].b = 2 - proto.repeated_message[0].c = 3 - proto.repeated_message[1].a = 1 - proto.repeated_message[1].c = 3 - proto.SerializeToString() - - def testSerializeAllPackedFields(self): - first_proto = unittest_pb2.TestPackedTypes() - second_proto = unittest_pb2.TestPackedTypes() - test_util.SetAllPackedFields(first_proto) - serialized = first_proto.SerializeToString() - self.assertEqual(first_proto.ByteSize(), len(serialized)) - bytes_read = second_proto.MergeFromString(serialized) - self.assertEqual(second_proto.ByteSize(), bytes_read) - self.assertEqual(first_proto, second_proto) - - def testSerializeAllPackedExtensions(self): - first_proto = unittest_pb2.TestPackedExtensions() - second_proto = unittest_pb2.TestPackedExtensions() - test_util.SetAllPackedExtensions(first_proto) - serialized = first_proto.SerializeToString() - bytes_read = second_proto.MergeFromString(serialized) - self.assertEqual(second_proto.ByteSize(), bytes_read) - self.assertEqual(first_proto, second_proto) - - def testMergePackedFromStringWhenSomeFieldsAlreadySet(self): - first_proto = unittest_pb2.TestPackedTypes() - first_proto.packed_int32.extend([1, 2]) - first_proto.packed_double.append(3.0) - serialized = first_proto.SerializeToString() - - second_proto = unittest_pb2.TestPackedTypes() - second_proto.packed_int32.append(3) - second_proto.packed_double.extend([1.0, 2.0]) - second_proto.packed_sint32.append(4) - - self.assertEqual( - len(serialized), - second_proto.MergeFromString(serialized)) - self.assertEqual([3, 1, 2], second_proto.packed_int32) - self.assertEqual([1.0, 2.0, 3.0], second_proto.packed_double) - self.assertEqual([4], second_proto.packed_sint32) - - def testPackedFieldsWireFormat(self): - proto = unittest_pb2.TestPackedTypes() - proto.packed_int32.extend([1, 2, 150, 3]) # 1 + 1 + 2 + 1 bytes - proto.packed_double.extend([1.0, 1000.0]) # 8 + 8 bytes - proto.packed_float.append(2.0) # 4 bytes, will be before double - serialized = proto.SerializeToString() - self.assertEqual(proto.ByteSize(), len(serialized)) - d = _MiniDecoder(serialized) - ReadTag = d.ReadFieldNumberAndWireType - self.assertEqual((90, wire_format.WIRETYPE_LENGTH_DELIMITED), ReadTag()) - self.assertEqual(1+1+1+2, d.ReadInt32()) - self.assertEqual(1, d.ReadInt32()) - self.assertEqual(2, d.ReadInt32()) - self.assertEqual(150, d.ReadInt32()) - self.assertEqual(3, d.ReadInt32()) - self.assertEqual((100, wire_format.WIRETYPE_LENGTH_DELIMITED), ReadTag()) - self.assertEqual(4, d.ReadInt32()) - self.assertEqual(2.0, d.ReadFloat()) - self.assertEqual((101, wire_format.WIRETYPE_LENGTH_DELIMITED), ReadTag()) - self.assertEqual(8+8, d.ReadInt32()) - self.assertEqual(1.0, d.ReadDouble()) - self.assertEqual(1000.0, d.ReadDouble()) - self.assertTrue(d.EndOfStream()) - - def testParsePackedFromUnpacked(self): - unpacked = unittest_pb2.TestUnpackedTypes() - test_util.SetAllUnpackedFields(unpacked) - packed = unittest_pb2.TestPackedTypes() - serialized = unpacked.SerializeToString() - self.assertEqual( - len(serialized), - packed.MergeFromString(serialized)) - expected = unittest_pb2.TestPackedTypes() - test_util.SetAllPackedFields(expected) - self.assertEqual(expected, packed) - - def testParseUnpackedFromPacked(self): - packed = unittest_pb2.TestPackedTypes() - test_util.SetAllPackedFields(packed) - unpacked = unittest_pb2.TestUnpackedTypes() - serialized = packed.SerializeToString() - self.assertEqual( - len(serialized), - unpacked.MergeFromString(serialized)) - expected = unittest_pb2.TestUnpackedTypes() - test_util.SetAllUnpackedFields(expected) - self.assertEqual(expected, unpacked) - - def testFieldNumbers(self): - proto = unittest_pb2.TestAllTypes() - self.assertEqual(unittest_pb2.TestAllTypes.NestedMessage.BB_FIELD_NUMBER, 1) - self.assertEqual(unittest_pb2.TestAllTypes.OPTIONAL_INT32_FIELD_NUMBER, 1) - self.assertEqual(unittest_pb2.TestAllTypes.OPTIONALGROUP_FIELD_NUMBER, 16) - self.assertEqual( - unittest_pb2.TestAllTypes.OPTIONAL_NESTED_MESSAGE_FIELD_NUMBER, 18) - self.assertEqual( - unittest_pb2.TestAllTypes.OPTIONAL_NESTED_ENUM_FIELD_NUMBER, 21) - self.assertEqual(unittest_pb2.TestAllTypes.REPEATED_INT32_FIELD_NUMBER, 31) - self.assertEqual(unittest_pb2.TestAllTypes.REPEATEDGROUP_FIELD_NUMBER, 46) - self.assertEqual( - unittest_pb2.TestAllTypes.REPEATED_NESTED_MESSAGE_FIELD_NUMBER, 48) - self.assertEqual( - unittest_pb2.TestAllTypes.REPEATED_NESTED_ENUM_FIELD_NUMBER, 51) - - def testExtensionFieldNumbers(self): - self.assertEqual(unittest_pb2.TestRequired.single.number, 1000) - self.assertEqual(unittest_pb2.TestRequired.SINGLE_FIELD_NUMBER, 1000) - self.assertEqual(unittest_pb2.TestRequired.multi.number, 1001) - self.assertEqual(unittest_pb2.TestRequired.MULTI_FIELD_NUMBER, 1001) - self.assertEqual(unittest_pb2.optional_int32_extension.number, 1) - self.assertEqual(unittest_pb2.OPTIONAL_INT32_EXTENSION_FIELD_NUMBER, 1) - self.assertEqual(unittest_pb2.optionalgroup_extension.number, 16) - self.assertEqual(unittest_pb2.OPTIONALGROUP_EXTENSION_FIELD_NUMBER, 16) - self.assertEqual(unittest_pb2.optional_nested_message_extension.number, 18) - self.assertEqual( - unittest_pb2.OPTIONAL_NESTED_MESSAGE_EXTENSION_FIELD_NUMBER, 18) - self.assertEqual(unittest_pb2.optional_nested_enum_extension.number, 21) - self.assertEqual(unittest_pb2.OPTIONAL_NESTED_ENUM_EXTENSION_FIELD_NUMBER, - 21) - self.assertEqual(unittest_pb2.repeated_int32_extension.number, 31) - self.assertEqual(unittest_pb2.REPEATED_INT32_EXTENSION_FIELD_NUMBER, 31) - self.assertEqual(unittest_pb2.repeatedgroup_extension.number, 46) - self.assertEqual(unittest_pb2.REPEATEDGROUP_EXTENSION_FIELD_NUMBER, 46) - self.assertEqual(unittest_pb2.repeated_nested_message_extension.number, 48) - self.assertEqual( - unittest_pb2.REPEATED_NESTED_MESSAGE_EXTENSION_FIELD_NUMBER, 48) - self.assertEqual(unittest_pb2.repeated_nested_enum_extension.number, 51) - self.assertEqual(unittest_pb2.REPEATED_NESTED_ENUM_EXTENSION_FIELD_NUMBER, - 51) - - def testFieldProperties(self): - cls = unittest_pb2.TestAllTypes - self.assertIs(cls.optional_int32.DESCRIPTOR, - cls.DESCRIPTOR.fields_by_name['optional_int32']) - self.assertEqual(cls.OPTIONAL_INT32_FIELD_NUMBER, - cls.optional_int32.DESCRIPTOR.number) - self.assertIs(cls.optional_nested_message.DESCRIPTOR, - cls.DESCRIPTOR.fields_by_name['optional_nested_message']) - self.assertEqual(cls.OPTIONAL_NESTED_MESSAGE_FIELD_NUMBER, - cls.optional_nested_message.DESCRIPTOR.number) - self.assertIs(cls.repeated_int32.DESCRIPTOR, - cls.DESCRIPTOR.fields_by_name['repeated_int32']) - self.assertEqual(cls.REPEATED_INT32_FIELD_NUMBER, - cls.repeated_int32.DESCRIPTOR.number) - - def testFieldDataDescriptor(self): - msg = unittest_pb2.TestAllTypes() - msg.optional_int32 = 42 - self.assertEqual(unittest_pb2.TestAllTypes.optional_int32.__get__(msg), 42) - unittest_pb2.TestAllTypes.optional_int32.__set__(msg, 25) - self.assertEqual(msg.optional_int32, 25) - with self.assertRaises(AttributeError): - del msg.optional_int32 - try: - unittest_pb2.ForeignMessage.c.__get__(msg) - except TypeError: - pass # The cpp implementation cannot mix fields from other messages. - # This test exercises a specific check that avoids a crash. - else: - pass # The python implementation allows fields from other messages. - # This is useless, but works. - - def testInitKwargs(self): - proto = unittest_pb2.TestAllTypes( - optional_int32=1, - optional_string='foo', - optional_bool=True, - optional_bytes=b'bar', - optional_nested_message=unittest_pb2.TestAllTypes.NestedMessage(bb=1), - optional_foreign_message=unittest_pb2.ForeignMessage(c=1), - optional_nested_enum=unittest_pb2.TestAllTypes.FOO, - optional_foreign_enum=unittest_pb2.FOREIGN_FOO, - repeated_int32=[1, 2, 3]) - self.assertTrue(proto.IsInitialized()) - self.assertTrue(proto.HasField('optional_int32')) - self.assertTrue(proto.HasField('optional_string')) - self.assertTrue(proto.HasField('optional_bool')) - self.assertTrue(proto.HasField('optional_bytes')) - self.assertTrue(proto.HasField('optional_nested_message')) - self.assertTrue(proto.HasField('optional_foreign_message')) - self.assertTrue(proto.HasField('optional_nested_enum')) - self.assertTrue(proto.HasField('optional_foreign_enum')) - self.assertEqual(1, proto.optional_int32) - self.assertEqual('foo', proto.optional_string) - self.assertEqual(True, proto.optional_bool) - self.assertEqual(b'bar', proto.optional_bytes) - self.assertEqual(1, proto.optional_nested_message.bb) - self.assertEqual(1, proto.optional_foreign_message.c) - self.assertEqual(unittest_pb2.TestAllTypes.FOO, - proto.optional_nested_enum) - self.assertEqual(unittest_pb2.FOREIGN_FOO, proto.optional_foreign_enum) - self.assertEqual([1, 2, 3], proto.repeated_int32) - - def testInitArgsUnknownFieldName(self): - def InitalizeEmptyMessageWithExtraKeywordArg(): - unused_proto = unittest_pb2.TestEmptyMessage(unknown='unknown') - self._CheckRaises( - ValueError, - InitalizeEmptyMessageWithExtraKeywordArg, - 'Protocol message TestEmptyMessage has no "unknown" field.') - - def testInitRequiredKwargs(self): - proto = unittest_pb2.TestRequired(a=1, b=1, c=1) - self.assertTrue(proto.IsInitialized()) - self.assertTrue(proto.HasField('a')) - self.assertTrue(proto.HasField('b')) - self.assertTrue(proto.HasField('c')) - self.assertFalse(proto.HasField('dummy2')) - self.assertEqual(1, proto.a) - self.assertEqual(1, proto.b) - self.assertEqual(1, proto.c) - - def testInitRequiredForeignKwargs(self): - proto = unittest_pb2.TestRequiredForeign( - optional_message=unittest_pb2.TestRequired(a=1, b=1, c=1)) - self.assertTrue(proto.IsInitialized()) - self.assertTrue(proto.HasField('optional_message')) - self.assertTrue(proto.optional_message.IsInitialized()) - self.assertTrue(proto.optional_message.HasField('a')) - self.assertTrue(proto.optional_message.HasField('b')) - self.assertTrue(proto.optional_message.HasField('c')) - self.assertFalse(proto.optional_message.HasField('dummy2')) - self.assertEqual(unittest_pb2.TestRequired(a=1, b=1, c=1), - proto.optional_message) - self.assertEqual(1, proto.optional_message.a) - self.assertEqual(1, proto.optional_message.b) - self.assertEqual(1, proto.optional_message.c) - - def testInitRepeatedKwargs(self): - proto = unittest_pb2.TestAllTypes(repeated_int32=[1, 2, 3]) - self.assertTrue(proto.IsInitialized()) - self.assertEqual(1, proto.repeated_int32[0]) - self.assertEqual(2, proto.repeated_int32[1]) - self.assertEqual(3, proto.repeated_int32[2]) - - -@testing_refleaks.TestCase -class OptionsTest(unittest.TestCase): - - def testMessageOptions(self): - proto = message_set_extensions_pb2.TestMessageSet() - self.assertEqual(True, - proto.DESCRIPTOR.GetOptions().message_set_wire_format) - proto = unittest_pb2.TestAllTypes() - self.assertEqual(False, - proto.DESCRIPTOR.GetOptions().message_set_wire_format) - - def testPackedOptions(self): - proto = unittest_pb2.TestAllTypes() - proto.optional_int32 = 1 - proto.optional_double = 3.0 - for field_descriptor, _ in proto.ListFields(): - self.assertEqual(False, field_descriptor.GetOptions().packed) - - proto = unittest_pb2.TestPackedTypes() - proto.packed_int32.append(1) - proto.packed_double.append(3.0) - for field_descriptor, _ in proto.ListFields(): - self.assertEqual(True, field_descriptor.GetOptions().packed) - self.assertEqual(descriptor.FieldDescriptor.LABEL_REPEATED, - field_descriptor.label) - - - -@testing_refleaks.TestCase -class ClassAPITest(unittest.TestCase): - - @unittest.skipIf( - api_implementation.Type() != 'python', - 'C++ implementation requires a call to MakeDescriptor()') - @testing_refleaks.SkipReferenceLeakChecker('MakeClass is not repeatable') - def testMakeClassWithNestedDescriptor(self): - leaf_desc = descriptor.Descriptor( - 'leaf', 'package.parent.child.leaf', '', - containing_type=None, fields=[], - nested_types=[], enum_types=[], - extensions=[], - # pylint: disable=protected-access - create_key=descriptor._internal_create_key) - child_desc = descriptor.Descriptor( - 'child', 'package.parent.child', '', - containing_type=None, fields=[], - nested_types=[leaf_desc], enum_types=[], - extensions=[], - # pylint: disable=protected-access - create_key=descriptor._internal_create_key) - sibling_desc = descriptor.Descriptor( - 'sibling', 'package.parent.sibling', - '', containing_type=None, fields=[], - nested_types=[], enum_types=[], - extensions=[], - # pylint: disable=protected-access - create_key=descriptor._internal_create_key) - parent_desc = descriptor.Descriptor( - 'parent', 'package.parent', '', - containing_type=None, fields=[], - nested_types=[child_desc, sibling_desc], - enum_types=[], extensions=[], - # pylint: disable=protected-access - create_key=descriptor._internal_create_key) - reflection.MakeClass(parent_desc) - - def _GetSerializedFileDescriptor(self, name): - """Get a serialized representation of a test FileDescriptorProto. - - Args: - name: All calls to this must use a unique message name, to avoid - collisions in the cpp descriptor pool. - Returns: - A string containing the serialized form of a test FileDescriptorProto. - """ - file_descriptor_str = ( - 'message_type {' - ' name: "' + name + '"' - ' field {' - ' name: "flat"' - ' number: 1' - ' label: LABEL_REPEATED' - ' type: TYPE_UINT32' - ' }' - ' field {' - ' name: "bar"' - ' number: 2' - ' label: LABEL_OPTIONAL' - ' type: TYPE_MESSAGE' - ' type_name: "Bar"' - ' }' - ' nested_type {' - ' name: "Bar"' - ' field {' - ' name: "baz"' - ' number: 3' - ' label: LABEL_OPTIONAL' - ' type: TYPE_MESSAGE' - ' type_name: "Baz"' - ' }' - ' nested_type {' - ' name: "Baz"' - ' enum_type {' - ' name: "deep_enum"' - ' value {' - ' name: "VALUE_A"' - ' number: 0' - ' }' - ' }' - ' field {' - ' name: "deep"' - ' number: 4' - ' label: LABEL_OPTIONAL' - ' type: TYPE_UINT32' - ' }' - ' }' - ' }' - '}') - file_descriptor = descriptor_pb2.FileDescriptorProto() - text_format.Merge(file_descriptor_str, file_descriptor) - return file_descriptor.SerializeToString() - - @testing_refleaks.SkipReferenceLeakChecker('MakeDescriptor is not repeatable') - # This test can only run once; the second time, it raises errors about - # conflicting message descriptors. - def testParsingFlatClassWithExplicitClassDeclaration(self): - """Test that the generated class can parse a flat message.""" - # TODO(xiaofeng): This test fails with cpp implementation in the call - # of six.with_metaclass(). The other two callsites of with_metaclass - # in this file are both excluded from cpp test, so it might be expected - # to fail. Need someone more familiar with the python code to take a - # look at this. - if api_implementation.Type() != 'python': - return - file_descriptor = descriptor_pb2.FileDescriptorProto() - file_descriptor.ParseFromString(self._GetSerializedFileDescriptor('A')) - msg_descriptor = descriptor.MakeDescriptor( - file_descriptor.message_type[0]) - - class MessageClass( - message.Message, metaclass=reflection.GeneratedProtocolMessageType): - DESCRIPTOR = msg_descriptor - msg = MessageClass() - msg_str = ( - 'flat: 0 ' - 'flat: 1 ' - 'flat: 2 ') - text_format.Merge(msg_str, msg) - self.assertEqual(msg.flat, [0, 1, 2]) - - @testing_refleaks.SkipReferenceLeakChecker('MakeDescriptor is not repeatable') - def testParsingFlatClass(self): - """Test that the generated class can parse a flat message.""" - file_descriptor = descriptor_pb2.FileDescriptorProto() - file_descriptor.ParseFromString(self._GetSerializedFileDescriptor('B')) - msg_descriptor = descriptor.MakeDescriptor( - file_descriptor.message_type[0]) - msg_class = reflection.MakeClass(msg_descriptor) - msg = msg_class() - msg_str = ( - 'flat: 0 ' - 'flat: 1 ' - 'flat: 2 ') - text_format.Merge(msg_str, msg) - self.assertEqual(msg.flat, [0, 1, 2]) - - @testing_refleaks.SkipReferenceLeakChecker('MakeDescriptor is not repeatable') - def testParsingNestedClass(self): - """Test that the generated class can parse a nested message.""" - file_descriptor = descriptor_pb2.FileDescriptorProto() - file_descriptor.ParseFromString(self._GetSerializedFileDescriptor('C')) - msg_descriptor = descriptor.MakeDescriptor( - file_descriptor.message_type[0]) - msg_class = reflection.MakeClass(msg_descriptor) - msg = msg_class() - msg_str = ( - 'bar {' - ' baz {' - ' deep: 4' - ' }' - '}') - text_format.Merge(msg_str, msg) - self.assertEqual(msg.bar.baz.deep, 4) - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/service_reflection_test.py b/ext/protobuf/Python/google/protobuf/internal/service_reflection_test.py deleted file mode 100644 index 8e72213b5..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/service_reflection_test.py +++ /dev/null @@ -1,139 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.internal.service_reflection.""" - -__author__ = 'petar@google.com (Petar Petrov)' - - -import unittest - -from google.protobuf import unittest_pb2 -from google.protobuf import service_reflection -from google.protobuf import service - - -class FooUnitTest(unittest.TestCase): - - def testService(self): - class MockRpcChannel(service.RpcChannel): - def CallMethod(self, method, controller, request, response, callback): - self.method = method - self.controller = controller - self.request = request - callback(response) - - class MockRpcController(service.RpcController): - def SetFailed(self, msg): - self.failure_message = msg - - self.callback_response = None - - class MyService(unittest_pb2.TestService): - pass - - self.callback_response = None - - def MyCallback(response): - self.callback_response = response - - rpc_controller = MockRpcController() - channel = MockRpcChannel() - srvc = MyService() - srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback) - self.assertEqual('Method Foo not implemented.', - rpc_controller.failure_message) - self.assertEqual(None, self.callback_response) - - rpc_controller.failure_message = None - - service_descriptor = unittest_pb2.TestService.GetDescriptor() - srvc.CallMethod(service_descriptor.methods[1], rpc_controller, - unittest_pb2.BarRequest(), MyCallback) - self.assertTrue(srvc.GetRequestClass(service_descriptor.methods[1]) is - unittest_pb2.BarRequest) - self.assertTrue(srvc.GetResponseClass(service_descriptor.methods[1]) is - unittest_pb2.BarResponse) - self.assertEqual('Method Bar not implemented.', - rpc_controller.failure_message) - self.assertEqual(None, self.callback_response) - - class MyServiceImpl(unittest_pb2.TestService): - def Foo(self, rpc_controller, request, done): - self.foo_called = True - def Bar(self, rpc_controller, request, done): - self.bar_called = True - - srvc = MyServiceImpl() - rpc_controller.failure_message = None - srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback) - self.assertEqual(None, rpc_controller.failure_message) - self.assertEqual(True, srvc.foo_called) - - rpc_controller.failure_message = None - srvc.CallMethod(service_descriptor.methods[1], rpc_controller, - unittest_pb2.BarRequest(), MyCallback) - self.assertEqual(None, rpc_controller.failure_message) - self.assertEqual(True, srvc.bar_called) - - def testServiceStub(self): - class MockRpcChannel(service.RpcChannel): - def CallMethod(self, method, controller, request, - response_class, callback): - self.method = method - self.controller = controller - self.request = request - callback(response_class()) - - self.callback_response = None - - def MyCallback(response): - self.callback_response = response - - channel = MockRpcChannel() - stub = unittest_pb2.TestService_Stub(channel) - rpc_controller = 'controller' - request = 'request' - - # GetDescriptor now static, still works as instance method for compatibility - self.assertEqual(unittest_pb2.TestService_Stub.GetDescriptor(), - stub.GetDescriptor()) - - # Invoke method. - stub.Foo(rpc_controller, request, MyCallback) - - self.assertIsInstance(self.callback_response, unittest_pb2.FooResponse) - self.assertEqual(request, channel.request) - self.assertEqual(rpc_controller, channel.controller) - self.assertEqual(stub.GetDescriptor().methods[0], channel.method) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/symbol_database_test.py b/ext/protobuf/Python/google/protobuf/internal/symbol_database_test.py deleted file mode 100644 index 4fdc4ee9c..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/symbol_database_test.py +++ /dev/null @@ -1,133 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.symbol_database.""" - -import unittest - -from google.protobuf import unittest_pb2 -from google.protobuf import descriptor -from google.protobuf import descriptor_pool -from google.protobuf import symbol_database - - -class SymbolDatabaseTest(unittest.TestCase): - - def _Database(self): - if descriptor._USE_C_DESCRIPTORS: - # The C++ implementation does not allow mixing descriptors from - # different pools. - db = symbol_database.SymbolDatabase(pool=descriptor_pool.Default()) - else: - db = symbol_database.SymbolDatabase() - # Register representative types from unittest_pb2. - db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR) - db.RegisterMessage(unittest_pb2.TestAllTypes) - db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage) - db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup) - db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup) - db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) - db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) - db.RegisterServiceDescriptor(unittest_pb2._TESTSERVICE) - return db - - def testGetPrototype(self): - instance = self._Database().GetPrototype( - unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertTrue(instance is unittest_pb2.TestAllTypes) - - def testGetMessages(self): - messages = self._Database().GetMessages( - ['google/protobuf/unittest.proto']) - self.assertTrue( - unittest_pb2.TestAllTypes is - messages['protobuf_unittest.TestAllTypes']) - - def testGetSymbol(self): - self.assertEqual( - unittest_pb2.TestAllTypes, self._Database().GetSymbol( - 'protobuf_unittest.TestAllTypes')) - self.assertEqual( - unittest_pb2.TestAllTypes.NestedMessage, self._Database().GetSymbol( - 'protobuf_unittest.TestAllTypes.NestedMessage')) - self.assertEqual( - unittest_pb2.TestAllTypes.OptionalGroup, self._Database().GetSymbol( - 'protobuf_unittest.TestAllTypes.OptionalGroup')) - self.assertEqual( - unittest_pb2.TestAllTypes.RepeatedGroup, self._Database().GetSymbol( - 'protobuf_unittest.TestAllTypes.RepeatedGroup')) - - def testEnums(self): - # Check registration of types in the pool. - self.assertEqual( - 'protobuf_unittest.ForeignEnum', - self._Database().pool.FindEnumTypeByName( - 'protobuf_unittest.ForeignEnum').full_name) - self.assertEqual( - 'protobuf_unittest.TestAllTypes.NestedEnum', - self._Database().pool.FindEnumTypeByName( - 'protobuf_unittest.TestAllTypes.NestedEnum').full_name) - - def testFindMessageTypeByName(self): - self.assertEqual( - 'protobuf_unittest.TestAllTypes', - self._Database().pool.FindMessageTypeByName( - 'protobuf_unittest.TestAllTypes').full_name) - self.assertEqual( - 'protobuf_unittest.TestAllTypes.NestedMessage', - self._Database().pool.FindMessageTypeByName( - 'protobuf_unittest.TestAllTypes.NestedMessage').full_name) - - def testFindServiceByName(self): - self.assertEqual( - 'protobuf_unittest.TestService', - self._Database().pool.FindServiceByName( - 'protobuf_unittest.TestService').full_name) - - def testFindFileContainingSymbol(self): - # Lookup based on either enum or message. - self.assertEqual( - 'google/protobuf/unittest.proto', - self._Database().pool.FindFileContainingSymbol( - 'protobuf_unittest.TestAllTypes.NestedEnum').name) - self.assertEqual( - 'google/protobuf/unittest.proto', - self._Database().pool.FindFileContainingSymbol( - 'protobuf_unittest.TestAllTypes').name) - - def testFindFileByName(self): - self.assertEqual( - 'google/protobuf/unittest.proto', - self._Database().pool.FindFileByName( - 'google/protobuf/unittest.proto').name) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/test_util.py b/ext/protobuf/Python/google/protobuf/internal/test_util.py deleted file mode 100644 index 32fb8fd27..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/test_util.py +++ /dev/null @@ -1,878 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Utilities for Python proto2 tests. - -This is intentionally modeled on C++ code in -//google/protobuf/test_util.*. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - -import numbers -import operator -import os.path - -from google.protobuf import unittest_import_pb2 -from google.protobuf import unittest_pb2 - -try: - long # Python 2 -except NameError: - long = int # Python 3 - - -# Tests whether the given TestAllTypes message is proto2 or not. -# This is used to gate several fields/features that only exist -# for the proto2 version of the message. -def IsProto2(message): - return message.DESCRIPTOR.syntax == "proto2" - - -def SetAllNonLazyFields(message): - """Sets every non-lazy field in the message to a unique value. - - Args: - message: A TestAllTypes instance. - """ - - # - # Optional fields. - # - - message.optional_int32 = 101 - message.optional_int64 = 102 - message.optional_uint32 = 103 - message.optional_uint64 = 104 - message.optional_sint32 = 105 - message.optional_sint64 = 106 - message.optional_fixed32 = 107 - message.optional_fixed64 = 108 - message.optional_sfixed32 = 109 - message.optional_sfixed64 = 110 - message.optional_float = 111 - message.optional_double = 112 - message.optional_bool = True - message.optional_string = u'115' - message.optional_bytes = b'116' - - if IsProto2(message): - message.optionalgroup.a = 117 - message.optional_nested_message.bb = 118 - message.optional_foreign_message.c = 119 - message.optional_import_message.d = 120 - message.optional_public_import_message.e = 126 - - message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ - message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ - if IsProto2(message): - message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ - - message.optional_string_piece = u'124' - message.optional_cord = u'125' - - # - # Repeated fields. - # - - message.repeated_int32.append(201) - message.repeated_int64.append(202) - message.repeated_uint32.append(203) - message.repeated_uint64.append(204) - message.repeated_sint32.append(205) - message.repeated_sint64.append(206) - message.repeated_fixed32.append(207) - message.repeated_fixed64.append(208) - message.repeated_sfixed32.append(209) - message.repeated_sfixed64.append(210) - message.repeated_float.append(211) - message.repeated_double.append(212) - message.repeated_bool.append(True) - message.repeated_string.append(u'215') - message.repeated_bytes.append(b'216') - - if IsProto2(message): - message.repeatedgroup.add().a = 217 - message.repeated_nested_message.add().bb = 218 - message.repeated_foreign_message.add().c = 219 - message.repeated_import_message.add().d = 220 - message.repeated_lazy_message.add().bb = 227 - - message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR) - message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR) - if IsProto2(message): - message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR) - - message.repeated_string_piece.append(u'224') - message.repeated_cord.append(u'225') - - # Add a second one of each field and set value by index. - message.repeated_int32.append(0) - message.repeated_int64.append(0) - message.repeated_uint32.append(0) - message.repeated_uint64.append(0) - message.repeated_sint32.append(0) - message.repeated_sint64.append(0) - message.repeated_fixed32.append(0) - message.repeated_fixed64.append(0) - message.repeated_sfixed32.append(0) - message.repeated_sfixed64.append(0) - message.repeated_float.append(0) - message.repeated_double.append(0) - message.repeated_bool.append(True) - message.repeated_string.append(u'0') - message.repeated_bytes.append(b'0') - message.repeated_int32[1] = 301 - message.repeated_int64[1] = 302 - message.repeated_uint32[1] = 303 - message.repeated_uint64[1] = 304 - message.repeated_sint32[1] = 305 - message.repeated_sint64[1] = 306 - message.repeated_fixed32[1] = 307 - message.repeated_fixed64[1] = 308 - message.repeated_sfixed32[1] = 309 - message.repeated_sfixed64[1] = 310 - message.repeated_float[1] = 311 - message.repeated_double[1] = 312 - message.repeated_bool[1] = False - message.repeated_string[1] = u'315' - message.repeated_bytes[1] = b'316' - - if IsProto2(message): - message.repeatedgroup.add().a = 317 - message.repeated_nested_message.add().bb = 318 - message.repeated_foreign_message.add().c = 319 - message.repeated_import_message.add().d = 320 - message.repeated_lazy_message.add().bb = 327 - - message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR) - message.repeated_nested_enum[1] = unittest_pb2.TestAllTypes.BAZ - message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ) - if IsProto2(message): - message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ) - - message.repeated_string_piece.append(u'324') - message.repeated_cord.append(u'325') - - # - # Fields that have defaults. - # - - if IsProto2(message): - message.default_int32 = 401 - message.default_int64 = 402 - message.default_uint32 = 403 - message.default_uint64 = 404 - message.default_sint32 = 405 - message.default_sint64 = 406 - message.default_fixed32 = 407 - message.default_fixed64 = 408 - message.default_sfixed32 = 409 - message.default_sfixed64 = 410 - message.default_float = 411 - message.default_double = 412 - message.default_bool = False - message.default_string = '415' - message.default_bytes = b'416' - - message.default_nested_enum = unittest_pb2.TestAllTypes.FOO - message.default_foreign_enum = unittest_pb2.FOREIGN_FOO - message.default_import_enum = unittest_import_pb2.IMPORT_FOO - - message.default_string_piece = '424' - message.default_cord = '425' - - message.oneof_uint32 = 601 - message.oneof_nested_message.bb = 602 - message.oneof_string = '603' - message.oneof_bytes = b'604' - - -def SetAllFields(message): - SetAllNonLazyFields(message) - message.optional_lazy_message.bb = 127 - message.optional_unverified_lazy_message.bb = 128 - - -def SetAllExtensions(message): - """Sets every extension in the message to a unique value. - - Args: - message: A unittest_pb2.TestAllExtensions instance. - """ - - extensions = message.Extensions - pb2 = unittest_pb2 - import_pb2 = unittest_import_pb2 - - # - # Optional fields. - # - - extensions[pb2.optional_int32_extension] = 101 - extensions[pb2.optional_int64_extension] = 102 - extensions[pb2.optional_uint32_extension] = 103 - extensions[pb2.optional_uint64_extension] = 104 - extensions[pb2.optional_sint32_extension] = 105 - extensions[pb2.optional_sint64_extension] = 106 - extensions[pb2.optional_fixed32_extension] = 107 - extensions[pb2.optional_fixed64_extension] = 108 - extensions[pb2.optional_sfixed32_extension] = 109 - extensions[pb2.optional_sfixed64_extension] = 110 - extensions[pb2.optional_float_extension] = 111 - extensions[pb2.optional_double_extension] = 112 - extensions[pb2.optional_bool_extension] = True - extensions[pb2.optional_string_extension] = u'115' - extensions[pb2.optional_bytes_extension] = b'116' - - extensions[pb2.optionalgroup_extension].a = 117 - extensions[pb2.optional_nested_message_extension].bb = 118 - extensions[pb2.optional_foreign_message_extension].c = 119 - extensions[pb2.optional_import_message_extension].d = 120 - extensions[pb2.optional_public_import_message_extension].e = 126 - extensions[pb2.optional_lazy_message_extension].bb = 127 - extensions[pb2.optional_unverified_lazy_message_extension].bb = 128 - - extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ - extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ - extensions[pb2.optional_foreign_enum_extension] = pb2.FOREIGN_BAZ - extensions[pb2.optional_import_enum_extension] = import_pb2.IMPORT_BAZ - - extensions[pb2.optional_string_piece_extension] = u'124' - extensions[pb2.optional_cord_extension] = u'125' - - # - # Repeated fields. - # - - extensions[pb2.repeated_int32_extension].append(201) - extensions[pb2.repeated_int64_extension].append(202) - extensions[pb2.repeated_uint32_extension].append(203) - extensions[pb2.repeated_uint64_extension].append(204) - extensions[pb2.repeated_sint32_extension].append(205) - extensions[pb2.repeated_sint64_extension].append(206) - extensions[pb2.repeated_fixed32_extension].append(207) - extensions[pb2.repeated_fixed64_extension].append(208) - extensions[pb2.repeated_sfixed32_extension].append(209) - extensions[pb2.repeated_sfixed64_extension].append(210) - extensions[pb2.repeated_float_extension].append(211) - extensions[pb2.repeated_double_extension].append(212) - extensions[pb2.repeated_bool_extension].append(True) - extensions[pb2.repeated_string_extension].append(u'215') - extensions[pb2.repeated_bytes_extension].append(b'216') - - extensions[pb2.repeatedgroup_extension].add().a = 217 - extensions[pb2.repeated_nested_message_extension].add().bb = 218 - extensions[pb2.repeated_foreign_message_extension].add().c = 219 - extensions[pb2.repeated_import_message_extension].add().d = 220 - extensions[pb2.repeated_lazy_message_extension].add().bb = 227 - - extensions[pb2.repeated_nested_enum_extension].append(pb2.TestAllTypes.BAR) - extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAR) - extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAR) - - extensions[pb2.repeated_string_piece_extension].append(u'224') - extensions[pb2.repeated_cord_extension].append(u'225') - - # Append a second one of each field. - extensions[pb2.repeated_int32_extension].append(301) - extensions[pb2.repeated_int64_extension].append(302) - extensions[pb2.repeated_uint32_extension].append(303) - extensions[pb2.repeated_uint64_extension].append(304) - extensions[pb2.repeated_sint32_extension].append(305) - extensions[pb2.repeated_sint64_extension].append(306) - extensions[pb2.repeated_fixed32_extension].append(307) - extensions[pb2.repeated_fixed64_extension].append(308) - extensions[pb2.repeated_sfixed32_extension].append(309) - extensions[pb2.repeated_sfixed64_extension].append(310) - extensions[pb2.repeated_float_extension].append(311) - extensions[pb2.repeated_double_extension].append(312) - extensions[pb2.repeated_bool_extension].append(False) - extensions[pb2.repeated_string_extension].append(u'315') - extensions[pb2.repeated_bytes_extension].append(b'316') - - extensions[pb2.repeatedgroup_extension].add().a = 317 - extensions[pb2.repeated_nested_message_extension].add().bb = 318 - extensions[pb2.repeated_foreign_message_extension].add().c = 319 - extensions[pb2.repeated_import_message_extension].add().d = 320 - extensions[pb2.repeated_lazy_message_extension].add().bb = 327 - - extensions[pb2.repeated_nested_enum_extension].append(pb2.TestAllTypes.BAZ) - extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAZ) - extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAZ) - - extensions[pb2.repeated_string_piece_extension].append(u'324') - extensions[pb2.repeated_cord_extension].append(u'325') - - # - # Fields with defaults. - # - - extensions[pb2.default_int32_extension] = 401 - extensions[pb2.default_int64_extension] = 402 - extensions[pb2.default_uint32_extension] = 403 - extensions[pb2.default_uint64_extension] = 404 - extensions[pb2.default_sint32_extension] = 405 - extensions[pb2.default_sint64_extension] = 406 - extensions[pb2.default_fixed32_extension] = 407 - extensions[pb2.default_fixed64_extension] = 408 - extensions[pb2.default_sfixed32_extension] = 409 - extensions[pb2.default_sfixed64_extension] = 410 - extensions[pb2.default_float_extension] = 411 - extensions[pb2.default_double_extension] = 412 - extensions[pb2.default_bool_extension] = False - extensions[pb2.default_string_extension] = u'415' - extensions[pb2.default_bytes_extension] = b'416' - - extensions[pb2.default_nested_enum_extension] = pb2.TestAllTypes.FOO - extensions[pb2.default_foreign_enum_extension] = pb2.FOREIGN_FOO - extensions[pb2.default_import_enum_extension] = import_pb2.IMPORT_FOO - - extensions[pb2.default_string_piece_extension] = u'424' - extensions[pb2.default_cord_extension] = '425' - - extensions[pb2.oneof_uint32_extension] = 601 - extensions[pb2.oneof_nested_message_extension].bb = 602 - extensions[pb2.oneof_string_extension] = u'603' - extensions[pb2.oneof_bytes_extension] = b'604' - - -def SetAllFieldsAndExtensions(message): - """Sets every field and extension in the message to a unique value. - - Args: - message: A unittest_pb2.TestAllExtensions message. - """ - message.my_int = 1 - message.my_string = 'foo' - message.my_float = 1.0 - message.Extensions[unittest_pb2.my_extension_int] = 23 - message.Extensions[unittest_pb2.my_extension_string] = 'bar' - - -def ExpectAllFieldsAndExtensionsInOrder(serialized): - """Ensures that serialized is the serialization we expect for a message - filled with SetAllFieldsAndExtensions(). (Specifically, ensures that the - serialization is in canonical, tag-number order). - """ - my_extension_int = unittest_pb2.my_extension_int - my_extension_string = unittest_pb2.my_extension_string - expected_strings = [] - message = unittest_pb2.TestFieldOrderings() - message.my_int = 1 # Field 1. - expected_strings.append(message.SerializeToString()) - message.Clear() - message.Extensions[my_extension_int] = 23 # Field 5. - expected_strings.append(message.SerializeToString()) - message.Clear() - message.my_string = 'foo' # Field 11. - expected_strings.append(message.SerializeToString()) - message.Clear() - message.Extensions[my_extension_string] = 'bar' # Field 50. - expected_strings.append(message.SerializeToString()) - message.Clear() - message.my_float = 1.0 - expected_strings.append(message.SerializeToString()) - message.Clear() - expected = b''.join(expected_strings) - - if expected != serialized: - raise ValueError('Expected %r, found %r' % (expected, serialized)) - - -def ExpectAllFieldsSet(test_case, message): - """Check all fields for correct values have after Set*Fields() is called.""" - test_case.assertTrue(message.HasField('optional_int32')) - test_case.assertTrue(message.HasField('optional_int64')) - test_case.assertTrue(message.HasField('optional_uint32')) - test_case.assertTrue(message.HasField('optional_uint64')) - test_case.assertTrue(message.HasField('optional_sint32')) - test_case.assertTrue(message.HasField('optional_sint64')) - test_case.assertTrue(message.HasField('optional_fixed32')) - test_case.assertTrue(message.HasField('optional_fixed64')) - test_case.assertTrue(message.HasField('optional_sfixed32')) - test_case.assertTrue(message.HasField('optional_sfixed64')) - test_case.assertTrue(message.HasField('optional_float')) - test_case.assertTrue(message.HasField('optional_double')) - test_case.assertTrue(message.HasField('optional_bool')) - test_case.assertTrue(message.HasField('optional_string')) - test_case.assertTrue(message.HasField('optional_bytes')) - - if IsProto2(message): - test_case.assertTrue(message.HasField('optionalgroup')) - test_case.assertTrue(message.HasField('optional_nested_message')) - test_case.assertTrue(message.HasField('optional_foreign_message')) - test_case.assertTrue(message.HasField('optional_import_message')) - - test_case.assertTrue(message.optionalgroup.HasField('a')) - test_case.assertTrue(message.optional_nested_message.HasField('bb')) - test_case.assertTrue(message.optional_foreign_message.HasField('c')) - test_case.assertTrue(message.optional_import_message.HasField('d')) - - test_case.assertTrue(message.HasField('optional_nested_enum')) - test_case.assertTrue(message.HasField('optional_foreign_enum')) - if IsProto2(message): - test_case.assertTrue(message.HasField('optional_import_enum')) - - test_case.assertTrue(message.HasField('optional_string_piece')) - test_case.assertTrue(message.HasField('optional_cord')) - - test_case.assertEqual(101, message.optional_int32) - test_case.assertEqual(102, message.optional_int64) - test_case.assertEqual(103, message.optional_uint32) - test_case.assertEqual(104, message.optional_uint64) - test_case.assertEqual(105, message.optional_sint32) - test_case.assertEqual(106, message.optional_sint64) - test_case.assertEqual(107, message.optional_fixed32) - test_case.assertEqual(108, message.optional_fixed64) - test_case.assertEqual(109, message.optional_sfixed32) - test_case.assertEqual(110, message.optional_sfixed64) - test_case.assertEqual(111, message.optional_float) - test_case.assertEqual(112, message.optional_double) - test_case.assertEqual(True, message.optional_bool) - test_case.assertEqual('115', message.optional_string) - test_case.assertEqual(b'116', message.optional_bytes) - - if IsProto2(message): - test_case.assertEqual(117, message.optionalgroup.a) - test_case.assertEqual(118, message.optional_nested_message.bb) - test_case.assertEqual(119, message.optional_foreign_message.c) - test_case.assertEqual(120, message.optional_import_message.d) - test_case.assertEqual(126, message.optional_public_import_message.e) - test_case.assertEqual(127, message.optional_lazy_message.bb) - test_case.assertEqual(128, message.optional_unverified_lazy_message.bb) - - test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ, - message.optional_nested_enum) - test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, - message.optional_foreign_enum) - if IsProto2(message): - test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, - message.optional_import_enum) - - # ----------------------------------------------------------------- - - test_case.assertEqual(2, len(message.repeated_int32)) - test_case.assertEqual(2, len(message.repeated_int64)) - test_case.assertEqual(2, len(message.repeated_uint32)) - test_case.assertEqual(2, len(message.repeated_uint64)) - test_case.assertEqual(2, len(message.repeated_sint32)) - test_case.assertEqual(2, len(message.repeated_sint64)) - test_case.assertEqual(2, len(message.repeated_fixed32)) - test_case.assertEqual(2, len(message.repeated_fixed64)) - test_case.assertEqual(2, len(message.repeated_sfixed32)) - test_case.assertEqual(2, len(message.repeated_sfixed64)) - test_case.assertEqual(2, len(message.repeated_float)) - test_case.assertEqual(2, len(message.repeated_double)) - test_case.assertEqual(2, len(message.repeated_bool)) - test_case.assertEqual(2, len(message.repeated_string)) - test_case.assertEqual(2, len(message.repeated_bytes)) - - if IsProto2(message): - test_case.assertEqual(2, len(message.repeatedgroup)) - test_case.assertEqual(2, len(message.repeated_nested_message)) - test_case.assertEqual(2, len(message.repeated_foreign_message)) - test_case.assertEqual(2, len(message.repeated_import_message)) - test_case.assertEqual(2, len(message.repeated_nested_enum)) - test_case.assertEqual(2, len(message.repeated_foreign_enum)) - if IsProto2(message): - test_case.assertEqual(2, len(message.repeated_import_enum)) - - test_case.assertEqual(2, len(message.repeated_string_piece)) - test_case.assertEqual(2, len(message.repeated_cord)) - - test_case.assertEqual(201, message.repeated_int32[0]) - test_case.assertEqual(202, message.repeated_int64[0]) - test_case.assertEqual(203, message.repeated_uint32[0]) - test_case.assertEqual(204, message.repeated_uint64[0]) - test_case.assertEqual(205, message.repeated_sint32[0]) - test_case.assertEqual(206, message.repeated_sint64[0]) - test_case.assertEqual(207, message.repeated_fixed32[0]) - test_case.assertEqual(208, message.repeated_fixed64[0]) - test_case.assertEqual(209, message.repeated_sfixed32[0]) - test_case.assertEqual(210, message.repeated_sfixed64[0]) - test_case.assertEqual(211, message.repeated_float[0]) - test_case.assertEqual(212, message.repeated_double[0]) - test_case.assertEqual(True, message.repeated_bool[0]) - test_case.assertEqual('215', message.repeated_string[0]) - test_case.assertEqual(b'216', message.repeated_bytes[0]) - - if IsProto2(message): - test_case.assertEqual(217, message.repeatedgroup[0].a) - test_case.assertEqual(218, message.repeated_nested_message[0].bb) - test_case.assertEqual(219, message.repeated_foreign_message[0].c) - test_case.assertEqual(220, message.repeated_import_message[0].d) - test_case.assertEqual(227, message.repeated_lazy_message[0].bb) - - test_case.assertEqual(unittest_pb2.TestAllTypes.BAR, - message.repeated_nested_enum[0]) - test_case.assertEqual(unittest_pb2.FOREIGN_BAR, - message.repeated_foreign_enum[0]) - if IsProto2(message): - test_case.assertEqual(unittest_import_pb2.IMPORT_BAR, - message.repeated_import_enum[0]) - - test_case.assertEqual(301, message.repeated_int32[1]) - test_case.assertEqual(302, message.repeated_int64[1]) - test_case.assertEqual(303, message.repeated_uint32[1]) - test_case.assertEqual(304, message.repeated_uint64[1]) - test_case.assertEqual(305, message.repeated_sint32[1]) - test_case.assertEqual(306, message.repeated_sint64[1]) - test_case.assertEqual(307, message.repeated_fixed32[1]) - test_case.assertEqual(308, message.repeated_fixed64[1]) - test_case.assertEqual(309, message.repeated_sfixed32[1]) - test_case.assertEqual(310, message.repeated_sfixed64[1]) - test_case.assertEqual(311, message.repeated_float[1]) - test_case.assertEqual(312, message.repeated_double[1]) - test_case.assertEqual(False, message.repeated_bool[1]) - test_case.assertEqual('315', message.repeated_string[1]) - test_case.assertEqual(b'316', message.repeated_bytes[1]) - - if IsProto2(message): - test_case.assertEqual(317, message.repeatedgroup[1].a) - test_case.assertEqual(318, message.repeated_nested_message[1].bb) - test_case.assertEqual(319, message.repeated_foreign_message[1].c) - test_case.assertEqual(320, message.repeated_import_message[1].d) - test_case.assertEqual(327, message.repeated_lazy_message[1].bb) - - test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ, - message.repeated_nested_enum[1]) - test_case.assertEqual(unittest_pb2.FOREIGN_BAZ, - message.repeated_foreign_enum[1]) - if IsProto2(message): - test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ, - message.repeated_import_enum[1]) - - # ----------------------------------------------------------------- - - if IsProto2(message): - test_case.assertTrue(message.HasField('default_int32')) - test_case.assertTrue(message.HasField('default_int64')) - test_case.assertTrue(message.HasField('default_uint32')) - test_case.assertTrue(message.HasField('default_uint64')) - test_case.assertTrue(message.HasField('default_sint32')) - test_case.assertTrue(message.HasField('default_sint64')) - test_case.assertTrue(message.HasField('default_fixed32')) - test_case.assertTrue(message.HasField('default_fixed64')) - test_case.assertTrue(message.HasField('default_sfixed32')) - test_case.assertTrue(message.HasField('default_sfixed64')) - test_case.assertTrue(message.HasField('default_float')) - test_case.assertTrue(message.HasField('default_double')) - test_case.assertTrue(message.HasField('default_bool')) - test_case.assertTrue(message.HasField('default_string')) - test_case.assertTrue(message.HasField('default_bytes')) - - test_case.assertTrue(message.HasField('default_nested_enum')) - test_case.assertTrue(message.HasField('default_foreign_enum')) - test_case.assertTrue(message.HasField('default_import_enum')) - - test_case.assertEqual(401, message.default_int32) - test_case.assertEqual(402, message.default_int64) - test_case.assertEqual(403, message.default_uint32) - test_case.assertEqual(404, message.default_uint64) - test_case.assertEqual(405, message.default_sint32) - test_case.assertEqual(406, message.default_sint64) - test_case.assertEqual(407, message.default_fixed32) - test_case.assertEqual(408, message.default_fixed64) - test_case.assertEqual(409, message.default_sfixed32) - test_case.assertEqual(410, message.default_sfixed64) - test_case.assertEqual(411, message.default_float) - test_case.assertEqual(412, message.default_double) - test_case.assertEqual(False, message.default_bool) - test_case.assertEqual('415', message.default_string) - test_case.assertEqual(b'416', message.default_bytes) - - test_case.assertEqual(unittest_pb2.TestAllTypes.FOO, - message.default_nested_enum) - test_case.assertEqual(unittest_pb2.FOREIGN_FOO, - message.default_foreign_enum) - test_case.assertEqual(unittest_import_pb2.IMPORT_FOO, - message.default_import_enum) - - -def GoldenFile(filename): - """Finds the given golden file and returns a file object representing it.""" - - # Search up the directory tree looking for the C++ protobuf source code. - path = '.' - while os.path.exists(path): - if os.path.exists(os.path.join(path, 'src/google/protobuf')): - # Found it. Load the golden file from the testdata directory. - full_path = os.path.join(path, 'src/google/protobuf/testdata', filename) - return open(full_path, 'rb') - path = os.path.join(path, '..') - - # Search internally. - path = '.' - full_path = os.path.join(path, 'third_party/py/google/protobuf/testdata', - filename) - if os.path.exists(full_path): - # Found it. Load the golden file from the testdata directory. - return open(full_path, 'rb') - - # Search for cross-repo path. - full_path = os.path.join('external/com_google_protobuf/src/google/protobuf/testdata', - filename) - if os.path.exists(full_path): - # Found it. Load the golden file from the testdata directory. - return open(full_path, 'rb') - - raise RuntimeError( - 'Could not find golden files. This test must be run from within the ' - 'protobuf source package so that it can read test data files from the ' - 'C++ source tree.') - - -def GoldenFileData(filename): - """Finds the given golden file and returns its contents.""" - with GoldenFile(filename) as f: - return f.read() - - -def SetAllPackedFields(message): - """Sets every field in the message to a unique value. - - Args: - message: A TestPackedTypes instance. - """ - message.packed_int32.extend([601, 701]) - message.packed_int64.extend([602, 702]) - message.packed_uint32.extend([603, 703]) - message.packed_uint64.extend([604, 704]) - message.packed_sint32.extend([605, 705]) - message.packed_sint64.extend([606, 706]) - message.packed_fixed32.extend([607, 707]) - message.packed_fixed64.extend([608, 708]) - message.packed_sfixed32.extend([609, 709]) - message.packed_sfixed64.extend([610, 710]) - message.packed_float.extend([611.0, 711.0]) - message.packed_double.extend([612.0, 712.0]) - message.packed_bool.extend([True, False]) - message.packed_enum.extend([unittest_pb2.FOREIGN_BAR, - unittest_pb2.FOREIGN_BAZ]) - - -def SetAllPackedExtensions(message): - """Sets every extension in the message to a unique value. - - Args: - message: A unittest_pb2.TestPackedExtensions instance. - """ - extensions = message.Extensions - pb2 = unittest_pb2 - - extensions[pb2.packed_int32_extension].extend([601, 701]) - extensions[pb2.packed_int64_extension].extend([602, 702]) - extensions[pb2.packed_uint32_extension].extend([603, 703]) - extensions[pb2.packed_uint64_extension].extend([604, 704]) - extensions[pb2.packed_sint32_extension].extend([605, 705]) - extensions[pb2.packed_sint64_extension].extend([606, 706]) - extensions[pb2.packed_fixed32_extension].extend([607, 707]) - extensions[pb2.packed_fixed64_extension].extend([608, 708]) - extensions[pb2.packed_sfixed32_extension].extend([609, 709]) - extensions[pb2.packed_sfixed64_extension].extend([610, 710]) - extensions[pb2.packed_float_extension].extend([611.0, 711.0]) - extensions[pb2.packed_double_extension].extend([612.0, 712.0]) - extensions[pb2.packed_bool_extension].extend([True, False]) - extensions[pb2.packed_enum_extension].extend([unittest_pb2.FOREIGN_BAR, - unittest_pb2.FOREIGN_BAZ]) - - -def SetAllUnpackedFields(message): - """Sets every field in the message to a unique value. - - Args: - message: A unittest_pb2.TestUnpackedTypes instance. - """ - message.unpacked_int32.extend([601, 701]) - message.unpacked_int64.extend([602, 702]) - message.unpacked_uint32.extend([603, 703]) - message.unpacked_uint64.extend([604, 704]) - message.unpacked_sint32.extend([605, 705]) - message.unpacked_sint64.extend([606, 706]) - message.unpacked_fixed32.extend([607, 707]) - message.unpacked_fixed64.extend([608, 708]) - message.unpacked_sfixed32.extend([609, 709]) - message.unpacked_sfixed64.extend([610, 710]) - message.unpacked_float.extend([611.0, 711.0]) - message.unpacked_double.extend([612.0, 712.0]) - message.unpacked_bool.extend([True, False]) - message.unpacked_enum.extend([unittest_pb2.FOREIGN_BAR, - unittest_pb2.FOREIGN_BAZ]) - - -class NonStandardInteger(numbers.Integral): - """An integer object that does not subclass int. - - This is used to verify that both C++ and regular proto systems can handle - integer others than int and long and that they handle them in predictable - ways. - - NonStandardInteger is the minimal legal specification for a custom Integral. - As such, it does not support 0 < x < 5 and it is not hashable. - - Note: This is added here instead of relying on numpy or a similar library - with custom integers to limit dependencies. - """ - - def __init__(self, val, error_string_on_conversion=None): - assert isinstance(val, numbers.Integral) - if isinstance(val, NonStandardInteger): - val = val.val - self.val = val - self.error_string_on_conversion = error_string_on_conversion - - def __long__(self): - if self.error_string_on_conversion: - raise RuntimeError(self.error_string_on_conversion) - return long(self.val) - - def __abs__(self): - return NonStandardInteger(operator.abs(self.val)) - - def __add__(self, y): - return NonStandardInteger(operator.add(self.val, y)) - - def __div__(self, y): - return NonStandardInteger(operator.div(self.val, y)) - - def __eq__(self, y): - return operator.eq(self.val, y) - - def __floordiv__(self, y): - return NonStandardInteger(operator.floordiv(self.val, y)) - - def __truediv__(self, y): - return NonStandardInteger(operator.truediv(self.val, y)) - - def __invert__(self): - return NonStandardInteger(operator.invert(self.val)) - - def __mod__(self, y): - return NonStandardInteger(operator.mod(self.val, y)) - - def __mul__(self, y): - return NonStandardInteger(operator.mul(self.val, y)) - - def __neg__(self): - return NonStandardInteger(operator.neg(self.val)) - - def __pos__(self): - return NonStandardInteger(operator.pos(self.val)) - - def __pow__(self, y): - return NonStandardInteger(operator.pow(self.val, y)) - - def __trunc__(self): - return int(self.val) - - def __radd__(self, y): - return NonStandardInteger(operator.add(y, self.val)) - - def __rdiv__(self, y): - return NonStandardInteger(operator.div(y, self.val)) - - def __rmod__(self, y): - return NonStandardInteger(operator.mod(y, self.val)) - - def __rmul__(self, y): - return NonStandardInteger(operator.mul(y, self.val)) - - def __rpow__(self, y): - return NonStandardInteger(operator.pow(y, self.val)) - - def __rfloordiv__(self, y): - return NonStandardInteger(operator.floordiv(y, self.val)) - - def __rtruediv__(self, y): - return NonStandardInteger(operator.truediv(y, self.val)) - - def __lshift__(self, y): - return NonStandardInteger(operator.lshift(self.val, y)) - - def __rshift__(self, y): - return NonStandardInteger(operator.rshift(self.val, y)) - - def __rlshift__(self, y): - return NonStandardInteger(operator.lshift(y, self.val)) - - def __rrshift__(self, y): - return NonStandardInteger(operator.rshift(y, self.val)) - - def __le__(self, y): - if isinstance(y, NonStandardInteger): - y = y.val - return operator.le(self.val, y) - - def __lt__(self, y): - if isinstance(y, NonStandardInteger): - y = y.val - return operator.lt(self.val, y) - - def __and__(self, y): - return NonStandardInteger(operator.and_(self.val, y)) - - def __or__(self, y): - return NonStandardInteger(operator.or_(self.val, y)) - - def __xor__(self, y): - return NonStandardInteger(operator.xor(self.val, y)) - - def __rand__(self, y): - return NonStandardInteger(operator.and_(y, self.val)) - - def __ror__(self, y): - return NonStandardInteger(operator.or_(y, self.val)) - - def __rxor__(self, y): - return NonStandardInteger(operator.xor(y, self.val)) - - def __bool__(self): - return self.val - - def __nonzero__(self): - return self.val - - def __ceil__(self): - return self - - def __floor__(self): - return self - - def __int__(self): - if self.error_string_on_conversion: - raise RuntimeError(self.error_string_on_conversion) - return int(self.val) - - def __round__(self): - return self - - def __repr__(self): - return 'NonStandardInteger(%s)' % self.val diff --git a/ext/protobuf/Python/google/protobuf/internal/text_encoding_test.py b/ext/protobuf/Python/google/protobuf/internal/text_encoding_test.py deleted file mode 100644 index f36a2cc5b..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/text_encoding_test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Tests for google.protobuf.text_encoding.""" - -import unittest - -from google.protobuf import text_encoding - -TEST_VALUES = [ - ("foo\\rbar\\nbaz\\t", - "foo\\rbar\\nbaz\\t", - b"foo\rbar\nbaz\t"), - ("\\'full of \\\"sound\\\" and \\\"fury\\\"\\'", - "\\'full of \\\"sound\\\" and \\\"fury\\\"\\'", - b"'full of \"sound\" and \"fury\"'"), - ("signi\\\\fying\\\\ nothing\\\\", - "signi\\\\fying\\\\ nothing\\\\", - b"signi\\fying\\ nothing\\"), - ("\\010\\t\\n\\013\\014\\r", - "\x08\\t\\n\x0b\x0c\\r", - b"\010\011\012\013\014\015")] - - -class TextEncodingTestCase(unittest.TestCase): - def testCEscape(self): - for escaped, escaped_utf8, unescaped in TEST_VALUES: - self.assertEqual(escaped, - text_encoding.CEscape(unescaped, as_utf8=False)) - self.assertEqual(escaped_utf8, - text_encoding.CEscape(unescaped, as_utf8=True)) - - def testCUnescape(self): - for escaped, escaped_utf8, unescaped in TEST_VALUES: - self.assertEqual(unescaped, text_encoding.CUnescape(escaped)) - self.assertEqual(unescaped, text_encoding.CUnescape(escaped_utf8)) - - -if __name__ == "__main__": - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/text_format_test.py b/ext/protobuf/Python/google/protobuf/internal/text_format_test.py deleted file mode 100644 index 18b784e5b..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/text_format_test.py +++ /dev/null @@ -1,2447 +0,0 @@ -# -*- coding: utf-8 -*- -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test for google.protobuf.text_format.""" - -import io -import math -import re -import string -import textwrap - -import unittest - -from google.protobuf import any_pb2 -from google.protobuf import struct_pb2 -from google.protobuf import any_test_pb2 -from google.protobuf import map_unittest_pb2 -from google.protobuf import unittest_custom_options_pb2 -from google.protobuf import unittest_mset_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import unittest_proto3_arena_pb2 -from google.protobuf import descriptor_pb2 -from google.protobuf.internal import any_test_pb2 as test_extend_any -from google.protobuf.internal import api_implementation -from google.protobuf.internal import message_set_extensions_pb2 -from google.protobuf.internal import test_proto3_optional_pb2 -from google.protobuf.internal import test_util -from google.protobuf import descriptor_pool -from google.protobuf import text_format -from google.protobuf.internal import _parameterized -# pylint: enable=g-import-not-at-top - - -# Low-level nuts-n-bolts tests. -class SimpleTextFormatTests(unittest.TestCase): - - # The members of _QUOTES are formatted into a regexp template that - # expects single characters. Therefore it's an error (in addition to being - # non-sensical in the first place) to try to specify a "quote mark" that is - # more than one character. - def testQuoteMarksAreSingleChars(self): - for quote in text_format._QUOTES: - self.assertEqual(1, len(quote)) - - -# Base class with some common functionality. -class TextFormatBase(unittest.TestCase): - - def ReadGolden(self, golden_filename): - with test_util.GoldenFile(golden_filename) as f: - return (f.readlines() if str is bytes else # PY3 - [golden_line.decode('utf-8') for golden_line in f]) - - def CompareToGoldenFile(self, text, golden_filename): - golden_lines = self.ReadGolden(golden_filename) - self.assertMultiLineEqual(text, ''.join(golden_lines)) - - def CompareToGoldenText(self, text, golden_text): - self.assertEqual(text, golden_text) - - def RemoveRedundantZeros(self, text): - # Some platforms print 1e+5 as 1e+005. This is fine, but we need to remove - # these zeros in order to match the golden file. - text = text.replace('e+0','e+').replace('e+0','e+') \ - .replace('e-0','e-').replace('e-0','e-') - # Floating point fields are printed with .0 suffix even if they are - # actually integer numbers. - text = re.compile(r'\.0$', re.MULTILINE).sub('', text) - return text - - -@_parameterized.parameters(unittest_pb2, unittest_proto3_arena_pb2) -class TextFormatMessageToStringTests(TextFormatBase): - - def testPrintExotic(self, message_module): - message = message_module.TestAllTypes() - message.repeated_int64.append(-9223372036854775808) - message.repeated_uint64.append(18446744073709551615) - message.repeated_double.append(123.456) - message.repeated_double.append(1.23e22) - message.repeated_double.append(1.23e-18) - message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"') - message.repeated_string.append(u'\u00fc\ua71f') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'repeated_int64: -9223372036854775808\n' - 'repeated_uint64: 18446744073709551615\n' - 'repeated_double: 123.456\n' - 'repeated_double: 1.23e+22\n' - 'repeated_double: 1.23e-18\n' - 'repeated_string:' - ' "\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n' - 'repeated_string: "\\303\\274\\352\\234\\237"\n') - - def testPrintFloatPrecision(self, message_module): - message = message_module.TestAllTypes() - - message.repeated_float.append(0.0) - message.repeated_float.append(0.8) - message.repeated_float.append(1.0) - message.repeated_float.append(1.2) - message.repeated_float.append(1.23) - message.repeated_float.append(1.234) - message.repeated_float.append(1.2345) - message.repeated_float.append(1.23456) - message.repeated_float.append(1.2e10) - message.repeated_float.append(1.23e10) - message.repeated_float.append(1.234e10) - message.repeated_float.append(1.2345e10) - message.repeated_float.append(1.23456e10) - message.repeated_float.append(float('NaN')) - message.repeated_float.append(float('inf')) - message.repeated_double.append(0.0) - message.repeated_double.append(0.8) - message.repeated_double.append(1.0) - message.repeated_double.append(1.2) - message.repeated_double.append(1.23) - message.repeated_double.append(1.234) - message.repeated_double.append(1.2345) - message.repeated_double.append(1.23456) - message.repeated_double.append(1.234567) - message.repeated_double.append(1.2345678) - message.repeated_double.append(1.23456789) - message.repeated_double.append(1.234567898) - message.repeated_double.append(1.2345678987) - message.repeated_double.append(1.23456789876) - message.repeated_double.append(1.234567898765) - message.repeated_double.append(1.2345678987654) - message.repeated_double.append(1.23456789876543) - message.repeated_double.append(1.2e100) - message.repeated_double.append(1.23e100) - message.repeated_double.append(1.234e100) - message.repeated_double.append(1.2345e100) - message.repeated_double.append(1.23456e100) - message.repeated_double.append(1.234567e100) - message.repeated_double.append(1.2345678e100) - message.repeated_double.append(1.23456789e100) - message.repeated_double.append(1.234567898e100) - message.repeated_double.append(1.2345678987e100) - message.repeated_double.append(1.23456789876e100) - message.repeated_double.append(1.234567898765e100) - message.repeated_double.append(1.2345678987654e100) - message.repeated_double.append(1.23456789876543e100) - # pylint: disable=g-long-ternary - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'repeated_float: 0\n' - 'repeated_float: 0.8\n' - 'repeated_float: 1\n' - 'repeated_float: 1.2\n' - 'repeated_float: 1.23\n' - 'repeated_float: 1.234\n' - 'repeated_float: 1.2345\n' - 'repeated_float: 1.23456\n' - # Note that these don't use scientific notation. - 'repeated_float: 12000000000\n' - 'repeated_float: 12300000000\n' - 'repeated_float: 12340000000\n' - 'repeated_float: 12345000000\n' - 'repeated_float: 12345600000\n' - 'repeated_float: nan\n' - 'repeated_float: inf\n' - 'repeated_double: 0\n' - 'repeated_double: 0.8\n' - 'repeated_double: 1\n' - 'repeated_double: 1.2\n' - 'repeated_double: 1.23\n' - 'repeated_double: 1.234\n' - 'repeated_double: 1.2345\n' - 'repeated_double: 1.23456\n' - 'repeated_double: 1.234567\n' - 'repeated_double: 1.2345678\n' - 'repeated_double: 1.23456789\n' - 'repeated_double: 1.234567898\n' - 'repeated_double: 1.2345678987\n' - 'repeated_double: 1.23456789876\n' - 'repeated_double: 1.234567898765\n' - 'repeated_double: 1.2345678987654\n' - 'repeated_double: 1.23456789876543\n' - 'repeated_double: 1.2e+100\n' - 'repeated_double: 1.23e+100\n' - 'repeated_double: 1.234e+100\n' - 'repeated_double: 1.2345e+100\n' - 'repeated_double: 1.23456e+100\n' - 'repeated_double: 1.234567e+100\n' - 'repeated_double: 1.2345678e+100\n' - 'repeated_double: 1.23456789e+100\n' - 'repeated_double: 1.234567898e+100\n' - 'repeated_double: 1.2345678987e+100\n' - 'repeated_double: 1.23456789876e+100\n' - 'repeated_double: 1.234567898765e+100\n' - 'repeated_double: 1.2345678987654e+100\n' - 'repeated_double: 1.23456789876543e+100\n') - - def testPrintExoticUnicodeSubclass(self, message_module): - - class UnicodeSub(str): - pass - - message = message_module.TestAllTypes() - message.repeated_string.append(UnicodeSub(u'\u00fc\ua71f')) - self.CompareToGoldenText( - text_format.MessageToString(message), - 'repeated_string: "\\303\\274\\352\\234\\237"\n') - - def testPrintNestedMessageAsOneLine(self, message_module): - message = message_module.TestAllTypes() - msg = message.repeated_nested_message.add() - msg.bb = 42 - self.CompareToGoldenText( - text_format.MessageToString(message, as_one_line=True), - 'repeated_nested_message { bb: 42 }') - - def testPrintRepeatedFieldsAsOneLine(self, message_module): - message = message_module.TestAllTypes() - message.repeated_int32.append(1) - message.repeated_int32.append(1) - message.repeated_int32.append(3) - message.repeated_string.append('Google') - message.repeated_string.append('Zurich') - self.CompareToGoldenText( - text_format.MessageToString(message, as_one_line=True), - 'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 ' - 'repeated_string: "Google" repeated_string: "Zurich"') - - def VerifyPrintShortFormatRepeatedFields(self, message_module, as_one_line): - message = message_module.TestAllTypes() - message.repeated_int32.append(1) - message.repeated_string.append('Google') - message.repeated_string.append('Hello,World') - message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_FOO) - message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR) - message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ) - message.optional_nested_message.bb = 3 - for i in (21, 32): - msg = message.repeated_nested_message.add() - msg.bb = i - expected_ascii = ( - 'optional_nested_message {\n bb: 3\n}\n' - 'repeated_int32: [1]\n' - 'repeated_string: "Google"\n' - 'repeated_string: "Hello,World"\n' - 'repeated_nested_message {\n bb: 21\n}\n' - 'repeated_nested_message {\n bb: 32\n}\n' - 'repeated_foreign_enum: [FOREIGN_FOO, FOREIGN_BAR, FOREIGN_BAZ]\n') - if as_one_line: - expected_ascii = expected_ascii.replace('\n', ' ') - expected_ascii = re.sub(r'\s+', ' ', expected_ascii) - expected_ascii = re.sub(r'\s$', '', expected_ascii) - - actual_ascii = text_format.MessageToString( - message, use_short_repeated_primitives=True, - as_one_line=as_one_line) - self.CompareToGoldenText(actual_ascii, expected_ascii) - parsed_message = message_module.TestAllTypes() - text_format.Parse(actual_ascii, parsed_message) - self.assertEqual(parsed_message, message) - - def testPrintShortFormatRepeatedFields(self, message_module): - self.VerifyPrintShortFormatRepeatedFields(message_module, False) - self.VerifyPrintShortFormatRepeatedFields(message_module, True) - - def testPrintNestedNewLineInStringAsOneLine(self, message_module): - message = message_module.TestAllTypes() - message.optional_string = 'a\nnew\nline' - self.CompareToGoldenText( - text_format.MessageToString(message, as_one_line=True), - 'optional_string: "a\\nnew\\nline"') - - def testPrintExoticAsOneLine(self, message_module): - message = message_module.TestAllTypes() - message.repeated_int64.append(-9223372036854775808) - message.repeated_uint64.append(18446744073709551615) - message.repeated_double.append(123.456) - message.repeated_double.append(1.23e22) - message.repeated_double.append(1.23e-18) - message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"') - message.repeated_string.append(u'\u00fc\ua71f') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_format.MessageToString( - message, as_one_line=True)), - 'repeated_int64: -9223372036854775808' - ' repeated_uint64: 18446744073709551615' - ' repeated_double: 123.456' - ' repeated_double: 1.23e+22' - ' repeated_double: 1.23e-18' - ' repeated_string: ' - '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""' - ' repeated_string: "\\303\\274\\352\\234\\237"') - - def testRoundTripExoticAsOneLine(self, message_module): - message = message_module.TestAllTypes() - message.repeated_int64.append(-9223372036854775808) - message.repeated_uint64.append(18446744073709551615) - message.repeated_double.append(123.456) - message.repeated_double.append(1.23e22) - message.repeated_double.append(1.23e-18) - message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"') - message.repeated_string.append(u'\u00fc\ua71f') - - # Test as_utf8 = False. - wire_text = text_format.MessageToString(message, - as_one_line=True, - as_utf8=False) - parsed_message = message_module.TestAllTypes() - r = text_format.Parse(wire_text, parsed_message) - self.assertIs(r, parsed_message) - self.assertEqual(message, parsed_message) - - # Test as_utf8 = True. - wire_text = text_format.MessageToString(message, - as_one_line=True, - as_utf8=True) - parsed_message = message_module.TestAllTypes() - r = text_format.Parse(wire_text, parsed_message) - self.assertIs(r, parsed_message) - self.assertEqual(message, parsed_message, - '\n%s != %s' % (message, parsed_message)) - - def testPrintRawUtf8String(self, message_module): - message = message_module.TestAllTypes() - message.repeated_string.append(u'\u00fc\t\ua71f') - text = text_format.MessageToString(message, as_utf8=True) - golden_unicode = u'repeated_string: "\u00fc\\t\ua71f"\n' - golden_text = golden_unicode - # MessageToString always returns a native str. - self.CompareToGoldenText(text, golden_text) - parsed_message = message_module.TestAllTypes() - text_format.Parse(text, parsed_message) - self.assertEqual( - message, parsed_message, '\n%s != %s (%s != %s)' % - (message, parsed_message, message.repeated_string[0], - parsed_message.repeated_string[0])) - - def testPrintFloatFormat(self, message_module): - # Check that float_format argument is passed to sub-message formatting. - message = message_module.NestedTestAllTypes() - message.payload.optional_float = 1.25 - # Check rounding at 15 significant digits - message.payload.optional_double = -.000003456789012345678 - # Check no decimal point. - message.payload.repeated_float.append(-5642) - # Check no trailing zeros. - message.payload.repeated_double.append(.000078900) - formatted_fields = ['optional_float: 1.25', - 'optional_double: -3.45678901234568e-6', - 'repeated_float: -5642', 'repeated_double: 7.89e-5'] - text_message = text_format.MessageToString(message, float_format='.15g') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_message), - 'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format( - *formatted_fields)) - # as_one_line=True is a separate code branch where float_format is passed. - text_message = text_format.MessageToString(message, - as_one_line=True, - float_format='.15g') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_message), - 'payload {{ {0} {1} {2} {3} }}'.format(*formatted_fields)) - - # 32-bit 1.2 is noisy when extended to 64-bit: - # >>> struct.unpack('f', struct.pack('f', 1.2))[0] - # 1.2000000476837158 - message.payload.optional_float = 1.2 - formatted_fields = ['optional_float: 1.2', - 'optional_double: -3.45678901234568e-6', - 'repeated_float: -5642', 'repeated_double: 7.89e-5'] - text_message = text_format.MessageToString(message, float_format='.7g', - double_format='.15g') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_message), - 'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format( - *formatted_fields)) - - # Test only set float_format affect both float and double fields. - formatted_fields = ['optional_float: 1.2', - 'optional_double: -3.456789e-6', - 'repeated_float: -5642', 'repeated_double: 7.89e-5'] - text_message = text_format.MessageToString(message, float_format='.7g') - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_message), - 'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format( - *formatted_fields)) - - # Test default float_format will automatic print shortest float. - message.payload.optional_float = 1.2345678912 - message.payload.optional_double = 1.2345678912 - formatted_fields = ['optional_float: 1.2345679', - 'optional_double: 1.2345678912', - 'repeated_float: -5642', 'repeated_double: 7.89e-5'] - text_message = text_format.MessageToString(message) - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_message), - 'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format( - *formatted_fields)) - - message.Clear() - message.payload.optional_float = 1.1000000000011 - self.assertEqual(text_format.MessageToString(message), - 'payload {\n optional_float: 1.1\n}\n') - message.payload.optional_float = 1.00000075e-36 - self.assertEqual(text_format.MessageToString(message), - 'payload {\n optional_float: 1.00000075e-36\n}\n') - message.payload.optional_float = 12345678912345e+11 - self.assertEqual(text_format.MessageToString(message), - 'payload {\n optional_float: 1.234568e+24\n}\n') - - def testMessageToString(self, message_module): - message = message_module.ForeignMessage() - message.c = 123 - self.assertEqual('c: 123\n', str(message)) - - def testMessageToStringUnicode(self, message_module): - golden_unicode = u'Á short desçription and a 🍌.' - golden_bytes = golden_unicode.encode('utf-8') - message = message_module.TestAllTypes() - message.optional_string = golden_unicode - message.optional_bytes = golden_bytes - text = text_format.MessageToString(message, as_utf8=True) - golden_message = textwrap.dedent( - 'optional_string: "Á short desçription and a 🍌."\n' - 'optional_bytes: ' - r'"\303\201 short des\303\247ription and a \360\237\215\214."' - '\n') - self.CompareToGoldenText(text, golden_message) - - def testMessageToStringASCII(self, message_module): - golden_unicode = u'Á short desçription and a 🍌.' - golden_bytes = golden_unicode.encode('utf-8') - message = message_module.TestAllTypes() - message.optional_string = golden_unicode - message.optional_bytes = golden_bytes - text = text_format.MessageToString(message, as_utf8=False) # ASCII - golden_message = ( - 'optional_string: ' - r'"\303\201 short des\303\247ription and a \360\237\215\214."' - '\n' - 'optional_bytes: ' - r'"\303\201 short des\303\247ription and a \360\237\215\214."' - '\n') - self.CompareToGoldenText(text, golden_message) - - def testPrintField(self, message_module): - message = message_module.TestAllTypes() - field = message.DESCRIPTOR.fields_by_name['optional_float'] - value = message.optional_float - out = text_format.TextWriter(False) - text_format.PrintField(field, value, out) - self.assertEqual('optional_float: 0.0\n', out.getvalue()) - out.close() - # Test Printer - out = text_format.TextWriter(False) - printer = text_format._Printer(out) - printer.PrintField(field, value) - self.assertEqual('optional_float: 0.0\n', out.getvalue()) - out.close() - - def testPrintFieldValue(self, message_module): - message = message_module.TestAllTypes() - field = message.DESCRIPTOR.fields_by_name['optional_float'] - value = message.optional_float - out = text_format.TextWriter(False) - text_format.PrintFieldValue(field, value, out) - self.assertEqual('0.0', out.getvalue()) - out.close() - # Test Printer - out = text_format.TextWriter(False) - printer = text_format._Printer(out) - printer.PrintFieldValue(field, value) - self.assertEqual('0.0', out.getvalue()) - out.close() - - def testCustomOptions(self, message_module): - message_descriptor = (unittest_custom_options_pb2. - TestMessageWithCustomOptions.DESCRIPTOR) - message_proto = descriptor_pb2.DescriptorProto() - message_descriptor.CopyToProto(message_proto) - expected_text = ( - 'name: "TestMessageWithCustomOptions"\n' - 'field {\n' - ' name: "field1"\n' - ' number: 1\n' - ' label: LABEL_OPTIONAL\n' - ' type: TYPE_STRING\n' - ' options {\n' - ' ctype: CORD\n' - ' [protobuf_unittest.field_opt1]: 8765432109\n' - ' }\n' - '}\n' - 'field {\n' - ' name: "oneof_field"\n' - ' number: 2\n' - ' label: LABEL_OPTIONAL\n' - ' type: TYPE_INT32\n' - ' oneof_index: 0\n' - '}\n' - 'field {\n' - ' name: "map_field"\n' - ' number: 3\n' - ' label: LABEL_REPEATED\n' - ' type: TYPE_MESSAGE\n' - ' type_name: ".protobuf_unittest.TestMessageWithCustomOptions.' - 'MapFieldEntry"\n' - ' options {\n' - ' [protobuf_unittest.field_opt1]: 12345\n' - ' }\n' - '}\n' - 'nested_type {\n' - ' name: "MapFieldEntry"\n' - ' field {\n' - ' name: "key"\n' - ' number: 1\n' - ' label: LABEL_OPTIONAL\n' - ' type: TYPE_STRING\n' - ' }\n' - ' field {\n' - ' name: "value"\n' - ' number: 2\n' - ' label: LABEL_OPTIONAL\n' - ' type: TYPE_STRING\n' - ' }\n' - ' options {\n' - ' map_entry: true\n' - ' }\n' - '}\n' - 'enum_type {\n' - ' name: "AnEnum"\n' - ' value {\n' - ' name: "ANENUM_VAL1"\n' - ' number: 1\n' - ' }\n' - ' value {\n' - ' name: "ANENUM_VAL2"\n' - ' number: 2\n' - ' options {\n' - ' [protobuf_unittest.enum_value_opt1]: 123\n' - ' }\n' - ' }\n' - ' options {\n' - ' [protobuf_unittest.enum_opt1]: -789\n' - ' }\n' - '}\n' - 'options {\n' - ' message_set_wire_format: false\n' - ' [protobuf_unittest.message_opt1]: -56\n' - '}\n' - 'oneof_decl {\n' - ' name: "AnOneof"\n' - ' options {\n' - ' [protobuf_unittest.oneof_opt1]: -99\n' - ' }\n' - '}\n') - self.assertEqual(expected_text, - text_format.MessageToString(message_proto)) - parsed_proto = descriptor_pb2.DescriptorProto() - text_format.Parse(expected_text, parsed_proto) - self.assertEqual(message_proto, parsed_proto) - - @unittest.skipIf( - api_implementation.Type() == 'upb', - "upb API doesn't support old UnknownField API. The TextFormat library " - "needs to convert to the new API.") - def testPrintUnknownFieldsEmbeddedMessageInBytes(self, message_module): - inner_msg = message_module.TestAllTypes() - inner_msg.optional_int32 = 101 - inner_msg.optional_double = 102.0 - inner_msg.optional_string = u'hello' - inner_msg.optional_bytes = b'103' - inner_msg.optional_nested_message.bb = 105 - inner_data = inner_msg.SerializeToString() - outer_message = message_module.TestAllTypes() - outer_message.optional_int32 = 101 - outer_message.optional_bytes = inner_data - all_data = outer_message.SerializeToString() - empty_message = message_module.TestEmptyMessage() - empty_message.ParseFromString(all_data) - - self.assertEqual(' 1: 101\n' - ' 15 {\n' - ' 1: 101\n' - ' 12: 4636878028842991616\n' - ' 14: "hello"\n' - ' 15: "103"\n' - ' 18 {\n' - ' 1: 105\n' - ' }\n' - ' }\n', - text_format.MessageToString(empty_message, - indent=2, - print_unknown_fields=True)) - self.assertEqual('1: 101 ' - '15 { ' - '1: 101 ' - '12: 4636878028842991616 ' - '14: "hello" ' - '15: "103" ' - '18 { 1: 105 } ' - '}', - text_format.MessageToString(empty_message, - print_unknown_fields=True, - as_one_line=True)) - - -@_parameterized.parameters(unittest_pb2, unittest_proto3_arena_pb2) -class TextFormatMessageToTextBytesTests(TextFormatBase): - - def testMessageToBytes(self, message_module): - message = message_module.ForeignMessage() - message.c = 123 - self.assertEqual(b'c: 123\n', text_format.MessageToBytes(message)) - - def testRawUtf8RoundTrip(self, message_module): - message = message_module.TestAllTypes() - message.repeated_string.append(u'\u00fc\t\ua71f') - utf8_text = text_format.MessageToBytes(message, as_utf8=True) - golden_bytes = b'repeated_string: "\xc3\xbc\\t\xea\x9c\x9f"\n' - self.CompareToGoldenText(utf8_text, golden_bytes) - parsed_message = message_module.TestAllTypes() - text_format.Parse(utf8_text, parsed_message) - self.assertEqual( - message, parsed_message, '\n%s != %s (%s != %s)' % - (message, parsed_message, message.repeated_string[0], - parsed_message.repeated_string[0])) - - def testEscapedUtf8ASCIIRoundTrip(self, message_module): - message = message_module.TestAllTypes() - message.repeated_string.append(u'\u00fc\t\ua71f') - ascii_text = text_format.MessageToBytes(message) # as_utf8=False default - golden_bytes = b'repeated_string: "\\303\\274\\t\\352\\234\\237"\n' - self.CompareToGoldenText(ascii_text, golden_bytes) - parsed_message = message_module.TestAllTypes() - text_format.Parse(ascii_text, parsed_message) - self.assertEqual( - message, parsed_message, '\n%s != %s (%s != %s)' % - (message, parsed_message, message.repeated_string[0], - parsed_message.repeated_string[0])) - - -@_parameterized.parameters(unittest_pb2, unittest_proto3_arena_pb2) -class TextFormatParserTests(TextFormatBase): - - def testParseAllFields(self, message_module): - message = message_module.TestAllTypes() - test_util.SetAllFields(message) - ascii_text = text_format.MessageToString(message) - - parsed_message = message_module.TestAllTypes() - text_format.Parse(ascii_text, parsed_message) - self.assertEqual(message, parsed_message) - if message_module is unittest_pb2: - test_util.ExpectAllFieldsSet(self, message) - - def testParseAndMergeUtf8(self, message_module): - message = message_module.TestAllTypes() - test_util.SetAllFields(message) - ascii_text = text_format.MessageToString(message) - ascii_text = ascii_text.encode('utf-8') - - parsed_message = message_module.TestAllTypes() - text_format.Parse(ascii_text, parsed_message) - self.assertEqual(message, parsed_message) - if message_module is unittest_pb2: - test_util.ExpectAllFieldsSet(self, message) - - parsed_message.Clear() - text_format.Merge(ascii_text, parsed_message) - self.assertEqual(message, parsed_message) - if message_module is unittest_pb2: - test_util.ExpectAllFieldsSet(self, message) - - msg2 = message_module.TestAllTypes() - text = (u'optional_string: "café"') - text_format.Merge(text, msg2) - self.assertEqual(msg2.optional_string, u'café') - msg2.Clear() - self.assertEqual(msg2.optional_string, u'') - text_format.Parse(text, msg2) - self.assertEqual(msg2.optional_string, u'café') - - def testParseDoubleToFloat(self, message_module): - message = message_module.TestAllTypes() - text = ('repeated_float: 3.4028235e+39\n' - 'repeated_float: 1.4028235e-39\n') - text_format.Parse(text, message) - self.assertEqual(message.repeated_float[0], float('inf')) - self.assertAlmostEqual(message.repeated_float[1], 1.4028235e-39) - - def testParseExotic(self, message_module): - message = message_module.TestAllTypes() - text = ('repeated_int64: -9223372036854775808\n' - 'repeated_uint64: 18446744073709551615\n' - 'repeated_double: 123.456\n' - 'repeated_double: 1.23e+22\n' - 'repeated_double: 1.23e-18\n' - 'repeated_string: \n' - '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n' - 'repeated_string: "foo" \'corge\' "grault"\n' - 'repeated_string: "\\303\\274\\352\\234\\237"\n' - 'repeated_string: "\\xc3\\xbc"\n' - 'repeated_string: "\xc3\xbc"\n') - text_format.Parse(text, message) - - self.assertEqual(-9223372036854775808, message.repeated_int64[0]) - self.assertEqual(18446744073709551615, message.repeated_uint64[0]) - self.assertEqual(123.456, message.repeated_double[0]) - self.assertEqual(1.23e22, message.repeated_double[1]) - self.assertEqual(1.23e-18, message.repeated_double[2]) - self.assertEqual('\000\001\a\b\f\n\r\t\v\\\'"', message.repeated_string[0]) - self.assertEqual('foocorgegrault', message.repeated_string[1]) - self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2]) - self.assertEqual(u'\u00fc', message.repeated_string[3]) - - def testParseTrailingCommas(self, message_module): - message = message_module.TestAllTypes() - text = ('repeated_int64: 100;\n' - 'repeated_int64: 200;\n' - 'repeated_int64: 300,\n' - 'repeated_string: "one",\n' - 'repeated_string: "two";\n') - text_format.Parse(text, message) - - self.assertEqual(100, message.repeated_int64[0]) - self.assertEqual(200, message.repeated_int64[1]) - self.assertEqual(300, message.repeated_int64[2]) - self.assertEqual(u'one', message.repeated_string[0]) - self.assertEqual(u'two', message.repeated_string[1]) - - def testParseRepeatedScalarShortFormat(self, message_module): - message = message_module.TestAllTypes() - text = ('repeated_int64: [100, 200];\n' - 'repeated_int64: []\n' - 'repeated_int64: 300,\n' - 'repeated_string: ["one", "two"];\n') - text_format.Parse(text, message) - - self.assertEqual(100, message.repeated_int64[0]) - self.assertEqual(200, message.repeated_int64[1]) - self.assertEqual(300, message.repeated_int64[2]) - self.assertEqual(u'one', message.repeated_string[0]) - self.assertEqual(u'two', message.repeated_string[1]) - - def testParseRepeatedMessageShortFormat(self, message_module): - message = message_module.TestAllTypes() - text = ('repeated_nested_message: [{bb: 100}, {bb: 200}],\n' - 'repeated_nested_message: {bb: 300}\n' - 'repeated_nested_message [{bb: 400}];\n') - text_format.Parse(text, message) - - self.assertEqual(100, message.repeated_nested_message[0].bb) - self.assertEqual(200, message.repeated_nested_message[1].bb) - self.assertEqual(300, message.repeated_nested_message[2].bb) - self.assertEqual(400, message.repeated_nested_message[3].bb) - - def testParseEmptyText(self, message_module): - message = message_module.TestAllTypes() - text = '' - text_format.Parse(text, message) - self.assertEqual(message_module.TestAllTypes(), message) - - def testParseInvalidUtf8(self, message_module): - message = message_module.TestAllTypes() - text = 'repeated_string: "\\xc3\\xc3"' - with self.assertRaises(text_format.ParseError) as e: - text_format.Parse(text, message) - self.assertEqual(e.exception.GetLine(), 1) - self.assertEqual(e.exception.GetColumn(), 28) - - def testParseSingleWord(self, message_module): - message = message_module.TestAllTypes() - text = 'foo' - self.assertRaisesRegex( - text_format.ParseError, - (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' - r'"foo".'), text_format.Parse, text, message) - - def testParseUnknownField(self, message_module): - message = message_module.TestAllTypes() - text = 'unknown_field: 8\n' - self.assertRaisesRegex( - text_format.ParseError, - (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' - r'"unknown_field".'), text_format.Parse, text, message) - text = ('optional_int32: 123\n' - 'unknown_field: 8\n' - 'optional_nested_message { bb: 45 }') - text_format.Parse(text, message, allow_unknown_field=True) - self.assertEqual(message.optional_nested_message.bb, 45) - self.assertEqual(message.optional_int32, 123) - - def testParseBadEnumValue(self, message_module): - message = message_module.TestAllTypes() - text = 'optional_nested_enum: BARR' - self.assertRaisesRegex(text_format.ParseError, - (r'1:23 : \'optional_nested_enum: BARR\': ' - r'Enum type "\w+.TestAllTypes.NestedEnum" ' - r'has no value named BARR.'), text_format.Parse, - text, message) - - def testParseBadIntValue(self, message_module): - message = message_module.TestAllTypes() - text = 'optional_int32: bork' - self.assertRaisesRegex(text_format.ParseError, - ('1:17 : \'optional_int32: bork\': ' - 'Couldn\'t parse integer: bork'), text_format.Parse, - text, message) - - def testParseStringFieldUnescape(self, message_module): - message = message_module.TestAllTypes() - text = r'''repeated_string: "\xf\x62" - repeated_string: "\\xf\\x62" - repeated_string: "\\\xf\\\x62" - repeated_string: "\\\\xf\\\\x62" - repeated_string: "\\\\\xf\\\\\x62" - repeated_string: "\x5cx20"''' - - text_format.Parse(text, message) - - SLASH = '\\' - self.assertEqual('\x0fb', message.repeated_string[0]) - self.assertEqual(SLASH + 'xf' + SLASH + 'x62', message.repeated_string[1]) - self.assertEqual(SLASH + '\x0f' + SLASH + 'b', message.repeated_string[2]) - self.assertEqual(SLASH + SLASH + 'xf' + SLASH + SLASH + 'x62', - message.repeated_string[3]) - self.assertEqual(SLASH + SLASH + '\x0f' + SLASH + SLASH + 'b', - message.repeated_string[4]) - self.assertEqual(SLASH + 'x20', message.repeated_string[5]) - - def testParseOneof(self, message_module): - m = message_module.TestAllTypes() - m.oneof_uint32 = 11 - m2 = message_module.TestAllTypes() - text_format.Parse(text_format.MessageToString(m), m2) - self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) - - def testParseMultipleOneof(self, message_module): - m_string = '\n'.join(['oneof_uint32: 11', 'oneof_string: "foo"']) - m2 = message_module.TestAllTypes() - with self.assertRaisesRegex(text_format.ParseError, - ' is specified along with field '): - text_format.Parse(m_string, m2) - - # This example contains non-ASCII codepoint unicode data as literals - # which should come through as utf-8 for bytes, and as the unicode - # itself for string fields. It also demonstrates escaped binary data. - # The ur"" string prefix is unfortunately missing from Python 3 - # so we resort to double escaping our \s so that they come through. - _UNICODE_SAMPLE = u""" - optional_bytes: 'Á short desçription' - optional_string: 'Á short desçription' - repeated_bytes: '\\303\\201 short des\\303\\247ription' - repeated_bytes: '\\x12\\x34\\x56\\x78\\x90\\xab\\xcd\\xef' - repeated_string: '\\xd0\\x9f\\xd1\\x80\\xd0\\xb8\\xd0\\xb2\\xd0\\xb5\\xd1\\x82' - """ - _BYTES_SAMPLE = _UNICODE_SAMPLE.encode('utf-8') - _GOLDEN_UNICODE = u'Á short desçription' - _GOLDEN_BYTES = _GOLDEN_UNICODE.encode('utf-8') - _GOLDEN_BYTES_1 = b'\x12\x34\x56\x78\x90\xab\xcd\xef' - _GOLDEN_STR_0 = u'Привет' - - def testParseUnicode(self, message_module): - m = message_module.TestAllTypes() - text_format.Parse(self._UNICODE_SAMPLE, m) - self.assertEqual(m.optional_bytes, self._GOLDEN_BYTES) - self.assertEqual(m.optional_string, self._GOLDEN_UNICODE) - self.assertEqual(m.repeated_bytes[0], self._GOLDEN_BYTES) - # repeated_bytes[1] contained simple \ escaped non-UTF-8 raw binary data. - self.assertEqual(m.repeated_bytes[1], self._GOLDEN_BYTES_1) - # repeated_string[0] contained \ escaped data representing the UTF-8 - # representation of _GOLDEN_STR_0 - it needs to decode as such. - self.assertEqual(m.repeated_string[0], self._GOLDEN_STR_0) - - def testParseBytes(self, message_module): - m = message_module.TestAllTypes() - text_format.Parse(self._BYTES_SAMPLE, m) - self.assertEqual(m.optional_bytes, self._GOLDEN_BYTES) - self.assertEqual(m.optional_string, self._GOLDEN_UNICODE) - self.assertEqual(m.repeated_bytes[0], self._GOLDEN_BYTES) - # repeated_bytes[1] contained simple \ escaped non-UTF-8 raw binary data. - self.assertEqual(m.repeated_bytes[1], self._GOLDEN_BYTES_1) - # repeated_string[0] contained \ escaped data representing the UTF-8 - # representation of _GOLDEN_STR_0 - it needs to decode as such. - self.assertEqual(m.repeated_string[0], self._GOLDEN_STR_0) - - def testFromBytesFile(self, message_module): - m = message_module.TestAllTypes() - f = io.BytesIO(self._BYTES_SAMPLE) - text_format.ParseLines(f, m) - self.assertEqual(m.optional_bytes, self._GOLDEN_BYTES) - self.assertEqual(m.optional_string, self._GOLDEN_UNICODE) - self.assertEqual(m.repeated_bytes[0], self._GOLDEN_BYTES) - - def testFromUnicodeFile(self, message_module): - m = message_module.TestAllTypes() - f = io.StringIO(self._UNICODE_SAMPLE) - text_format.ParseLines(f, m) - self.assertEqual(m.optional_bytes, self._GOLDEN_BYTES) - self.assertEqual(m.optional_string, self._GOLDEN_UNICODE) - self.assertEqual(m.repeated_bytes[0], self._GOLDEN_BYTES) - - def testFromBytesLines(self, message_module): - m = message_module.TestAllTypes() - text_format.ParseLines(self._BYTES_SAMPLE.split(b'\n'), m) - self.assertEqual(m.optional_bytes, self._GOLDEN_BYTES) - self.assertEqual(m.optional_string, self._GOLDEN_UNICODE) - self.assertEqual(m.repeated_bytes[0], self._GOLDEN_BYTES) - - def testFromUnicodeLines(self, message_module): - m = message_module.TestAllTypes() - text_format.ParseLines(self._UNICODE_SAMPLE.split(u'\n'), m) - self.assertEqual(m.optional_bytes, self._GOLDEN_BYTES) - self.assertEqual(m.optional_string, self._GOLDEN_UNICODE) - self.assertEqual(m.repeated_bytes[0], self._GOLDEN_BYTES) - - def testParseDuplicateMessages(self, message_module): - message = message_module.TestAllTypes() - text = ('optional_nested_message { bb: 1 } ' - 'optional_nested_message { bb: 2 }') - self.assertRaisesRegex( - text_format.ParseError, - (r'1:59 : Message type "\w+.TestAllTypes" ' - r'should not have multiple "optional_nested_message" fields.'), - text_format.Parse, text, message) - - def testParseDuplicateScalars(self, message_module): - message = message_module.TestAllTypes() - text = ('optional_int32: 42 ' 'optional_int32: 67') - self.assertRaisesRegex( - text_format.ParseError, - (r'1:36 : Message type "\w+.TestAllTypes" should not ' - r'have multiple "optional_int32" fields.'), text_format.Parse, text, - message) - - def testParseExistingScalarInMessage(self, message_module): - message = message_module.TestAllTypes(optional_int32=42) - text = 'optional_int32: 67' - self.assertRaisesRegex(text_format.ParseError, - (r'Message type "\w+.TestAllTypes" should not ' - r'have multiple "optional_int32" fields.'), - text_format.Parse, text, message) - - -@_parameterized.parameters(unittest_pb2, unittest_proto3_arena_pb2) -class TextFormatMergeTests(TextFormatBase): - - def testMergeDuplicateScalarsInText(self, message_module): - message = message_module.TestAllTypes() - text = ('optional_int32: 42 ' 'optional_int32: 67') - r = text_format.Merge(text, message) - self.assertIs(r, message) - self.assertEqual(67, message.optional_int32) - - def testMergeDuplicateNestedMessageScalars(self, message_module): - message = message_module.TestAllTypes() - text = ('optional_nested_message { bb: 1 } ' - 'optional_nested_message { bb: 2 }') - r = text_format.Merge(text, message) - self.assertTrue(r is message) - self.assertEqual(2, message.optional_nested_message.bb) - - def testReplaceScalarInMessage(self, message_module): - message = message_module.TestAllTypes(optional_int32=42) - text = 'optional_int32: 67' - r = text_format.Merge(text, message) - self.assertIs(r, message) - self.assertEqual(67, message.optional_int32) - - def testReplaceMessageInMessage(self, message_module): - message = message_module.TestAllTypes( - optional_int32=42, optional_nested_message=dict()) - self.assertTrue(message.HasField('optional_nested_message')) - text = 'optional_nested_message{ bb: 3 }' - r = text_format.Merge(text, message) - self.assertIs(r, message) - self.assertEqual(3, message.optional_nested_message.bb) - - def testMergeMultipleOneof(self, message_module): - m_string = '\n'.join(['oneof_uint32: 11', 'oneof_string: "foo"']) - m2 = message_module.TestAllTypes() - text_format.Merge(m_string, m2) - self.assertEqual('oneof_string', m2.WhichOneof('oneof_field')) - - -# These are tests that aren't fundamentally specific to proto2, but are at -# the moment because of differences between the proto2 and proto3 test schemas. -# Ideally the schemas would be made more similar so these tests could pass. -class OnlyWorksWithProto2RightNowTests(TextFormatBase): - - def testPrintAllFieldsPointy(self): - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString( - message, pointy_brackets=True)), - 'text_format_unittest_data_pointy_oneof.txt') - - def testParseGolden(self): - golden_text = '\n'.join(self.ReadGolden( - 'text_format_unittest_data_oneof_implemented.txt')) - parsed_message = unittest_pb2.TestAllTypes() - r = text_format.Parse(golden_text, parsed_message) - self.assertIs(r, parsed_message) - - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.assertEqual(message, parsed_message) - - def testPrintAllFields(self): - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'text_format_unittest_data_oneof_implemented.txt') - - def testPrintUnknownFields(self): - message = unittest_pb2.TestAllTypes() - message.optional_int32 = 101 - message.optional_double = 102.0 - message.optional_string = u'hello' - message.optional_bytes = b'103' - message.optionalgroup.a = 104 - message.optional_nested_message.bb = 105 - all_data = message.SerializeToString() - empty_message = unittest_pb2.TestEmptyMessage() - empty_message.ParseFromString(all_data) - self.assertEqual(' 1: 101\n' - ' 12: 4636878028842991616\n' - ' 14: "hello"\n' - ' 15: "103"\n' - ' 16 {\n' - ' 17: 104\n' - ' }\n' - ' 18 {\n' - ' 1: 105\n' - ' }\n', - text_format.MessageToString(empty_message, - indent=2, - print_unknown_fields=True)) - self.assertEqual('1: 101 ' - '12: 4636878028842991616 ' - '14: "hello" ' - '15: "103" ' - '16 { 17: 104 } ' - '18 { 1: 105 }', - text_format.MessageToString(empty_message, - print_unknown_fields=True, - as_one_line=True)) - - def testPrintInIndexOrder(self): - message = unittest_pb2.TestFieldOrderings() - # Fields are listed in index order instead of field number. - message.my_string = 'str' - message.my_int = 101 - message.my_float = 111 - message.optional_nested_message.oo = 0 - message.optional_nested_message.bb = 1 - message.Extensions[unittest_pb2.my_extension_string] = 'ext_str0' - # Extensions are listed based on the order of extension number. - # Extension number 12. - message.Extensions[unittest_pb2.TestExtensionOrderings2. - test_ext_orderings2].my_string = 'ext_str2' - # Extension number 13. - message.Extensions[unittest_pb2.TestExtensionOrderings1. - test_ext_orderings1].my_string = 'ext_str1' - # Extension number 14. - message.Extensions[ - unittest_pb2.TestExtensionOrderings2.TestExtensionOrderings3. - test_ext_orderings3].my_string = 'ext_str3' - - # Print in index order. - self.CompareToGoldenText( - self.RemoveRedundantZeros( - text_format.MessageToString(message, use_index_order=True)), - 'my_string: "str"\n' - 'my_int: 101\n' - 'my_float: 111\n' - 'optional_nested_message {\n' - ' oo: 0\n' - ' bb: 1\n' - '}\n' - '[protobuf_unittest.TestExtensionOrderings2.test_ext_orderings2] {\n' - ' my_string: "ext_str2"\n' - '}\n' - '[protobuf_unittest.TestExtensionOrderings1.test_ext_orderings1] {\n' - ' my_string: "ext_str1"\n' - '}\n' - '[protobuf_unittest.TestExtensionOrderings2.TestExtensionOrderings3' - '.test_ext_orderings3] {\n' - ' my_string: "ext_str3"\n' - '}\n' - '[protobuf_unittest.my_extension_string]: "ext_str0"\n') - # By default, print in field number order. - self.CompareToGoldenText( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'my_int: 101\n' - 'my_string: "str"\n' - '[protobuf_unittest.TestExtensionOrderings2.test_ext_orderings2] {\n' - ' my_string: "ext_str2"\n' - '}\n' - '[protobuf_unittest.TestExtensionOrderings1.test_ext_orderings1] {\n' - ' my_string: "ext_str1"\n' - '}\n' - '[protobuf_unittest.TestExtensionOrderings2.TestExtensionOrderings3' - '.test_ext_orderings3] {\n' - ' my_string: "ext_str3"\n' - '}\n' - '[protobuf_unittest.my_extension_string]: "ext_str0"\n' - 'my_float: 111\n' - 'optional_nested_message {\n' - ' bb: 1\n' - ' oo: 0\n' - '}\n') - - def testMergeLinesGolden(self): - opened = self.ReadGolden('text_format_unittest_data_oneof_implemented.txt') - parsed_message = unittest_pb2.TestAllTypes() - r = text_format.MergeLines(opened, parsed_message) - self.assertIs(r, parsed_message) - - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.assertEqual(message, parsed_message) - - def testParseLinesGolden(self): - opened = self.ReadGolden('text_format_unittest_data_oneof_implemented.txt') - parsed_message = unittest_pb2.TestAllTypes() - r = text_format.ParseLines(opened, parsed_message) - self.assertIs(r, parsed_message) - - message = unittest_pb2.TestAllTypes() - test_util.SetAllFields(message) - self.assertEqual(message, parsed_message) - - def testPrintMap(self): - message = map_unittest_pb2.TestMap() - - message.map_int32_int32[-123] = -456 - message.map_int64_int64[-2**33] = -2**34 - message.map_uint32_uint32[123] = 456 - message.map_uint64_uint64[2**33] = 2**34 - message.map_string_string['abc'] = '123' - message.map_int32_foreign_message[111].c = 5 - - # Maps are serialized to text format using their underlying repeated - # representation. - self.CompareToGoldenText( - text_format.MessageToString(message), 'map_int32_int32 {\n' - ' key: -123\n' - ' value: -456\n' - '}\n' - 'map_int64_int64 {\n' - ' key: -8589934592\n' - ' value: -17179869184\n' - '}\n' - 'map_uint32_uint32 {\n' - ' key: 123\n' - ' value: 456\n' - '}\n' - 'map_uint64_uint64 {\n' - ' key: 8589934592\n' - ' value: 17179869184\n' - '}\n' - 'map_string_string {\n' - ' key: "abc"\n' - ' value: "123"\n' - '}\n' - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' c: 5\n' - ' }\n' - '}\n') - - def testDuplicateMapKey(self): - message = map_unittest_pb2.TestMap() - text = ( - 'map_uint64_uint64 {\n' - ' key: 123\n' - ' value: 17179869184\n' - '}\n' - 'map_string_string {\n' - ' key: "abc"\n' - ' value: "first"\n' - '}\n' - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' c: 5\n' - ' }\n' - '}\n' - 'map_uint64_uint64 {\n' - ' key: 123\n' - ' value: 321\n' - '}\n' - 'map_string_string {\n' - ' key: "abc"\n' - ' value: "second"\n' - '}\n' - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' d: 5\n' - ' }\n' - '}\n') - text_format.Parse(text, message) - self.CompareToGoldenText( - text_format.MessageToString(message), 'map_uint64_uint64 {\n' - ' key: 123\n' - ' value: 321\n' - '}\n' - 'map_string_string {\n' - ' key: "abc"\n' - ' value: "second"\n' - '}\n' - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' d: 5\n' - ' }\n' - '}\n') - - # In cpp implementation, __str__ calls the cpp implementation of text format. - def testPrintMapUsingCppImplementation(self): - message = map_unittest_pb2.TestMap() - inner_msg = message.map_int32_foreign_message[111] - inner_msg.c = 1 - self.assertEqual( - str(message), - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' c: 1\n' - ' }\n' - '}\n') - inner_msg.c = 2 - self.assertEqual( - str(message), - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' c: 2\n' - ' }\n' - '}\n') - - def testMapOrderEnforcement(self): - message = map_unittest_pb2.TestMap() - for letter in string.ascii_uppercase[13:26]: - message.map_string_string[letter] = 'dummy' - for letter in reversed(string.ascii_uppercase[0:13]): - message.map_string_string[letter] = 'dummy' - golden = ''.join(('map_string_string {\n key: "%c"\n value: "dummy"\n}\n' - % (letter,) for letter in string.ascii_uppercase)) - self.CompareToGoldenText(text_format.MessageToString(message), golden) - - # TODO(teboring): In c/137553523, not serializing default value for map entry - # message has been fixed. This test needs to be disabled in order to submit - # that cl. Add this back when c/137553523 has been submitted. - # def testMapOrderSemantics(self): - # golden_lines = self.ReadGolden('map_test_data.txt') - - # message = map_unittest_pb2.TestMap() - # text_format.ParseLines(golden_lines, message) - # candidate = text_format.MessageToString(message) - # # The Python implementation emits "1.0" for the double value that the C++ - # # implementation emits as "1". - # candidate = candidate.replace('1.0', '1', 2) - # candidate = candidate.replace('0.0', '0', 2) - # self.assertMultiLineEqual(candidate, ''.join(golden_lines)) - - -# Tests of proto2-only features (MessageSet, extensions, etc.). -class Proto2Tests(TextFormatBase): - - def testPrintMessageSet(self): - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - self.CompareToGoldenText( - text_format.MessageToString(message), 'message_set {\n' - ' [protobuf_unittest.TestMessageSetExtension1] {\n' - ' i: 23\n' - ' }\n' - ' [protobuf_unittest.TestMessageSetExtension2] {\n' - ' str: \"foo\"\n' - ' }\n' - '}\n') - - message = message_set_extensions_pb2.TestMessageSet() - ext = message_set_extensions_pb2.message_set_extension3 - message.Extensions[ext].text = 'bar' - self.CompareToGoldenText( - text_format.MessageToString(message), - '[google.protobuf.internal.TestMessageSetExtension3] {\n' - ' text: \"bar\"\n' - '}\n') - - def testPrintMessageSetByFieldNumber(self): - out = text_format.TextWriter(False) - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - text_format.PrintMessage(message, out, use_field_number=True) - self.CompareToGoldenText(out.getvalue(), '1 {\n' - ' 1545008 {\n' - ' 15: 23\n' - ' }\n' - ' 1547769 {\n' - ' 25: \"foo\"\n' - ' }\n' - '}\n') - out.close() - - def testPrintMessageSetAsOneLine(self): - message = unittest_mset_pb2.TestMessageSetContainer() - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - message.message_set.Extensions[ext1].i = 23 - message.message_set.Extensions[ext2].str = 'foo' - self.CompareToGoldenText( - text_format.MessageToString(message, as_one_line=True), - 'message_set {' - ' [protobuf_unittest.TestMessageSetExtension1] {' - ' i: 23' - ' }' - ' [protobuf_unittest.TestMessageSetExtension2] {' - ' str: \"foo\"' - ' }' - ' }') - - def testParseMessageSet(self): - message = unittest_pb2.TestAllTypes() - text = ('repeated_uint64: 1\n' 'repeated_uint64: 2\n') - text_format.Parse(text, message) - self.assertEqual(1, message.repeated_uint64[0]) - self.assertEqual(2, message.repeated_uint64[1]) - - message = unittest_mset_pb2.TestMessageSetContainer() - text = ('message_set {\n' - ' [protobuf_unittest.TestMessageSetExtension1] {\n' - ' i: 23\n' - ' }\n' - ' [protobuf_unittest.TestMessageSetExtension2] {\n' - ' str: \"foo\"\n' - ' }\n' - '}\n') - text_format.Parse(text, message) - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - self.assertEqual(23, message.message_set.Extensions[ext1].i) - self.assertEqual('foo', message.message_set.Extensions[ext2].str) - - def testExtensionInsideAnyMessage(self): - message = test_extend_any.TestAny() - text = ('value {\n' - ' [type.googleapis.com/google.protobuf.internal.TestAny] {\n' - ' [google.protobuf.internal.TestAnyExtension1.extension1] {\n' - ' i: 10\n' - ' }\n' - ' }\n' - '}\n') - text_format.Merge(text, message, descriptor_pool=descriptor_pool.Default()) - self.CompareToGoldenText( - text_format.MessageToString( - message, descriptor_pool=descriptor_pool.Default()), - text) - - def testParseMessageByFieldNumber(self): - message = unittest_pb2.TestAllTypes() - text = ('34: 1\n' 'repeated_uint64: 2\n') - text_format.Parse(text, message, allow_field_number=True) - self.assertEqual(1, message.repeated_uint64[0]) - self.assertEqual(2, message.repeated_uint64[1]) - - message = unittest_mset_pb2.TestMessageSetContainer() - text = ('1 {\n' - ' 1545008 {\n' - ' 15: 23\n' - ' }\n' - ' 1547769 {\n' - ' 25: \"foo\"\n' - ' }\n' - '}\n') - text_format.Parse(text, message, allow_field_number=True) - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - self.assertEqual(23, message.message_set.Extensions[ext1].i) - self.assertEqual('foo', message.message_set.Extensions[ext2].str) - - # Can't parse field number without set allow_field_number=True. - message = unittest_pb2.TestAllTypes() - text = '34:1\n' - self.assertRaisesRegex( - text_format.ParseError, - (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' - r'"34".'), text_format.Parse, text, message) - - # Can't parse if field number is not found. - text = '1234:1\n' - self.assertRaisesRegex( - text_format.ParseError, - (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' - r'"1234".'), - text_format.Parse, - text, - message, - allow_field_number=True) - - def testPrintAllExtensions(self): - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString(message)), - 'text_format_unittest_extensions_data.txt') - - def testPrintAllExtensionsPointy(self): - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - self.CompareToGoldenFile( - self.RemoveRedundantZeros(text_format.MessageToString( - message, pointy_brackets=True)), - 'text_format_unittest_extensions_data_pointy.txt') - - def testParseGoldenExtensions(self): - golden_text = '\n'.join(self.ReadGolden( - 'text_format_unittest_extensions_data.txt')) - parsed_message = unittest_pb2.TestAllExtensions() - text_format.Parse(golden_text, parsed_message) - - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - self.assertEqual(message, parsed_message) - - def testParseAllExtensions(self): - message = unittest_pb2.TestAllExtensions() - test_util.SetAllExtensions(message) - ascii_text = text_format.MessageToString(message) - - parsed_message = unittest_pb2.TestAllExtensions() - text_format.Parse(ascii_text, parsed_message) - self.assertEqual(message, parsed_message) - - def testParseAllowedUnknownExtension(self): - # Skip over unknown extension correctly. - message = unittest_mset_pb2.TestMessageSetContainer() - text = ('message_set {\n' - ' [unknown_extension] {\n' - ' i: 23\n' - ' repeated_i: []\n' - ' bin: "\xe0"\n' - ' [nested_unknown_ext]: {\n' - ' i: 23\n' - ' repeated_i: [1, 2]\n' - ' x: x\n' - ' test: "test_string"\n' - ' floaty_float: -0.315\n' - ' num: -inf\n' - ' multiline_str: "abc"\n' - ' "def"\n' - ' "xyz."\n' - ' [nested_unknown_ext.ext]: <\n' - ' i: 23\n' - ' i: 24\n' - ' pointfloat: .3\n' - ' test: "test_string"\n' - ' repeated_test: ["test_string1", "test_string2"]\n' - ' floaty_float: -0.315\n' - ' num: -inf\n' - ' long_string: "test" "test2" \n' - ' >\n' - ' }\n' - ' }\n' - ' [unknown_extension]: 5\n' - ' [unknown_extension_with_number_field] {\n' - ' 1: "some_field"\n' - ' 2: -0.451\n' - ' }\n' - '}\n') - text_format.Parse(text, message, allow_unknown_extension=True) - golden = 'message_set {\n}\n' - self.CompareToGoldenText(text_format.MessageToString(message), golden) - - # Catch parse errors in unknown extension. - message = unittest_mset_pb2.TestMessageSetContainer() - malformed = ('message_set {\n' - ' [unknown_extension] {\n' - ' i:\n' # Missing value. - ' }\n' - '}\n') - self.assertRaisesRegex( - text_format.ParseError, - 'Invalid field value: }', - text_format.Parse, - malformed, - message, - allow_unknown_extension=True) - - message = unittest_mset_pb2.TestMessageSetContainer() - malformed = ('message_set {\n' - ' [unknown_extension] {\n' - ' str: "malformed string\n' # Missing closing quote. - ' }\n' - '}\n') - self.assertRaisesRegex( - text_format.ParseError, - 'Invalid field value: "', - text_format.Parse, - malformed, - message, - allow_unknown_extension=True) - - message = unittest_mset_pb2.TestMessageSetContainer() - malformed = ('message_set {\n' - ' [unknown_extension] {\n' - ' str: "malformed\n multiline\n string\n' - ' }\n' - '}\n') - self.assertRaisesRegex( - text_format.ParseError, - 'Invalid field value: "', - text_format.Parse, - malformed, - message, - allow_unknown_extension=True) - - message = unittest_mset_pb2.TestMessageSetContainer() - malformed = ('message_set {\n' - ' [malformed_extension] <\n' - ' i: -5\n' - ' \n' # Missing '>' here. - '}\n') - self.assertRaisesRegex( - text_format.ParseError, - '5:1 : \'}\': Expected ">".', - text_format.Parse, - malformed, - message, - allow_unknown_extension=True) - - # Don't allow unknown fields with allow_unknown_extension=True. - message = unittest_mset_pb2.TestMessageSetContainer() - malformed = ('message_set {\n' - ' unknown_field: true\n' - '}\n') - self.assertRaisesRegex( - text_format.ParseError, - ('2:3 : Message type ' - '"proto2_wireformat_unittest.TestMessageSet" has no' - ' field named "unknown_field".'), - text_format.Parse, - malformed, - message, - allow_unknown_extension=True) - - # Parse known extension correctly. - message = unittest_mset_pb2.TestMessageSetContainer() - text = ('message_set {\n' - ' [protobuf_unittest.TestMessageSetExtension1] {\n' - ' i: 23\n' - ' }\n' - ' [protobuf_unittest.TestMessageSetExtension2] {\n' - ' str: \"foo\"\n' - ' }\n' - '}\n') - text_format.Parse(text, message, allow_unknown_extension=True) - ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension - ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension - self.assertEqual(23, message.message_set.Extensions[ext1].i) - self.assertEqual('foo', message.message_set.Extensions[ext2].str) - - def testParseBadIdentifier(self): - message = unittest_pb2.TestAllTypes() - text = ('optional_nested_message { "bb": 1 }') - with self.assertRaises(text_format.ParseError) as e: - text_format.Parse(text, message) - self.assertEqual(str(e.exception), - '1:27 : \'optional_nested_message { "bb": 1 }\': ' - 'Expected identifier or number, got "bb".') - - def testParseBadExtension(self): - message = unittest_pb2.TestAllExtensions() - text = '[unknown_extension]: 8\n' - self.assertRaisesRegex( - text_format.ParseError, - '1:2 : Extension "unknown_extension" not registered.', - text_format.Parse, text, message) - message = unittest_pb2.TestAllTypes() - self.assertRaisesRegex( - text_format.ParseError, - ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' - 'extensions.'), text_format.Parse, text, message) - - def testParseNumericUnknownEnum(self): - message = unittest_pb2.TestAllTypes() - text = 'optional_nested_enum: 100' - self.assertRaisesRegex(text_format.ParseError, - (r'1:23 : \'optional_nested_enum: 100\': ' - r'Enum type "\w+.TestAllTypes.NestedEnum" ' - r'has no value with number 100.'), - text_format.Parse, text, message) - - def testMergeDuplicateExtensionScalars(self): - message = unittest_pb2.TestAllExtensions() - text = ('[protobuf_unittest.optional_int32_extension]: 42 ' - '[protobuf_unittest.optional_int32_extension]: 67') - text_format.Merge(text, message) - self.assertEqual(67, - message.Extensions[unittest_pb2.optional_int32_extension]) - - def testParseDuplicateExtensionScalars(self): - message = unittest_pb2.TestAllExtensions() - text = ('[protobuf_unittest.optional_int32_extension]: 42 ' - '[protobuf_unittest.optional_int32_extension]: 67') - self.assertRaisesRegex( - text_format.ParseError, - ('1:96 : Message type "protobuf_unittest.TestAllExtensions" ' - 'should not have multiple ' - '"protobuf_unittest.optional_int32_extension" extensions.'), - text_format.Parse, text, message) - - def testParseDuplicateExtensionMessages(self): - message = unittest_pb2.TestAllExtensions() - text = ('[protobuf_unittest.optional_nested_message_extension]: {} ' - '[protobuf_unittest.optional_nested_message_extension]: {}') - self.assertRaisesRegex( - text_format.ParseError, - ('1:114 : Message type "protobuf_unittest.TestAllExtensions" ' - 'should not have multiple ' - '"protobuf_unittest.optional_nested_message_extension" extensions.'), - text_format.Parse, text, message) - - def testParseGroupNotClosed(self): - message = unittest_pb2.TestAllTypes() - text = 'RepeatedGroup: <' - self.assertRaisesRegex(text_format.ParseError, '1:16 : Expected ">".', - text_format.Parse, text, message) - text = 'RepeatedGroup: {' - self.assertRaisesRegex(text_format.ParseError, '1:16 : Expected "}".', - text_format.Parse, text, message) - - def testParseEmptyGroup(self): - message = unittest_pb2.TestAllTypes() - text = 'OptionalGroup: {}' - text_format.Parse(text, message) - self.assertTrue(message.HasField('optionalgroup')) - - message.Clear() - - message = unittest_pb2.TestAllTypes() - text = 'OptionalGroup: <>' - text_format.Parse(text, message) - self.assertTrue(message.HasField('optionalgroup')) - - # Maps aren't really proto2-only, but our test schema only has maps for - # proto2. - def testParseMap(self): - text = ('map_int32_int32 {\n' - ' key: -123\n' - ' value: -456\n' - '}\n' - 'map_int64_int64 {\n' - ' key: -8589934592\n' - ' value: -17179869184\n' - '}\n' - 'map_uint32_uint32 {\n' - ' key: 123\n' - ' value: 456\n' - '}\n' - 'map_uint64_uint64 {\n' - ' key: 8589934592\n' - ' value: 17179869184\n' - '}\n' - 'map_string_string {\n' - ' key: "abc"\n' - ' value: "123"\n' - '}\n' - 'map_int32_foreign_message {\n' - ' key: 111\n' - ' value {\n' - ' c: 5\n' - ' }\n' - '}\n') - message = map_unittest_pb2.TestMap() - text_format.Parse(text, message) - - self.assertEqual(-456, message.map_int32_int32[-123]) - self.assertEqual(-2**34, message.map_int64_int64[-2**33]) - self.assertEqual(456, message.map_uint32_uint32[123]) - self.assertEqual(2**34, message.map_uint64_uint64[2**33]) - self.assertEqual('123', message.map_string_string['abc']) - self.assertEqual(5, message.map_int32_foreign_message[111].c) - - -class Proto3Tests(unittest.TestCase): - - def testPrintMessageExpandAny(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - self.assertEqual( - text_format.MessageToString(message, - descriptor_pool=descriptor_pool.Default()), - 'any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string"\n' - ' }\n' - '}\n') - - def testTopAnyMessage(self): - packed_msg = unittest_pb2.OneString() - msg = any_pb2.Any() - msg.Pack(packed_msg) - text = text_format.MessageToString(msg) - other_msg = text_format.Parse(text, any_pb2.Any()) - self.assertEqual(msg, other_msg) - - def testPrintMessageExpandAnyRepeated(self): - packed_message = unittest_pb2.OneString() - message = any_test_pb2.TestAny() - packed_message.data = 'string0' - message.repeated_any_value.add().Pack(packed_message) - packed_message.data = 'string1' - message.repeated_any_value.add().Pack(packed_message) - self.assertEqual( - text_format.MessageToString(message), - 'repeated_any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string0"\n' - ' }\n' - '}\n' - 'repeated_any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string1"\n' - ' }\n' - '}\n') - - def testPrintMessageExpandAnyDescriptorPoolMissingType(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - empty_pool = descriptor_pool.DescriptorPool() - self.assertEqual( - text_format.MessageToString(message, descriptor_pool=empty_pool), - 'any_value {\n' - ' type_url: "type.googleapis.com/protobuf_unittest.OneString"\n' - ' value: "\\n\\006string"\n' - '}\n') - - def testPrintMessageExpandAnyPointyBrackets(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - self.assertEqual( - text_format.MessageToString(message, - pointy_brackets=True), - 'any_value <\n' - ' [type.googleapis.com/protobuf_unittest.OneString] <\n' - ' data: "string"\n' - ' >\n' - '>\n') - - def testPrintMessageExpandAnyAsOneLine(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - self.assertEqual( - text_format.MessageToString(message, - as_one_line=True), - 'any_value {' - ' [type.googleapis.com/protobuf_unittest.OneString]' - ' { data: "string" } ' - '}') - - def testPrintMessageExpandAnyAsOneLinePointyBrackets(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - self.assertEqual( - text_format.MessageToString(message, - as_one_line=True, - pointy_brackets=True, - descriptor_pool=descriptor_pool.Default()), - 'any_value <' - ' [type.googleapis.com/protobuf_unittest.OneString]' - ' < data: "string" > ' - '>') - - def testPrintAndParseMessageInvalidAny(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - # Only include string after last '/' in type_url. - message.any_value.type_url = message.any_value.TypeName() - text = text_format.MessageToString(message) - self.assertEqual( - text, 'any_value {\n' - ' type_url: "protobuf_unittest.OneString"\n' - ' value: "\\n\\006string"\n' - '}\n') - - parsed_message = any_test_pb2.TestAny() - text_format.Parse(text, parsed_message) - self.assertEqual(message, parsed_message) - - def testUnknownEnums(self): - message = unittest_proto3_arena_pb2.TestAllTypes() - message2 = unittest_proto3_arena_pb2.TestAllTypes() - message.optional_nested_enum = 999 - text_string = text_format.MessageToString(message) - text_format.Parse(text_string, message2) - self.assertEqual(999, message2.optional_nested_enum) - - def testMergeExpandedAny(self): - message = any_test_pb2.TestAny() - text = ('any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string"\n' - ' }\n' - '}\n') - text_format.Merge(text, message) - packed_message = unittest_pb2.OneString() - message.any_value.Unpack(packed_message) - self.assertEqual('string', packed_message.data) - message.Clear() - text_format.Parse(text, message) - packed_message = unittest_pb2.OneString() - message.any_value.Unpack(packed_message) - self.assertEqual('string', packed_message.data) - - def testMergeExpandedAnyRepeated(self): - message = any_test_pb2.TestAny() - text = ('repeated_any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string0"\n' - ' }\n' - '}\n' - 'repeated_any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string1"\n' - ' }\n' - '}\n') - text_format.Merge(text, message) - packed_message = unittest_pb2.OneString() - message.repeated_any_value[0].Unpack(packed_message) - self.assertEqual('string0', packed_message.data) - message.repeated_any_value[1].Unpack(packed_message) - self.assertEqual('string1', packed_message.data) - - def testMergeExpandedAnyPointyBrackets(self): - message = any_test_pb2.TestAny() - text = ('any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] <\n' - ' data: "string"\n' - ' >\n' - '}\n') - text_format.Merge(text, message) - packed_message = unittest_pb2.OneString() - message.any_value.Unpack(packed_message) - self.assertEqual('string', packed_message.data) - - def testMergeAlternativeUrl(self): - message = any_test_pb2.TestAny() - text = ('any_value {\n' - ' [type.otherapi.com/protobuf_unittest.OneString] {\n' - ' data: "string"\n' - ' }\n' - '}\n') - text_format.Merge(text, message) - packed_message = unittest_pb2.OneString() - self.assertEqual('type.otherapi.com/protobuf_unittest.OneString', - message.any_value.type_url) - - def testMergeExpandedAnyDescriptorPoolMissingType(self): - message = any_test_pb2.TestAny() - text = ('any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string"\n' - ' }\n' - '}\n') - with self.assertRaises(text_format.ParseError) as e: - empty_pool = descriptor_pool.DescriptorPool() - text_format.Merge(text, message, descriptor_pool=empty_pool) - self.assertEqual( - str(e.exception), - 'Type protobuf_unittest.OneString not found in descriptor pool') - - def testMergeUnexpandedAny(self): - text = ('any_value {\n' - ' type_url: "type.googleapis.com/protobuf_unittest.OneString"\n' - ' value: "\\n\\006string"\n' - '}\n') - message = any_test_pb2.TestAny() - text_format.Merge(text, message) - packed_message = unittest_pb2.OneString() - message.any_value.Unpack(packed_message) - self.assertEqual('string', packed_message.data) - - def testMergeMissingAnyEndToken(self): - message = any_test_pb2.TestAny() - text = ('any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' - ' data: "string"\n') - with self.assertRaises(text_format.ParseError) as e: - text_format.Merge(text, message) - self.assertEqual(str(e.exception), '3:11 : Expected "}".') - - def testParseExpandedAnyListValue(self): - any_msg = any_pb2.Any() - any_msg.Pack(struct_pb2.ListValue()) - msg = any_test_pb2.TestAny(any_value=any_msg) - text = ('any_value {\n' - ' [type.googleapis.com/google.protobuf.ListValue] {}\n' - '}\n') - parsed_msg = text_format.Parse(text, any_test_pb2.TestAny()) - self.assertEqual(msg, parsed_msg) - - def testProto3Optional(self): - msg = test_proto3_optional_pb2.TestProto3Optional() - self.assertEqual(text_format.MessageToString(msg), '') - msg.optional_int32 = 0 - msg.optional_float = 0.0 - msg.optional_string = '' - msg.optional_nested_message.bb = 0 - text = ('optional_int32: 0\n' - 'optional_float: 0.0\n' - 'optional_string: ""\n' - 'optional_nested_message {\n' - ' bb: 0\n' - '}\n') - self.assertEqual(text_format.MessageToString(msg), text) - msg2 = test_proto3_optional_pb2.TestProto3Optional() - text_format.Parse(text, msg2) - self.assertEqual(text_format.MessageToString(msg2), text) - - -class TokenizerTest(unittest.TestCase): - - def testSimpleTokenCases(self): - text = ('identifier1:"string1"\n \n\n' - 'identifier2 : \n \n123 \n identifier3 :\'string\'\n' - 'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:\'aaaa\\\'bbbb\'\n' - 'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n' - 'ID9: 22 ID10: -111111111111111111 ID11: -22\n' - 'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f ' - 'false_bool: 0 true_BOOL:t \n true_bool1: 1 false_BOOL1:f ' - 'False_bool: False True_bool: True X:iNf Y:-inF Z:nAN') - tokenizer = text_format.Tokenizer(text.splitlines()) - methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), ':', - (tokenizer.ConsumeString, 'string1'), - (tokenizer.ConsumeIdentifier, 'identifier2'), ':', - (tokenizer.ConsumeInteger, 123), - (tokenizer.ConsumeIdentifier, 'identifier3'), ':', - (tokenizer.ConsumeString, 'string'), - (tokenizer.ConsumeIdentifier, 'identifiER_4'), ':', - (tokenizer.ConsumeFloat, 1.1e+2), - (tokenizer.ConsumeIdentifier, 'ID5'), ':', - (tokenizer.ConsumeFloat, -0.23), - (tokenizer.ConsumeIdentifier, 'ID6'), ':', - (tokenizer.ConsumeString, 'aaaa\'bbbb'), - (tokenizer.ConsumeIdentifier, 'ID7'), ':', - (tokenizer.ConsumeString, 'aa\"bb'), - (tokenizer.ConsumeIdentifier, 'ID8'), ':', '{', - (tokenizer.ConsumeIdentifier, 'A'), ':', - (tokenizer.ConsumeFloat, float('inf')), - (tokenizer.ConsumeIdentifier, 'B'), ':', - (tokenizer.ConsumeFloat, -float('inf')), - (tokenizer.ConsumeIdentifier, 'C'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'D'), ':', - (tokenizer.ConsumeBool, False), '}', - (tokenizer.ConsumeIdentifier, 'ID9'), ':', - (tokenizer.ConsumeInteger, 22), - (tokenizer.ConsumeIdentifier, 'ID10'), ':', - (tokenizer.ConsumeInteger, -111111111111111111), - (tokenizer.ConsumeIdentifier, 'ID11'), ':', - (tokenizer.ConsumeInteger, -22), - (tokenizer.ConsumeIdentifier, 'ID12'), ':', - (tokenizer.ConsumeInteger, 2222222222222222222), - (tokenizer.ConsumeIdentifier, 'ID13'), ':', - (tokenizer.ConsumeFloat, 1.23456), - (tokenizer.ConsumeIdentifier, 'ID14'), ':', - (tokenizer.ConsumeFloat, 1.2e+2), - (tokenizer.ConsumeIdentifier, 'false_bool'), ':', - (tokenizer.ConsumeBool, False), - (tokenizer.ConsumeIdentifier, 'true_BOOL'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'true_bool1'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'false_BOOL1'), ':', - (tokenizer.ConsumeBool, False), - (tokenizer.ConsumeIdentifier, 'False_bool'), ':', - (tokenizer.ConsumeBool, False), - (tokenizer.ConsumeIdentifier, 'True_bool'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'X'), ':', - (tokenizer.ConsumeFloat, float('inf')), - (tokenizer.ConsumeIdentifier, 'Y'), ':', - (tokenizer.ConsumeFloat, float('-inf')), - (tokenizer.ConsumeIdentifier, 'Z'), ':', - (tokenizer.ConsumeFloat, float('nan'))] - - i = 0 - while not tokenizer.AtEnd(): - m = methods[i] - if isinstance(m, str): - token = tokenizer.token - self.assertEqual(token, m) - tokenizer.NextToken() - elif isinstance(m[1], float) and math.isnan(m[1]): - self.assertTrue(math.isnan(m[0]())) - else: - self.assertEqual(m[1], m[0]()) - i += 1 - - def testConsumeAbstractIntegers(self): - # This test only tests the failures in the integer parsing methods as well - # as the '0' special cases. - int64_max = (1 << 63) - 1 - uint32_max = (1 << 32) - 1 - text = '-1 %d %d' % (uint32_max + 1, int64_max + 1) - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertEqual(-1, tokenizer.ConsumeInteger()) - - self.assertEqual(uint32_max + 1, tokenizer.ConsumeInteger()) - - self.assertEqual(int64_max + 1, tokenizer.ConsumeInteger()) - self.assertTrue(tokenizer.AtEnd()) - - text = '-0 0 0 1.2' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertEqual(0, tokenizer.ConsumeInteger()) - self.assertEqual(0, tokenizer.ConsumeInteger()) - self.assertEqual(True, tokenizer.TryConsumeInteger()) - self.assertEqual(False, tokenizer.TryConsumeInteger()) - with self.assertRaises(text_format.ParseError): - tokenizer.ConsumeInteger() - self.assertEqual(1.2, tokenizer.ConsumeFloat()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeIntegers(self): - # This test only tests the failures in the integer parsing methods as well - # as the '0' special cases. - int64_max = (1 << 63) - 1 - uint32_max = (1 << 32) - 1 - text = '-1 %d %d' % (uint32_max + 1, int64_max + 1) - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, - text_format._ConsumeUint32, tokenizer) - self.assertRaises(text_format.ParseError, - text_format._ConsumeUint64, tokenizer) - self.assertEqual(-1, text_format._ConsumeInt32(tokenizer)) - - self.assertRaises(text_format.ParseError, - text_format._ConsumeUint32, tokenizer) - self.assertRaises(text_format.ParseError, - text_format._ConsumeInt32, tokenizer) - self.assertEqual(uint32_max + 1, text_format._ConsumeInt64(tokenizer)) - - self.assertRaises(text_format.ParseError, - text_format._ConsumeInt64, tokenizer) - self.assertEqual(int64_max + 1, text_format._ConsumeUint64(tokenizer)) - self.assertTrue(tokenizer.AtEnd()) - - text = '-0 -0 0 0' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertEqual(0, text_format._ConsumeUint32(tokenizer)) - self.assertEqual(0, text_format._ConsumeUint64(tokenizer)) - self.assertEqual(0, text_format._ConsumeUint32(tokenizer)) - self.assertEqual(0, text_format._ConsumeUint64(tokenizer)) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeOctalIntegers(self): - """Test support for C style octal integers.""" - text = '00 -00 04 0755 -010 007 -0033 08 -09 01' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertEqual(0, tokenizer.ConsumeInteger()) - self.assertEqual(0, tokenizer.ConsumeInteger()) - self.assertEqual(4, tokenizer.ConsumeInteger()) - self.assertEqual(0o755, tokenizer.ConsumeInteger()) - self.assertEqual(-0o10, tokenizer.ConsumeInteger()) - self.assertEqual(7, tokenizer.ConsumeInteger()) - self.assertEqual(-0o033, tokenizer.ConsumeInteger()) - with self.assertRaises(text_format.ParseError): - tokenizer.ConsumeInteger() # 08 - tokenizer.NextToken() - with self.assertRaises(text_format.ParseError): - tokenizer.ConsumeInteger() # -09 - tokenizer.NextToken() - self.assertEqual(1, tokenizer.ConsumeInteger()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeByteString(self): - text = '"string1\'' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) - - text = 'string1"' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) - - text = '\n"\\xt"' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) - - text = '\n"\\"' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) - - text = '\n"\\x"' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString) - - def testConsumeBool(self): - text = 'not-a-bool' - tokenizer = text_format.Tokenizer(text.splitlines()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool) - - def testSkipComment(self): - tokenizer = text_format.Tokenizer('# some comment'.splitlines()) - self.assertTrue(tokenizer.AtEnd()) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment) - - def testConsumeComment(self): - tokenizer = text_format.Tokenizer('# some comment'.splitlines(), - skip_comments=False) - self.assertFalse(tokenizer.AtEnd()) - self.assertEqual('# some comment', tokenizer.ConsumeComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeTwoComments(self): - text = '# some comment\n# another comment' - tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) - self.assertEqual('# some comment', tokenizer.ConsumeComment()) - self.assertFalse(tokenizer.AtEnd()) - self.assertEqual('# another comment', tokenizer.ConsumeComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeTrailingComment(self): - text = 'some_number: 4\n# some comment' - tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) - self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment) - - self.assertEqual('some_number', tokenizer.ConsumeIdentifier()) - self.assertEqual(tokenizer.token, ':') - tokenizer.NextToken() - self.assertRaises(text_format.ParseError, tokenizer.ConsumeComment) - self.assertEqual(4, tokenizer.ConsumeInteger()) - self.assertFalse(tokenizer.AtEnd()) - - self.assertEqual('# some comment', tokenizer.ConsumeComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeLineComment(self): - tokenizer = text_format.Tokenizer('# some comment'.splitlines(), - skip_comments=False) - self.assertFalse(tokenizer.AtEnd()) - self.assertEqual((False, '# some comment'), - tokenizer.ConsumeCommentOrTrailingComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeTwoLineComments(self): - text = '# some comment\n# another comment' - tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) - self.assertEqual((False, '# some comment'), - tokenizer.ConsumeCommentOrTrailingComment()) - self.assertFalse(tokenizer.AtEnd()) - self.assertEqual((False, '# another comment'), - tokenizer.ConsumeCommentOrTrailingComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testConsumeAndCheckTrailingComment(self): - text = 'some_number: 4 # some comment' # trailing comment on the same line - tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) - self.assertRaises(text_format.ParseError, - tokenizer.ConsumeCommentOrTrailingComment) - - self.assertEqual('some_number', tokenizer.ConsumeIdentifier()) - self.assertEqual(tokenizer.token, ':') - tokenizer.NextToken() - self.assertRaises(text_format.ParseError, - tokenizer.ConsumeCommentOrTrailingComment) - self.assertEqual(4, tokenizer.ConsumeInteger()) - self.assertFalse(tokenizer.AtEnd()) - - self.assertEqual((True, '# some comment'), - tokenizer.ConsumeCommentOrTrailingComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testHashinComment(self): - text = 'some_number: 4 # some comment # not a new comment' - tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) - self.assertEqual('some_number', tokenizer.ConsumeIdentifier()) - self.assertEqual(tokenizer.token, ':') - tokenizer.NextToken() - self.assertEqual(4, tokenizer.ConsumeInteger()) - self.assertEqual((True, '# some comment # not a new comment'), - tokenizer.ConsumeCommentOrTrailingComment()) - self.assertTrue(tokenizer.AtEnd()) - - def testHugeString(self): - # With pathologic backtracking, fails with Forge OOM. - text = '"' + 'a' * (10 * 1024 * 1024) + '"' - tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) - tokenizer.ConsumeString() - - -# Tests for pretty printer functionality. -@_parameterized.parameters((unittest_pb2), (unittest_proto3_arena_pb2)) -class PrettyPrinterTest(TextFormatBase): - - def testPrettyPrintNoMatch(self, message_module): - - def printer(message, indent, as_one_line): - del message, indent, as_one_line - return None - - message = message_module.TestAllTypes() - msg = message.repeated_nested_message.add() - msg.bb = 42 - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=True, message_formatter=printer), - 'repeated_nested_message { bb: 42 }') - - def testPrettyPrintOneLine(self, message_module): - - def printer(m, indent, as_one_line): - del indent, as_one_line - if m.DESCRIPTOR == message_module.TestAllTypes.NestedMessage.DESCRIPTOR: - return 'My lucky number is %s' % m.bb - - message = message_module.TestAllTypes() - msg = message.repeated_nested_message.add() - msg.bb = 42 - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=True, message_formatter=printer), - 'repeated_nested_message { My lucky number is 42 }') - - def testPrettyPrintMultiLine(self, message_module): - - def printer(m, indent, as_one_line): - if m.DESCRIPTOR == message_module.TestAllTypes.NestedMessage.DESCRIPTOR: - line_deliminator = (' ' if as_one_line else '\n') + ' ' * indent - return 'My lucky number is:%s%s' % (line_deliminator, m.bb) - return None - - message = message_module.TestAllTypes() - msg = message.repeated_nested_message.add() - msg.bb = 42 - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=True, message_formatter=printer), - 'repeated_nested_message { My lucky number is: 42 }') - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=False, message_formatter=printer), - 'repeated_nested_message {\n My lucky number is:\n 42\n}\n') - - def testPrettyPrintEntireMessage(self, message_module): - - def printer(m, indent, as_one_line): - del indent, as_one_line - if m.DESCRIPTOR == message_module.TestAllTypes.DESCRIPTOR: - return 'The is the message!' - return None - - message = message_module.TestAllTypes() - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=False, message_formatter=printer), - 'The is the message!\n') - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=True, message_formatter=printer), - 'The is the message!') - - def testPrettyPrintMultipleParts(self, message_module): - - def printer(m, indent, as_one_line): - del indent, as_one_line - if m.DESCRIPTOR == message_module.TestAllTypes.NestedMessage.DESCRIPTOR: - return 'My lucky number is %s' % m.bb - return None - - message = message_module.TestAllTypes() - message.optional_int32 = 61 - msg = message.repeated_nested_message.add() - msg.bb = 42 - msg = message.repeated_nested_message.add() - msg.bb = 99 - msg = message.optional_nested_message - msg.bb = 1 - self.CompareToGoldenText( - text_format.MessageToString( - message, as_one_line=True, message_formatter=printer), - ('optional_int32: 61 ' - 'optional_nested_message { My lucky number is 1 } ' - 'repeated_nested_message { My lucky number is 42 } ' - 'repeated_nested_message { My lucky number is 99 }')) - - out = text_format.TextWriter(False) - text_format.PrintField( - message_module.TestAllTypes.DESCRIPTOR.fields_by_name[ - 'optional_nested_message'], - message.optional_nested_message, - out, - message_formatter=printer) - self.assertEqual( - 'optional_nested_message {\n My lucky number is 1\n}\n', - out.getvalue()) - out.close() - - out = text_format.TextWriter(False) - text_format.PrintFieldValue( - message_module.TestAllTypes.DESCRIPTOR.fields_by_name[ - 'optional_nested_message'], - message.optional_nested_message, - out, - message_formatter=printer) - self.assertEqual( - '{\n My lucky number is 1\n}', - out.getvalue()) - out.close() - - -class WhitespaceTest(TextFormatBase): - - def setUp(self): - self.out = text_format.TextWriter(False) - self.addCleanup(self.out.close) - self.message = unittest_pb2.NestedTestAllTypes() - self.message.child.payload.optional_string = 'value' - self.field = self.message.DESCRIPTOR.fields_by_name['child'] - self.value = self.message.child - - def testMessageToString(self): - self.CompareToGoldenText( - text_format.MessageToString(self.message), - textwrap.dedent("""\ - child { - payload { - optional_string: "value" - } - } - """)) - - def testPrintMessage(self): - text_format.PrintMessage(self.message, self.out) - self.CompareToGoldenText( - self.out.getvalue(), - textwrap.dedent("""\ - child { - payload { - optional_string: "value" - } - } - """)) - - def testPrintField(self): - text_format.PrintField(self.field, self.value, self.out) - self.CompareToGoldenText( - self.out.getvalue(), - textwrap.dedent("""\ - child { - payload { - optional_string: "value" - } - } - """)) - - def testPrintFieldValue(self): - text_format.PrintFieldValue( - self.field, self.value, self.out) - self.CompareToGoldenText( - self.out.getvalue(), - textwrap.dedent("""\ - { - payload { - optional_string: "value" - } - }""")) - - -class OptionalColonMessageToStringTest(unittest.TestCase): - - def testForcePrintOptionalColon(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - output = text_format.MessageToString( - message, - force_colon=True) - expected = ('any_value: {\n' - ' [type.googleapis.com/protobuf_unittest.OneString]: {\n' - ' data: "string"\n' - ' }\n' - '}\n') - self.assertEqual(expected, output) - - def testPrintShortFormatRepeatedFields(self): - message = unittest_pb2.TestAllTypes() - message.repeated_int32.append(1) - output = text_format.MessageToString( - message, use_short_repeated_primitives=True, force_colon=True) - self.assertEqual('repeated_int32: [1]\n', output) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/type_checkers.py b/ext/protobuf/Python/google/protobuf/internal/type_checkers.py index a53e71fe8..165dcd8c2 100644 --- a/ext/protobuf/Python/google/protobuf/internal/type_checkers.py +++ b/ext/protobuf/Python/google/protobuf/internal/type_checkers.py @@ -75,10 +75,6 @@ def ToShortestFloat(original): return rounded -def SupportsOpenEnums(field_descriptor): - return field_descriptor.containing_type.syntax == 'proto3' - - def GetTypeChecker(field): """Returns a type checker for a message field of the specified types. @@ -93,11 +89,11 @@ def GetTypeChecker(field): field.type == _FieldDescriptor.TYPE_STRING): return UnicodeValueChecker() if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM: - if SupportsOpenEnums(field): + if field.enum_type.is_closed: + return EnumValueChecker(field.enum_type) + else: # When open enums are supported, any int32 can be assigned. return _VALUE_CHECKERS[_FieldDescriptor.CPPTYPE_INT32] - else: - return EnumValueChecker(field.enum_type) return _VALUE_CHECKERS[field.cpp_type] diff --git a/ext/protobuf/Python/google/protobuf/internal/unknown_fields_test.py b/ext/protobuf/Python/google/protobuf/internal/unknown_fields_test.py deleted file mode 100644 index 64a036782..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/unknown_fields_test.py +++ /dev/null @@ -1,461 +0,0 @@ -# -*- coding: utf-8 -*- -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test for preservation of unknown fields in the pure Python implementation.""" - -__author__ = 'bohdank@google.com (Bohdan Koval)' - -import sys -import unittest - -from google.protobuf import map_unittest_pb2 -from google.protobuf import unittest_mset_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf import unittest_proto3_arena_pb2 -from google.protobuf.internal import api_implementation -from google.protobuf.internal import encoder -from google.protobuf.internal import message_set_extensions_pb2 -from google.protobuf.internal import missing_enum_values_pb2 -from google.protobuf.internal import test_util -from google.protobuf.internal import testing_refleaks -from google.protobuf.internal import type_checkers -from google.protobuf.internal import wire_format -from google.protobuf import descriptor -from google.protobuf import unknown_fields -try: - import tracemalloc # pylint: disable=g-import-not-at-top -except ImportError: - # Requires python 3.4+ - pass - - -@testing_refleaks.TestCase -class UnknownFieldsTest(unittest.TestCase): - - def setUp(self): - self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - self.all_fields = unittest_pb2.TestAllTypes() - test_util.SetAllFields(self.all_fields) - self.all_fields_data = self.all_fields.SerializeToString() - self.empty_message = unittest_pb2.TestEmptyMessage() - self.empty_message.ParseFromString(self.all_fields_data) - - def testSerialize(self): - data = self.empty_message.SerializeToString() - - # Don't use assertEqual because we don't want to dump raw binary data to - # stdout. - self.assertTrue(data == self.all_fields_data) - - def testSerializeProto3(self): - # Verify proto3 unknown fields behavior. - message = unittest_proto3_arena_pb2.TestEmptyMessage() - message.ParseFromString(self.all_fields_data) - self.assertEqual(self.all_fields_data, message.SerializeToString()) - - def testByteSize(self): - self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize()) - - def testListFields(self): - # Make sure ListFields doesn't return unknown fields. - self.assertEqual(0, len(self.empty_message.ListFields())) - - def testSerializeMessageSetWireFormatUnknownExtension(self): - # Create a message using the message set wire format with an unknown - # message. - raw = unittest_mset_pb2.RawMessageSet() - - # Add an unknown extension. - item = raw.item.add() - item.type_id = 98218603 - message1 = message_set_extensions_pb2.TestMessageSetExtension1() - message1.i = 12345 - item.message = message1.SerializeToString() - - serialized = raw.SerializeToString() - - # Parse message using the message set wire format. - proto = message_set_extensions_pb2.TestMessageSet() - proto.MergeFromString(serialized) - - unknown_field_set = unknown_fields.UnknownFieldSet(proto) - self.assertEqual(len(unknown_field_set), 1) - # Unknown field should have wire format data which can be parsed back to - # original message. - self.assertEqual(unknown_field_set[0].field_number, item.type_id) - self.assertEqual(unknown_field_set[0].wire_type, - wire_format.WIRETYPE_LENGTH_DELIMITED) - d = unknown_field_set[0].data - message_new = message_set_extensions_pb2.TestMessageSetExtension1() - message_new.ParseFromString(d) - self.assertEqual(message1, message_new) - - # Verify that the unknown extension is serialized unchanged - reserialized = proto.SerializeToString() - new_raw = unittest_mset_pb2.RawMessageSet() - new_raw.MergeFromString(reserialized) - self.assertEqual(raw, new_raw) - - def testEquals(self): - message = unittest_pb2.TestEmptyMessage() - message.ParseFromString(self.all_fields_data) - self.assertEqual(self.empty_message, message) - - self.all_fields.ClearField('optional_string') - message.ParseFromString(self.all_fields.SerializeToString()) - self.assertNotEqual(self.empty_message, message) - - def testDiscardUnknownFields(self): - self.empty_message.DiscardUnknownFields() - self.assertEqual(b'', self.empty_message.SerializeToString()) - # Test message field and repeated message field. - message = unittest_pb2.TestAllTypes() - other_message = unittest_pb2.TestAllTypes() - other_message.optional_string = 'discard' - message.optional_nested_message.ParseFromString( - other_message.SerializeToString()) - message.repeated_nested_message.add().ParseFromString( - other_message.SerializeToString()) - self.assertNotEqual( - b'', message.optional_nested_message.SerializeToString()) - self.assertNotEqual( - b'', message.repeated_nested_message[0].SerializeToString()) - message.DiscardUnknownFields() - self.assertEqual(b'', message.optional_nested_message.SerializeToString()) - self.assertEqual( - b'', message.repeated_nested_message[0].SerializeToString()) - - msg = map_unittest_pb2.TestMap() - msg.map_int32_all_types[1].optional_nested_message.ParseFromString( - other_message.SerializeToString()) - msg.map_string_string['1'] = 'test' - self.assertNotEqual( - b'', - msg.map_int32_all_types[1].optional_nested_message.SerializeToString()) - msg.DiscardUnknownFields() - self.assertEqual( - b'', - msg.map_int32_all_types[1].optional_nested_message.SerializeToString()) - - -@testing_refleaks.TestCase -class UnknownFieldsAccessorsTest(unittest.TestCase): - - def setUp(self): - self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - self.all_fields = unittest_pb2.TestAllTypes() - test_util.SetAllFields(self.all_fields) - self.all_fields_data = self.all_fields.SerializeToString() - self.empty_message = unittest_pb2.TestEmptyMessage() - self.empty_message.ParseFromString(self.all_fields_data) - - # InternalCheckUnknownField() is an additional Pure Python check which checks - # a detail of unknown fields. It cannot be used by the C++ - # implementation because some protect members are called. - # The test is added for historical reasons. It is not necessary as - # serialized string is checked. - # TODO(jieluo): Remove message._unknown_fields. - def InternalCheckUnknownField(self, name, expected_value): - if api_implementation.Type() != 'python': - return - field_descriptor = self.descriptor.fields_by_name[name] - wire_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type] - field_tag = encoder.TagBytes(field_descriptor.number, wire_type) - result_dict = {} - for tag_bytes, value in self.empty_message._unknown_fields: - if tag_bytes == field_tag: - decoder = unittest_pb2.TestAllTypes._decoders_by_tag[tag_bytes][0] - decoder(memoryview(value), 0, len(value), self.all_fields, result_dict) - self.assertEqual(expected_value, result_dict[field_descriptor]) - - def CheckUnknownField(self, name, unknown_field_set, expected_value): - field_descriptor = self.descriptor.fields_by_name[name] - expected_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[ - field_descriptor.type] - for unknown_field in unknown_field_set: - if unknown_field.field_number == field_descriptor.number: - self.assertEqual(expected_type, unknown_field.wire_type) - if expected_type == 3: - # Check group - self.assertEqual(expected_value[0], - unknown_field.data[0].field_number) - self.assertEqual(expected_value[1], unknown_field.data[0].wire_type) - self.assertEqual(expected_value[2], unknown_field.data[0].data) - continue - if expected_type == wire_format.WIRETYPE_LENGTH_DELIMITED: - self.assertIn(type(unknown_field.data), (str, bytes)) - if field_descriptor.label == descriptor.FieldDescriptor.LABEL_REPEATED: - self.assertIn(unknown_field.data, expected_value) - else: - self.assertEqual(expected_value, unknown_field.data) - - def testCheckUnknownFieldValue(self): - unknown_field_set = unknown_fields.UnknownFieldSet(self.empty_message) - # Test enum. - self.CheckUnknownField('optional_nested_enum', - unknown_field_set, - self.all_fields.optional_nested_enum) - self.InternalCheckUnknownField('optional_nested_enum', - self.all_fields.optional_nested_enum) - - # Test repeated enum. - self.CheckUnknownField('repeated_nested_enum', - unknown_field_set, - self.all_fields.repeated_nested_enum) - self.InternalCheckUnknownField('repeated_nested_enum', - self.all_fields.repeated_nested_enum) - - # Test varint. - self.CheckUnknownField('optional_int32', - unknown_field_set, - self.all_fields.optional_int32) - self.InternalCheckUnknownField('optional_int32', - self.all_fields.optional_int32) - - # Test fixed32. - self.CheckUnknownField('optional_fixed32', - unknown_field_set, - self.all_fields.optional_fixed32) - self.InternalCheckUnknownField('optional_fixed32', - self.all_fields.optional_fixed32) - - # Test fixed64. - self.CheckUnknownField('optional_fixed64', - unknown_field_set, - self.all_fields.optional_fixed64) - self.InternalCheckUnknownField('optional_fixed64', - self.all_fields.optional_fixed64) - - # Test length delimited. - self.CheckUnknownField('optional_string', - unknown_field_set, - self.all_fields.optional_string.encode('utf-8')) - self.InternalCheckUnknownField('optional_string', - self.all_fields.optional_string) - - # Test group. - self.CheckUnknownField('optionalgroup', - unknown_field_set, - (17, 0, 117)) - self.InternalCheckUnknownField('optionalgroup', - self.all_fields.optionalgroup) - - self.assertEqual(98, len(unknown_field_set)) - - def testCopyFrom(self): - message = unittest_pb2.TestEmptyMessage() - message.CopyFrom(self.empty_message) - self.assertEqual(message.SerializeToString(), self.all_fields_data) - - def testMergeFrom(self): - message = unittest_pb2.TestAllTypes() - message.optional_int32 = 1 - message.optional_uint32 = 2 - source = unittest_pb2.TestEmptyMessage() - source.ParseFromString(message.SerializeToString()) - - message.ClearField('optional_int32') - message.optional_int64 = 3 - message.optional_uint32 = 4 - destination = unittest_pb2.TestEmptyMessage() - unknown_field_set = unknown_fields.UnknownFieldSet(destination) - self.assertEqual(0, len(unknown_field_set)) - destination.ParseFromString(message.SerializeToString()) - self.assertEqual(0, len(unknown_field_set)) - unknown_field_set = unknown_fields.UnknownFieldSet(destination) - self.assertEqual(2, len(unknown_field_set)) - destination.MergeFrom(source) - self.assertEqual(2, len(unknown_field_set)) - # Check that the fields where correctly merged, even stored in the unknown - # fields set. - message.ParseFromString(destination.SerializeToString()) - self.assertEqual(message.optional_int32, 1) - self.assertEqual(message.optional_uint32, 2) - self.assertEqual(message.optional_int64, 3) - - def testClear(self): - unknown_field_set = unknown_fields.UnknownFieldSet(self.empty_message) - self.empty_message.Clear() - # All cleared, even unknown fields. - self.assertEqual(self.empty_message.SerializeToString(), b'') - self.assertEqual(len(unknown_field_set), 98) - - @unittest.skipIf((sys.version_info.major, sys.version_info.minor) < (3, 4), - 'tracemalloc requires python 3.4+') - def testUnknownFieldsNoMemoryLeak(self): - # Call to UnknownFields must not leak memory - nb_leaks = 1234 - - def leaking_function(): - for _ in range(nb_leaks): - unknown_fields.UnknownFieldSet(self.empty_message) - - tracemalloc.start() - snapshot1 = tracemalloc.take_snapshot() - leaking_function() - snapshot2 = tracemalloc.take_snapshot() - top_stats = snapshot2.compare_to(snapshot1, 'lineno') - tracemalloc.stop() - # There's no easy way to look for a precise leak source. - # Rely on a "marker" count value while checking allocated memory. - self.assertEqual([], [x for x in top_stats if x.count_diff == nb_leaks]) - - def testSubUnknownFields(self): - message = unittest_pb2.TestAllTypes() - message.optionalgroup.a = 123 - destination = unittest_pb2.TestEmptyMessage() - destination.ParseFromString(message.SerializeToString()) - sub_unknown_fields = unknown_fields.UnknownFieldSet(destination)[0].data - self.assertEqual(1, len(sub_unknown_fields)) - self.assertEqual(sub_unknown_fields[0].data, 123) - destination.Clear() - self.assertEqual(1, len(sub_unknown_fields)) - self.assertEqual(sub_unknown_fields[0].data, 123) - message.Clear() - message.optional_uint32 = 456 - nested_message = unittest_pb2.NestedTestAllTypes() - nested_message.payload.optional_nested_message.ParseFromString( - message.SerializeToString()) - unknown_field_set = unknown_fields.UnknownFieldSet( - nested_message.payload.optional_nested_message) - self.assertEqual(unknown_field_set[0].data, 456) - nested_message.ClearField('payload') - self.assertEqual(unknown_field_set[0].data, 456) - unknown_field_set = unknown_fields.UnknownFieldSet( - nested_message.payload.optional_nested_message) - self.assertEqual(0, len(unknown_field_set)) - - def testUnknownField(self): - message = unittest_pb2.TestAllTypes() - message.optional_int32 = 123 - destination = unittest_pb2.TestEmptyMessage() - destination.ParseFromString(message.SerializeToString()) - unknown_field = unknown_fields.UnknownFieldSet(destination)[0] - destination.Clear() - self.assertEqual(unknown_field.data, 123) - - def testUnknownExtensions(self): - message = unittest_pb2.TestEmptyMessageWithExtensions() - message.ParseFromString(self.all_fields_data) - self.assertEqual(len(unknown_fields.UnknownFieldSet(message)), 98) - self.assertEqual(message.SerializeToString(), self.all_fields_data) - - -@testing_refleaks.TestCase -class UnknownEnumValuesTest(unittest.TestCase): - - def setUp(self): - self.descriptor = missing_enum_values_pb2.TestEnumValues.DESCRIPTOR - - self.message = missing_enum_values_pb2.TestEnumValues() - # TestEnumValues.ZERO = 0, but does not exist in the other NestedEnum. - self.message.optional_nested_enum = ( - missing_enum_values_pb2.TestEnumValues.ZERO) - self.message.repeated_nested_enum.extend([ - missing_enum_values_pb2.TestEnumValues.ZERO, - missing_enum_values_pb2.TestEnumValues.ONE, - ]) - self.message.packed_nested_enum.extend([ - missing_enum_values_pb2.TestEnumValues.ZERO, - missing_enum_values_pb2.TestEnumValues.ONE, - ]) - self.message_data = self.message.SerializeToString() - self.missing_message = missing_enum_values_pb2.TestMissingEnumValues() - self.missing_message.ParseFromString(self.message_data) - - # CheckUnknownField() is an additional Pure Python check which checks - # a detail of unknown fields. It cannot be used by the C++ - # implementation because some protect members are called. - # The test is added for historical reasons. It is not necessary as - # serialized string is checked. - - def CheckUnknownField(self, name, expected_value): - field_descriptor = self.descriptor.fields_by_name[name] - unknown_field_set = unknown_fields.UnknownFieldSet(self.missing_message) - self.assertIsInstance(unknown_field_set, unknown_fields.UnknownFieldSet) - count = 0 - for field in unknown_field_set: - if field.field_number == field_descriptor.number: - count += 1 - if field_descriptor.label == descriptor.FieldDescriptor.LABEL_REPEATED: - self.assertIn(field.data, expected_value) - else: - self.assertEqual(expected_value, field.data) - if field_descriptor.label == descriptor.FieldDescriptor.LABEL_REPEATED: - self.assertEqual(count, len(expected_value)) - else: - self.assertEqual(count, 1) - - def testUnknownParseMismatchEnumValue(self): - just_string = missing_enum_values_pb2.JustString() - just_string.dummy = 'blah' - - missing = missing_enum_values_pb2.TestEnumValues() - # The parse is invalid, storing the string proto into the set of - # unknown fields. - missing.ParseFromString(just_string.SerializeToString()) - - # Fetching the enum field shouldn't crash, instead returning the - # default value. - self.assertEqual(missing.optional_nested_enum, 0) - - def testUnknownEnumValue(self): - self.assertFalse(self.missing_message.HasField('optional_nested_enum')) - self.assertEqual(self.missing_message.optional_nested_enum, 2) - # Clear does not do anything. - serialized = self.missing_message.SerializeToString() - self.missing_message.ClearField('optional_nested_enum') - self.assertEqual(self.missing_message.SerializeToString(), serialized) - - def testUnknownRepeatedEnumValue(self): - self.assertEqual([], self.missing_message.repeated_nested_enum) - - def testUnknownPackedEnumValue(self): - self.assertEqual([], self.missing_message.packed_nested_enum) - - def testCheckUnknownFieldValueForEnum(self): - unknown_field_set = unknown_fields.UnknownFieldSet(self.missing_message) - self.assertEqual(len(unknown_field_set), 5) - self.CheckUnknownField('optional_nested_enum', - self.message.optional_nested_enum) - self.CheckUnknownField('repeated_nested_enum', - self.message.repeated_nested_enum) - self.CheckUnknownField('packed_nested_enum', - self.message.packed_nested_enum) - - def testRoundTrip(self): - new_message = missing_enum_values_pb2.TestEnumValues() - new_message.ParseFromString(self.missing_message.SerializeToString()) - self.assertEqual(self.message, new_message) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/well_known_types.py b/ext/protobuf/Python/google/protobuf/internal/well_known_types.py index 8881d758a..e340f9087 100644 --- a/ext/protobuf/Python/google/protobuf/internal/well_known_types.py +++ b/ext/protobuf/Python/google/protobuf/internal/well_known_types.py @@ -44,7 +44,9 @@ import collections.abc import datetime -from google.protobuf.descriptor import FieldDescriptor +from google.protobuf.internal import field_mask + +FieldMask = field_mask.FieldMask _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' _NANOS_PER_SECOND = 1000000000 @@ -430,306 +432,6 @@ def _RoundTowardZero(value, divider): return result -class FieldMask(object): - """Class for FieldMask message type.""" - - __slots__ = () - - def ToJsonString(self): - """Converts FieldMask to string according to proto3 JSON spec.""" - camelcase_paths = [] - for path in self.paths: - camelcase_paths.append(_SnakeCaseToCamelCase(path)) - return ','.join(camelcase_paths) - - def FromJsonString(self, value): - """Converts string to FieldMask according to proto3 JSON spec.""" - if not isinstance(value, str): - raise ValueError('FieldMask JSON value not a string: {!r}'.format(value)) - self.Clear() - if value: - for path in value.split(','): - self.paths.append(_CamelCaseToSnakeCase(path)) - - def IsValidForDescriptor(self, message_descriptor): - """Checks whether the FieldMask is valid for Message Descriptor.""" - for path in self.paths: - if not _IsValidPath(message_descriptor, path): - return False - return True - - def AllFieldsFromDescriptor(self, message_descriptor): - """Gets all direct fields of Message Descriptor to FieldMask.""" - self.Clear() - for field in message_descriptor.fields: - self.paths.append(field.name) - - def CanonicalFormFromMask(self, mask): - """Converts a FieldMask to the canonical form. - - Removes paths that are covered by another path. For example, - "foo.bar" is covered by "foo" and will be removed if "foo" - is also in the FieldMask. Then sorts all paths in alphabetical order. - - Args: - mask: The original FieldMask to be converted. - """ - tree = _FieldMaskTree(mask) - tree.ToFieldMask(self) - - def Union(self, mask1, mask2): - """Merges mask1 and mask2 into this FieldMask.""" - _CheckFieldMaskMessage(mask1) - _CheckFieldMaskMessage(mask2) - tree = _FieldMaskTree(mask1) - tree.MergeFromFieldMask(mask2) - tree.ToFieldMask(self) - - def Intersect(self, mask1, mask2): - """Intersects mask1 and mask2 into this FieldMask.""" - _CheckFieldMaskMessage(mask1) - _CheckFieldMaskMessage(mask2) - tree = _FieldMaskTree(mask1) - intersection = _FieldMaskTree() - for path in mask2.paths: - tree.IntersectPath(path, intersection) - intersection.ToFieldMask(self) - - def MergeMessage( - self, source, destination, - replace_message_field=False, replace_repeated_field=False): - """Merges fields specified in FieldMask from source to destination. - - Args: - source: Source message. - destination: The destination message to be merged into. - replace_message_field: Replace message field if True. Merge message - field if False. - replace_repeated_field: Replace repeated field if True. Append - elements of repeated field if False. - """ - tree = _FieldMaskTree(self) - tree.MergeMessage( - source, destination, replace_message_field, replace_repeated_field) - - -def _IsValidPath(message_descriptor, path): - """Checks whether the path is valid for Message Descriptor.""" - parts = path.split('.') - last = parts.pop() - for name in parts: - field = message_descriptor.fields_by_name.get(name) - if (field is None or - field.label == FieldDescriptor.LABEL_REPEATED or - field.type != FieldDescriptor.TYPE_MESSAGE): - return False - message_descriptor = field.message_type - return last in message_descriptor.fields_by_name - - -def _CheckFieldMaskMessage(message): - """Raises ValueError if message is not a FieldMask.""" - message_descriptor = message.DESCRIPTOR - if (message_descriptor.name != 'FieldMask' or - message_descriptor.file.name != 'google/protobuf/field_mask.proto'): - raise ValueError('Message {0} is not a FieldMask.'.format( - message_descriptor.full_name)) - - -def _SnakeCaseToCamelCase(path_name): - """Converts a path name from snake_case to camelCase.""" - result = [] - after_underscore = False - for c in path_name: - if c.isupper(): - raise ValueError( - 'Fail to print FieldMask to Json string: Path name ' - '{0} must not contain uppercase letters.'.format(path_name)) - if after_underscore: - if c.islower(): - result.append(c.upper()) - after_underscore = False - else: - raise ValueError( - 'Fail to print FieldMask to Json string: The ' - 'character after a "_" must be a lowercase letter ' - 'in path name {0}.'.format(path_name)) - elif c == '_': - after_underscore = True - else: - result += c - - if after_underscore: - raise ValueError('Fail to print FieldMask to Json string: Trailing "_" ' - 'in path name {0}.'.format(path_name)) - return ''.join(result) - - -def _CamelCaseToSnakeCase(path_name): - """Converts a field name from camelCase to snake_case.""" - result = [] - for c in path_name: - if c == '_': - raise ValueError('Fail to parse FieldMask: Path name ' - '{0} must not contain "_"s.'.format(path_name)) - if c.isupper(): - result += '_' - result += c.lower() - else: - result += c - return ''.join(result) - - -class _FieldMaskTree(object): - """Represents a FieldMask in a tree structure. - - For example, given a FieldMask "foo.bar,foo.baz,bar.baz", - the FieldMaskTree will be: - [_root] -+- foo -+- bar - | | - | +- baz - | - +- bar --- baz - In the tree, each leaf node represents a field path. - """ - - __slots__ = ('_root',) - - def __init__(self, field_mask=None): - """Initializes the tree by FieldMask.""" - self._root = {} - if field_mask: - self.MergeFromFieldMask(field_mask) - - def MergeFromFieldMask(self, field_mask): - """Merges a FieldMask to the tree.""" - for path in field_mask.paths: - self.AddPath(path) - - def AddPath(self, path): - """Adds a field path into the tree. - - If the field path to add is a sub-path of an existing field path - in the tree (i.e., a leaf node), it means the tree already matches - the given path so nothing will be added to the tree. If the path - matches an existing non-leaf node in the tree, that non-leaf node - will be turned into a leaf node with all its children removed because - the path matches all the node's children. Otherwise, a new path will - be added. - - Args: - path: The field path to add. - """ - node = self._root - for name in path.split('.'): - if name not in node: - node[name] = {} - elif not node[name]: - # Pre-existing empty node implies we already have this entire tree. - return - node = node[name] - # Remove any sub-trees we might have had. - node.clear() - - def ToFieldMask(self, field_mask): - """Converts the tree to a FieldMask.""" - field_mask.Clear() - _AddFieldPaths(self._root, '', field_mask) - - def IntersectPath(self, path, intersection): - """Calculates the intersection part of a field path with this tree. - - Args: - path: The field path to calculates. - intersection: The out tree to record the intersection part. - """ - node = self._root - for name in path.split('.'): - if name not in node: - return - elif not node[name]: - intersection.AddPath(path) - return - node = node[name] - intersection.AddLeafNodes(path, node) - - def AddLeafNodes(self, prefix, node): - """Adds leaf nodes begin with prefix to this tree.""" - if not node: - self.AddPath(prefix) - for name in node: - child_path = prefix + '.' + name - self.AddLeafNodes(child_path, node[name]) - - def MergeMessage( - self, source, destination, - replace_message, replace_repeated): - """Merge all fields specified by this tree from source to destination.""" - _MergeMessage( - self._root, source, destination, replace_message, replace_repeated) - - -def _StrConvert(value): - """Converts value to str if it is not.""" - # This file is imported by c extension and some methods like ClearField - # requires string for the field name. py2/py3 has different text - # type and may use unicode. - if not isinstance(value, str): - return value.encode('utf-8') - return value - - -def _MergeMessage( - node, source, destination, replace_message, replace_repeated): - """Merge all fields specified by a sub-tree from source to destination.""" - source_descriptor = source.DESCRIPTOR - for name in node: - child = node[name] - field = source_descriptor.fields_by_name[name] - if field is None: - raise ValueError('Error: Can\'t find field {0} in message {1}.'.format( - name, source_descriptor.full_name)) - if child: - # Sub-paths are only allowed for singular message fields. - if (field.label == FieldDescriptor.LABEL_REPEATED or - field.cpp_type != FieldDescriptor.CPPTYPE_MESSAGE): - raise ValueError('Error: Field {0} in message {1} is not a singular ' - 'message field and cannot have sub-fields.'.format( - name, source_descriptor.full_name)) - if source.HasField(name): - _MergeMessage( - child, getattr(source, name), getattr(destination, name), - replace_message, replace_repeated) - continue - if field.label == FieldDescriptor.LABEL_REPEATED: - if replace_repeated: - destination.ClearField(_StrConvert(name)) - repeated_source = getattr(source, name) - repeated_destination = getattr(destination, name) - repeated_destination.MergeFrom(repeated_source) - else: - if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: - if replace_message: - destination.ClearField(_StrConvert(name)) - if source.HasField(name): - getattr(destination, name).MergeFrom(getattr(source, name)) - else: - setattr(destination, name, getattr(source, name)) - - -def _AddFieldPaths(node, prefix, field_mask): - """Adds the field paths descended from node to field_mask.""" - if not node and prefix: - field_mask.paths.append(prefix) - return - for name in sorted(node): - if prefix: - child_path = prefix + '.' + name - else: - child_path = name - _AddFieldPaths(node[name], child_path, field_mask) - - def _SetStructValue(struct_value, value): if value is None: struct_value.null_value = 0 diff --git a/ext/protobuf/Python/google/protobuf/internal/well_known_types_test.py b/ext/protobuf/Python/google/protobuf/internal/well_known_types_test.py deleted file mode 100644 index a32459a9e..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/well_known_types_test.py +++ /dev/null @@ -1,1013 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test for google.protobuf.internal.well_known_types.""" - -__author__ = 'jieluo@google.com (Jie Luo)' - -import collections.abc as collections_abc -import datetime -import unittest - -from google.protobuf import any_pb2 -from google.protobuf import duration_pb2 -from google.protobuf import field_mask_pb2 -from google.protobuf import struct_pb2 -from google.protobuf import timestamp_pb2 -from google.protobuf import map_unittest_pb2 -from google.protobuf import unittest_pb2 -from google.protobuf.internal import any_test_pb2 -from google.protobuf.internal import test_util -from google.protobuf.internal import well_known_types -from google.protobuf import descriptor -from google.protobuf import text_format -from google.protobuf.internal import _parameterized - -try: - # New module in Python 3.9: - import zoneinfo # pylint:disable=g-import-not-at-top - _TZ_JAPAN = zoneinfo.ZoneInfo('Japan') - _TZ_PACIFIC = zoneinfo.ZoneInfo('US/Pacific') -except ImportError: - _TZ_JAPAN = datetime.timezone(datetime.timedelta(hours=9), 'Japan') - _TZ_PACIFIC = datetime.timezone(datetime.timedelta(hours=-8), 'US/Pacific') - - -class TimeUtilTestBase(_parameterized.TestCase): - - def CheckTimestampConversion(self, message, text): - self.assertEqual(text, message.ToJsonString()) - parsed_message = timestamp_pb2.Timestamp() - parsed_message.FromJsonString(text) - self.assertEqual(message, parsed_message) - - def CheckDurationConversion(self, message, text): - self.assertEqual(text, message.ToJsonString()) - parsed_message = duration_pb2.Duration() - parsed_message.FromJsonString(text) - self.assertEqual(message, parsed_message) - - -class TimeUtilTest(TimeUtilTestBase): - - def testTimestampSerializeAndParse(self): - message = timestamp_pb2.Timestamp() - # Generated output should contain 3, 6, or 9 fractional digits. - message.seconds = 0 - message.nanos = 0 - self.CheckTimestampConversion(message, '1970-01-01T00:00:00Z') - message.nanos = 10000000 - self.CheckTimestampConversion(message, '1970-01-01T00:00:00.010Z') - message.nanos = 10000 - self.CheckTimestampConversion(message, '1970-01-01T00:00:00.000010Z') - message.nanos = 10 - self.CheckTimestampConversion(message, '1970-01-01T00:00:00.000000010Z') - # Test min timestamps. - message.seconds = -62135596800 - message.nanos = 0 - self.CheckTimestampConversion(message, '0001-01-01T00:00:00Z') - # Test max timestamps. - message.seconds = 253402300799 - message.nanos = 999999999 - self.CheckTimestampConversion(message, '9999-12-31T23:59:59.999999999Z') - # Test negative timestamps. - message.seconds = -1 - self.CheckTimestampConversion(message, '1969-12-31T23:59:59.999999999Z') - - # Parsing accepts an fractional digits as long as they fit into nano - # precision. - message.FromJsonString('1970-01-01T00:00:00.1Z') - self.assertEqual(0, message.seconds) - self.assertEqual(100000000, message.nanos) - # Parsing accepts offsets. - message.FromJsonString('1970-01-01T00:00:00-08:00') - self.assertEqual(8 * 3600, message.seconds) - self.assertEqual(0, message.nanos) - - # It is not easy to check with current time. For test coverage only. - message.GetCurrentTime() - self.assertNotEqual(8 * 3600, message.seconds) - - def testDurationSerializeAndParse(self): - message = duration_pb2.Duration() - # Generated output should contain 3, 6, or 9 fractional digits. - message.seconds = 0 - message.nanos = 0 - self.CheckDurationConversion(message, '0s') - message.nanos = 10000000 - self.CheckDurationConversion(message, '0.010s') - message.nanos = 10000 - self.CheckDurationConversion(message, '0.000010s') - message.nanos = 10 - self.CheckDurationConversion(message, '0.000000010s') - - # Test min and max - message.seconds = 315576000000 - message.nanos = 999999999 - self.CheckDurationConversion(message, '315576000000.999999999s') - message.seconds = -315576000000 - message.nanos = -999999999 - self.CheckDurationConversion(message, '-315576000000.999999999s') - - # Parsing accepts an fractional digits as long as they fit into nano - # precision. - message.FromJsonString('0.1s') - self.assertEqual(100000000, message.nanos) - message.FromJsonString('0.0000001s') - self.assertEqual(100, message.nanos) - - def testTimestampIntegerConversion(self): - message = timestamp_pb2.Timestamp() - message.FromNanoseconds(1) - self.assertEqual('1970-01-01T00:00:00.000000001Z', - message.ToJsonString()) - self.assertEqual(1, message.ToNanoseconds()) - - message.FromNanoseconds(-1) - self.assertEqual('1969-12-31T23:59:59.999999999Z', - message.ToJsonString()) - self.assertEqual(-1, message.ToNanoseconds()) - - message.FromMicroseconds(1) - self.assertEqual('1970-01-01T00:00:00.000001Z', - message.ToJsonString()) - self.assertEqual(1, message.ToMicroseconds()) - - message.FromMicroseconds(-1) - self.assertEqual('1969-12-31T23:59:59.999999Z', - message.ToJsonString()) - self.assertEqual(-1, message.ToMicroseconds()) - - message.FromMilliseconds(1) - self.assertEqual('1970-01-01T00:00:00.001Z', - message.ToJsonString()) - self.assertEqual(1, message.ToMilliseconds()) - - message.FromMilliseconds(-1) - self.assertEqual('1969-12-31T23:59:59.999Z', - message.ToJsonString()) - self.assertEqual(-1, message.ToMilliseconds()) - - message.FromSeconds(1) - self.assertEqual('1970-01-01T00:00:01Z', - message.ToJsonString()) - self.assertEqual(1, message.ToSeconds()) - - message.FromSeconds(-1) - self.assertEqual('1969-12-31T23:59:59Z', - message.ToJsonString()) - self.assertEqual(-1, message.ToSeconds()) - - message.FromNanoseconds(1999) - self.assertEqual(1, message.ToMicroseconds()) - # For negative values, Timestamp will be rounded down. - # For example, "1969-12-31T23:59:59.5Z" (i.e., -0.5s) rounded to seconds - # will be "1969-12-31T23:59:59Z" (i.e., -1s) rather than - # "1970-01-01T00:00:00Z" (i.e., 0s). - message.FromNanoseconds(-1999) - self.assertEqual(-2, message.ToMicroseconds()) - - def testDurationIntegerConversion(self): - message = duration_pb2.Duration() - message.FromNanoseconds(1) - self.assertEqual('0.000000001s', - message.ToJsonString()) - self.assertEqual(1, message.ToNanoseconds()) - - message.FromNanoseconds(-1) - self.assertEqual('-0.000000001s', - message.ToJsonString()) - self.assertEqual(-1, message.ToNanoseconds()) - - message.FromMicroseconds(1) - self.assertEqual('0.000001s', - message.ToJsonString()) - self.assertEqual(1, message.ToMicroseconds()) - - message.FromMicroseconds(-1) - self.assertEqual('-0.000001s', - message.ToJsonString()) - self.assertEqual(-1, message.ToMicroseconds()) - - message.FromMilliseconds(1) - self.assertEqual('0.001s', - message.ToJsonString()) - self.assertEqual(1, message.ToMilliseconds()) - - message.FromMilliseconds(-1) - self.assertEqual('-0.001s', - message.ToJsonString()) - self.assertEqual(-1, message.ToMilliseconds()) - - message.FromSeconds(1) - self.assertEqual('1s', message.ToJsonString()) - self.assertEqual(1, message.ToSeconds()) - - message.FromSeconds(-1) - self.assertEqual('-1s', - message.ToJsonString()) - self.assertEqual(-1, message.ToSeconds()) - - # Test truncation behavior. - message.FromNanoseconds(1999) - self.assertEqual(1, message.ToMicroseconds()) - - # For negative values, Duration will be rounded towards 0. - message.FromNanoseconds(-1999) - self.assertEqual(-1, message.ToMicroseconds()) - - def testTimezoneNaiveDatetimeConversion(self): - message = timestamp_pb2.Timestamp() - naive_utc_epoch = datetime.datetime(1970, 1, 1) - message.FromDatetime(naive_utc_epoch) - self.assertEqual(0, message.seconds) - self.assertEqual(0, message.nanos) - - self.assertEqual(naive_utc_epoch, message.ToDatetime()) - - naive_epoch_morning = datetime.datetime(1970, 1, 1, 8, 0, 0, 1) - message.FromDatetime(naive_epoch_morning) - self.assertEqual(8 * 3600, message.seconds) - self.assertEqual(1000, message.nanos) - - self.assertEqual(naive_epoch_morning, message.ToDatetime()) - - message.FromMilliseconds(1999) - self.assertEqual(1, message.seconds) - self.assertEqual(999_000_000, message.nanos) - - self.assertEqual(datetime.datetime(1970, 1, 1, 0, 0, 1, 999000), - message.ToDatetime()) - - naive_future = datetime.datetime(2555, 2, 22, 1, 2, 3, 456789) - message.FromDatetime(naive_future) - self.assertEqual(naive_future, message.ToDatetime()) - - naive_end_of_time = datetime.datetime.max - message.FromDatetime(naive_end_of_time) - self.assertEqual(naive_end_of_time, message.ToDatetime()) - - # Two hours after the Unix Epoch, around the world. - @_parameterized.named_parameters( - ('London', [1970, 1, 1, 2], datetime.timezone.utc), - ('Tokyo', [1970, 1, 1, 11], _TZ_JAPAN), - ('LA', [1969, 12, 31, 18], _TZ_PACIFIC), - ) - def testTimezoneAwareDatetimeConversion(self, date_parts, tzinfo): - original_datetime = datetime.datetime(*date_parts, tzinfo=tzinfo) # pylint:disable=g-tzinfo-datetime - - message = timestamp_pb2.Timestamp() - message.FromDatetime(original_datetime) - self.assertEqual(7200, message.seconds) - self.assertEqual(0, message.nanos) - - # ToDatetime() with no parameters produces a naive UTC datetime, i.e. it not - # only loses the original timezone information (e.g. US/Pacific) as it's - # "normalised" to UTC, but also drops the information that the datetime - # represents a UTC one. - naive_datetime = message.ToDatetime() - self.assertEqual(datetime.datetime(1970, 1, 1, 2), naive_datetime) - self.assertIsNone(naive_datetime.tzinfo) - self.assertNotEqual(original_datetime, naive_datetime) # not even for UTC! - - # In contrast, ToDatetime(tzinfo=) produces an aware datetime in the given - # timezone. - aware_datetime = message.ToDatetime(tzinfo=tzinfo) - self.assertEqual(original_datetime, aware_datetime) - self.assertEqual( - datetime.datetime(1970, 1, 1, 2, tzinfo=datetime.timezone.utc), - aware_datetime) - self.assertEqual(tzinfo, aware_datetime.tzinfo) - - def testTimedeltaConversion(self): - message = duration_pb2.Duration() - message.FromNanoseconds(1999999999) - td = message.ToTimedelta() - self.assertEqual(1, td.seconds) - self.assertEqual(999999, td.microseconds) - - message.FromNanoseconds(-1999999999) - td = message.ToTimedelta() - self.assertEqual(-1, td.days) - self.assertEqual(86398, td.seconds) - self.assertEqual(1, td.microseconds) - - message.FromMicroseconds(-1) - td = message.ToTimedelta() - self.assertEqual(-1, td.days) - self.assertEqual(86399, td.seconds) - self.assertEqual(999999, td.microseconds) - converted_message = duration_pb2.Duration() - converted_message.FromTimedelta(td) - self.assertEqual(message, converted_message) - - def testInvalidTimestamp(self): - message = timestamp_pb2.Timestamp() - self.assertRaisesRegex( - ValueError, 'Failed to parse timestamp: missing valid timezone offset.', - message.FromJsonString, '') - self.assertRaisesRegex( - ValueError, 'Failed to parse timestamp: invalid trailing data ' - '1970-01-01T00:00:01Ztrail.', message.FromJsonString, - '1970-01-01T00:00:01Ztrail') - self.assertRaisesRegex( - ValueError, 'time data \'10000-01-01T00:00:00\' does not match' - ' format \'%Y-%m-%dT%H:%M:%S\'', message.FromJsonString, - '10000-01-01T00:00:00.00Z') - self.assertRaisesRegex( - ValueError, 'nanos 0123456789012 more than 9 fractional digits.', - message.FromJsonString, '1970-01-01T00:00:00.0123456789012Z') - self.assertRaisesRegex( - ValueError, - (r'Invalid timezone offset value: \+08.'), - message.FromJsonString, - '1972-01-01T01:00:00.01+08', - ) - self.assertRaisesRegex(ValueError, 'year (0 )?is out of range', - message.FromJsonString, '0000-01-01T00:00:00Z') - message.seconds = 253402300800 - self.assertRaisesRegex(OverflowError, 'date value out of range', - message.ToJsonString) - - def testInvalidDuration(self): - message = duration_pb2.Duration() - self.assertRaisesRegex(ValueError, 'Duration must end with letter "s": 1.', - message.FromJsonString, '1') - self.assertRaisesRegex(ValueError, 'Couldn\'t parse duration: 1...2s.', - message.FromJsonString, '1...2s') - text = '-315576000001.000000000s' - self.assertRaisesRegex( - ValueError, - r'Duration is not valid\: Seconds -315576000001 must be in range' - r' \[-315576000000\, 315576000000\].', message.FromJsonString, text) - text = '315576000001.000000000s' - self.assertRaisesRegex( - ValueError, - r'Duration is not valid\: Seconds 315576000001 must be in range' - r' \[-315576000000\, 315576000000\].', message.FromJsonString, text) - message.seconds = -315576000001 - message.nanos = 0 - self.assertRaisesRegex( - ValueError, - r'Duration is not valid\: Seconds -315576000001 must be in range' - r' \[-315576000000\, 315576000000\].', message.ToJsonString) - message.seconds = 0 - message.nanos = 999999999 + 1 - self.assertRaisesRegex( - ValueError, r'Duration is not valid\: Nanos 1000000000 must be in range' - r' \[-999999999\, 999999999\].', message.ToJsonString) - message.seconds = -1 - message.nanos = 1 - self.assertRaisesRegex(ValueError, - r'Duration is not valid\: Sign mismatch.', - message.ToJsonString) - - -class FieldMaskTest(unittest.TestCase): - - def testStringFormat(self): - mask = field_mask_pb2.FieldMask() - self.assertEqual('', mask.ToJsonString()) - mask.paths.append('foo') - self.assertEqual('foo', mask.ToJsonString()) - mask.paths.append('bar') - self.assertEqual('foo,bar', mask.ToJsonString()) - - mask.FromJsonString('') - self.assertEqual('', mask.ToJsonString()) - mask.FromJsonString('foo') - self.assertEqual(['foo'], mask.paths) - mask.FromJsonString('foo,bar') - self.assertEqual(['foo', 'bar'], mask.paths) - - # Test camel case - mask.Clear() - mask.paths.append('foo_bar') - self.assertEqual('fooBar', mask.ToJsonString()) - mask.paths.append('bar_quz') - self.assertEqual('fooBar,barQuz', mask.ToJsonString()) - - mask.FromJsonString('') - self.assertEqual('', mask.ToJsonString()) - self.assertEqual([], mask.paths) - mask.FromJsonString('fooBar') - self.assertEqual(['foo_bar'], mask.paths) - mask.FromJsonString('fooBar,barQuz') - self.assertEqual(['foo_bar', 'bar_quz'], mask.paths) - - def testDescriptorToFieldMask(self): - mask = field_mask_pb2.FieldMask() - msg_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - mask.AllFieldsFromDescriptor(msg_descriptor) - self.assertEqual(76, len(mask.paths)) - self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) - for field in msg_descriptor.fields: - self.assertTrue(field.name in mask.paths) - - def testIsValidForDescriptor(self): - msg_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR - # Empty mask - mask = field_mask_pb2.FieldMask() - self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) - # All fields from descriptor - mask.AllFieldsFromDescriptor(msg_descriptor) - self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) - # Child under optional message - mask.paths.append('optional_nested_message.bb') - self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) - # Repeated field is only allowed in the last position of path - mask.paths.append('repeated_nested_message.bb') - self.assertFalse(mask.IsValidForDescriptor(msg_descriptor)) - # Invalid top level field - mask = field_mask_pb2.FieldMask() - mask.paths.append('xxx') - self.assertFalse(mask.IsValidForDescriptor(msg_descriptor)) - # Invalid field in root - mask = field_mask_pb2.FieldMask() - mask.paths.append('xxx.zzz') - self.assertFalse(mask.IsValidForDescriptor(msg_descriptor)) - # Invalid field in internal node - mask = field_mask_pb2.FieldMask() - mask.paths.append('optional_nested_message.xxx.zzz') - self.assertFalse(mask.IsValidForDescriptor(msg_descriptor)) - # Invalid field in leaf - mask = field_mask_pb2.FieldMask() - mask.paths.append('optional_nested_message.xxx') - self.assertFalse(mask.IsValidForDescriptor(msg_descriptor)) - - def testCanonicalFrom(self): - mask = field_mask_pb2.FieldMask() - out_mask = field_mask_pb2.FieldMask() - # Paths will be sorted. - mask.FromJsonString('baz.quz,bar,foo') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('bar,baz.quz,foo', out_mask.ToJsonString()) - # Duplicated paths will be removed. - mask.FromJsonString('foo,bar,foo') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('bar,foo', out_mask.ToJsonString()) - # Sub-paths of other paths will be removed. - mask.FromJsonString('foo.b1,bar.b1,foo.b2,bar') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('bar,foo.b1,foo.b2', out_mask.ToJsonString()) - - # Test more deeply nested cases. - mask.FromJsonString( - 'foo.bar.baz1,foo.bar.baz2.quz,foo.bar.baz2') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('foo.bar.baz1,foo.bar.baz2', - out_mask.ToJsonString()) - mask.FromJsonString( - 'foo.bar.baz1,foo.bar.baz2,foo.bar.baz2.quz') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('foo.bar.baz1,foo.bar.baz2', - out_mask.ToJsonString()) - mask.FromJsonString( - 'foo.bar.baz1,foo.bar.baz2,foo.bar.baz2.quz,foo.bar') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('foo.bar', out_mask.ToJsonString()) - mask.FromJsonString( - 'foo.bar.baz1,foo.bar.baz2,foo.bar.baz2.quz,foo') - out_mask.CanonicalFormFromMask(mask) - self.assertEqual('foo', out_mask.ToJsonString()) - - def testUnion(self): - mask1 = field_mask_pb2.FieldMask() - mask2 = field_mask_pb2.FieldMask() - out_mask = field_mask_pb2.FieldMask() - mask1.FromJsonString('foo,baz') - mask2.FromJsonString('bar,quz') - out_mask.Union(mask1, mask2) - self.assertEqual('bar,baz,foo,quz', out_mask.ToJsonString()) - # Overlap with duplicated paths. - mask1.FromJsonString('foo,baz.bb') - mask2.FromJsonString('baz.bb,quz') - out_mask.Union(mask1, mask2) - self.assertEqual('baz.bb,foo,quz', out_mask.ToJsonString()) - # Overlap with paths covering some other paths. - mask1.FromJsonString('foo.bar.baz,quz') - mask2.FromJsonString('foo.bar,bar') - out_mask.Union(mask1, mask2) - self.assertEqual('bar,foo.bar,quz', out_mask.ToJsonString()) - src = unittest_pb2.TestAllTypes() - with self.assertRaises(ValueError): - out_mask.Union(src, mask2) - - def testIntersect(self): - mask1 = field_mask_pb2.FieldMask() - mask2 = field_mask_pb2.FieldMask() - out_mask = field_mask_pb2.FieldMask() - # Test cases without overlapping. - mask1.FromJsonString('foo,baz') - mask2.FromJsonString('bar,quz') - out_mask.Intersect(mask1, mask2) - self.assertEqual('', out_mask.ToJsonString()) - self.assertEqual(len(out_mask.paths), 0) - self.assertEqual(out_mask.paths, []) - # Overlap with duplicated paths. - mask1.FromJsonString('foo,baz.bb') - mask2.FromJsonString('baz.bb,quz') - out_mask.Intersect(mask1, mask2) - self.assertEqual('baz.bb', out_mask.ToJsonString()) - # Overlap with paths covering some other paths. - mask1.FromJsonString('foo.bar.baz,quz') - mask2.FromJsonString('foo.bar,bar') - out_mask.Intersect(mask1, mask2) - self.assertEqual('foo.bar.baz', out_mask.ToJsonString()) - mask1.FromJsonString('foo.bar,bar') - mask2.FromJsonString('foo.bar.baz,quz') - out_mask.Intersect(mask1, mask2) - self.assertEqual('foo.bar.baz', out_mask.ToJsonString()) - # Intersect '' with '' - mask1.Clear() - mask2.Clear() - mask1.paths.append('') - mask2.paths.append('') - self.assertEqual(mask1.paths, ['']) - self.assertEqual('', mask1.ToJsonString()) - out_mask.Intersect(mask1, mask2) - self.assertEqual(out_mask.paths, []) - - def testMergeMessageWithoutMapFields(self): - # Test merge one field. - src = unittest_pb2.TestAllTypes() - test_util.SetAllFields(src) - for field in src.DESCRIPTOR.fields: - if field.containing_oneof: - continue - field_name = field.name - dst = unittest_pb2.TestAllTypes() - # Only set one path to mask. - mask = field_mask_pb2.FieldMask() - mask.paths.append(field_name) - mask.MergeMessage(src, dst) - # The expected result message. - msg = unittest_pb2.TestAllTypes() - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - repeated_src = getattr(src, field_name) - repeated_msg = getattr(msg, field_name) - if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - for item in repeated_src: - repeated_msg.add().CopyFrom(item) - else: - repeated_msg.extend(repeated_src) - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - getattr(msg, field_name).CopyFrom(getattr(src, field_name)) - else: - setattr(msg, field_name, getattr(src, field_name)) - # Only field specified in mask is merged. - self.assertEqual(msg, dst) - - # Test merge nested fields. - nested_src = unittest_pb2.NestedTestAllTypes() - nested_dst = unittest_pb2.NestedTestAllTypes() - nested_src.child.payload.optional_int32 = 1234 - nested_src.child.child.payload.optional_int32 = 5678 - mask = field_mask_pb2.FieldMask() - mask.FromJsonString('child.payload') - mask.MergeMessage(nested_src, nested_dst) - self.assertEqual(1234, nested_dst.child.payload.optional_int32) - self.assertEqual(0, nested_dst.child.child.payload.optional_int32) - - mask.FromJsonString('child.child.payload') - mask.MergeMessage(nested_src, nested_dst) - self.assertEqual(1234, nested_dst.child.payload.optional_int32) - self.assertEqual(5678, nested_dst.child.child.payload.optional_int32) - - nested_dst.Clear() - mask.FromJsonString('child.child.payload') - mask.MergeMessage(nested_src, nested_dst) - self.assertEqual(0, nested_dst.child.payload.optional_int32) - self.assertEqual(5678, nested_dst.child.child.payload.optional_int32) - - nested_dst.Clear() - mask.FromJsonString('child') - mask.MergeMessage(nested_src, nested_dst) - self.assertEqual(1234, nested_dst.child.payload.optional_int32) - self.assertEqual(5678, nested_dst.child.child.payload.optional_int32) - - # Test MergeOptions. - nested_dst.Clear() - nested_dst.child.payload.optional_int64 = 4321 - # Message fields will be merged by default. - mask.FromJsonString('child.payload') - mask.MergeMessage(nested_src, nested_dst) - self.assertEqual(1234, nested_dst.child.payload.optional_int32) - self.assertEqual(4321, nested_dst.child.payload.optional_int64) - # Change the behavior to replace message fields. - mask.FromJsonString('child.payload') - mask.MergeMessage(nested_src, nested_dst, True, False) - self.assertEqual(1234, nested_dst.child.payload.optional_int32) - self.assertEqual(0, nested_dst.child.payload.optional_int64) - - # By default, fields missing in source are not cleared in destination. - nested_dst.payload.optional_int32 = 1234 - self.assertTrue(nested_dst.HasField('payload')) - mask.FromJsonString('payload') - mask.MergeMessage(nested_src, nested_dst) - self.assertTrue(nested_dst.HasField('payload')) - # But they are cleared when replacing message fields. - nested_dst.Clear() - nested_dst.payload.optional_int32 = 1234 - mask.FromJsonString('payload') - mask.MergeMessage(nested_src, nested_dst, True, False) - self.assertFalse(nested_dst.HasField('payload')) - - nested_src.payload.repeated_int32.append(1234) - nested_dst.payload.repeated_int32.append(5678) - # Repeated fields will be appended by default. - mask.FromJsonString('payload.repeatedInt32') - mask.MergeMessage(nested_src, nested_dst) - self.assertEqual(2, len(nested_dst.payload.repeated_int32)) - self.assertEqual(5678, nested_dst.payload.repeated_int32[0]) - self.assertEqual(1234, nested_dst.payload.repeated_int32[1]) - # Change the behavior to replace repeated fields. - mask.FromJsonString('payload.repeatedInt32') - mask.MergeMessage(nested_src, nested_dst, False, True) - self.assertEqual(1, len(nested_dst.payload.repeated_int32)) - self.assertEqual(1234, nested_dst.payload.repeated_int32[0]) - - # Test Merge oneof field. - new_msg = unittest_pb2.TestOneof2() - dst = unittest_pb2.TestOneof2() - dst.foo_message.moo_int = 1 - mask = field_mask_pb2.FieldMask() - mask.FromJsonString('fooMessage,fooLazyMessage.mooInt') - mask.MergeMessage(new_msg, dst) - self.assertTrue(dst.HasField('foo_message')) - self.assertFalse(dst.HasField('foo_lazy_message')) - - def testMergeMessageWithMapField(self): - empty_map = map_unittest_pb2.TestRecursiveMapMessage() - src_level_2 = map_unittest_pb2.TestRecursiveMapMessage() - src_level_2.a['src level 2'].CopyFrom(empty_map) - src = map_unittest_pb2.TestRecursiveMapMessage() - src.a['common key'].CopyFrom(src_level_2) - src.a['src level 1'].CopyFrom(src_level_2) - - dst_level_2 = map_unittest_pb2.TestRecursiveMapMessage() - dst_level_2.a['dst level 2'].CopyFrom(empty_map) - dst = map_unittest_pb2.TestRecursiveMapMessage() - dst.a['common key'].CopyFrom(dst_level_2) - dst.a['dst level 1'].CopyFrom(empty_map) - - mask = field_mask_pb2.FieldMask() - mask.FromJsonString('a') - mask.MergeMessage(src, dst) - - # map from dst is replaced with map from src. - self.assertEqual(dst.a['common key'], src_level_2) - self.assertEqual(dst.a['src level 1'], src_level_2) - self.assertEqual(dst.a['dst level 1'], empty_map) - - def testMergeErrors(self): - src = unittest_pb2.TestAllTypes() - dst = unittest_pb2.TestAllTypes() - mask = field_mask_pb2.FieldMask() - test_util.SetAllFields(src) - mask.FromJsonString('optionalInt32.field') - with self.assertRaises(ValueError) as e: - mask.MergeMessage(src, dst) - self.assertEqual('Error: Field optional_int32 in message ' - 'protobuf_unittest.TestAllTypes is not a singular ' - 'message field and cannot have sub-fields.', - str(e.exception)) - - def testSnakeCaseToCamelCase(self): - self.assertEqual('fooBar', - well_known_types._SnakeCaseToCamelCase('foo_bar')) - self.assertEqual('FooBar', - well_known_types._SnakeCaseToCamelCase('_foo_bar')) - self.assertEqual('foo3Bar', - well_known_types._SnakeCaseToCamelCase('foo3_bar')) - - # No uppercase letter is allowed. - self.assertRaisesRegex( - ValueError, - 'Fail to print FieldMask to Json string: Path name Foo must ' - 'not contain uppercase letters.', - well_known_types._SnakeCaseToCamelCase, 'Foo') - # Any character after a "_" must be a lowercase letter. - # 1. "_" cannot be followed by another "_". - # 2. "_" cannot be followed by a digit. - # 3. "_" cannot appear as the last character. - self.assertRaisesRegex( - ValueError, - 'Fail to print FieldMask to Json string: The character after a ' - '"_" must be a lowercase letter in path name foo__bar.', - well_known_types._SnakeCaseToCamelCase, 'foo__bar') - self.assertRaisesRegex( - ValueError, - 'Fail to print FieldMask to Json string: The character after a ' - '"_" must be a lowercase letter in path name foo_3bar.', - well_known_types._SnakeCaseToCamelCase, 'foo_3bar') - self.assertRaisesRegex( - ValueError, - 'Fail to print FieldMask to Json string: Trailing "_" in path ' - 'name foo_bar_.', well_known_types._SnakeCaseToCamelCase, 'foo_bar_') - - def testCamelCaseToSnakeCase(self): - self.assertEqual('foo_bar', - well_known_types._CamelCaseToSnakeCase('fooBar')) - self.assertEqual('_foo_bar', - well_known_types._CamelCaseToSnakeCase('FooBar')) - self.assertEqual('foo3_bar', - well_known_types._CamelCaseToSnakeCase('foo3Bar')) - self.assertRaisesRegex( - ValueError, - 'Fail to parse FieldMask: Path name foo_bar must not contain "_"s.', - well_known_types._CamelCaseToSnakeCase, 'foo_bar') - - -class StructTest(unittest.TestCase): - - def testStruct(self): - struct = struct_pb2.Struct() - self.assertIsInstance(struct, collections_abc.Mapping) - self.assertEqual(0, len(struct)) - struct_class = struct.__class__ - - struct['key1'] = 5 - struct['key2'] = 'abc' - struct['key3'] = True - struct.get_or_create_struct('key4')['subkey'] = 11.0 - struct_list = struct.get_or_create_list('key5') - self.assertIsInstance(struct_list, collections_abc.Sequence) - struct_list.extend([6, 'seven', True, False, None]) - struct_list.add_struct()['subkey2'] = 9 - struct['key6'] = {'subkey': {}} - struct['key7'] = [2, False] - - self.assertEqual(7, len(struct)) - self.assertTrue(isinstance(struct, well_known_types.Struct)) - self.assertEqual(5, struct['key1']) - self.assertEqual('abc', struct['key2']) - self.assertIs(True, struct['key3']) - self.assertEqual(11, struct['key4']['subkey']) - inner_struct = struct_class() - inner_struct['subkey2'] = 9 - self.assertEqual([6, 'seven', True, False, None, inner_struct], - list(struct['key5'].items())) - self.assertEqual({}, dict(struct['key6']['subkey'].fields)) - self.assertEqual([2, False], list(struct['key7'].items())) - - serialized = struct.SerializeToString() - struct2 = struct_pb2.Struct() - struct2.ParseFromString(serialized) - - self.assertEqual(struct, struct2) - for key, value in struct.items(): - self.assertIn(key, struct) - self.assertIn(key, struct2) - self.assertEqual(value, struct2[key]) - - self.assertEqual(7, len(struct.keys())) - self.assertEqual(7, len(struct.values())) - for key in struct.keys(): - self.assertIn(key, struct) - self.assertIn(key, struct2) - self.assertEqual(struct[key], struct2[key]) - - item = (next(iter(struct.keys())), next(iter(struct.values()))) - self.assertEqual(item, next(iter(struct.items()))) - - self.assertTrue(isinstance(struct2, well_known_types.Struct)) - self.assertEqual(5, struct2['key1']) - self.assertEqual('abc', struct2['key2']) - self.assertIs(True, struct2['key3']) - self.assertEqual(11, struct2['key4']['subkey']) - self.assertEqual([6, 'seven', True, False, None, inner_struct], - list(struct2['key5'].items())) - - struct_list = struct2['key5'] - self.assertEqual(6, struct_list[0]) - self.assertEqual('seven', struct_list[1]) - self.assertEqual(True, struct_list[2]) - self.assertEqual(False, struct_list[3]) - self.assertEqual(None, struct_list[4]) - self.assertEqual(inner_struct, struct_list[5]) - - struct_list[1] = 7 - self.assertEqual(7, struct_list[1]) - - struct_list.add_list().extend([1, 'two', True, False, None]) - self.assertEqual([1, 'two', True, False, None], - list(struct_list[6].items())) - struct_list.extend([{'nested_struct': 30}, ['nested_list', 99], {}, []]) - self.assertEqual(11, len(struct_list.values)) - self.assertEqual(30, struct_list[7]['nested_struct']) - self.assertEqual('nested_list', struct_list[8][0]) - self.assertEqual(99, struct_list[8][1]) - self.assertEqual({}, dict(struct_list[9].fields)) - self.assertEqual([], list(struct_list[10].items())) - struct_list[0] = {'replace': 'set'} - struct_list[1] = ['replace', 'set'] - self.assertEqual('set', struct_list[0]['replace']) - self.assertEqual(['replace', 'set'], list(struct_list[1].items())) - - text_serialized = str(struct) - struct3 = struct_pb2.Struct() - text_format.Merge(text_serialized, struct3) - self.assertEqual(struct, struct3) - - struct.get_or_create_struct('key3')['replace'] = 12 - self.assertEqual(12, struct['key3']['replace']) - - # Tests empty list. - struct.get_or_create_list('empty_list') - empty_list = struct['empty_list'] - self.assertEqual([], list(empty_list.items())) - list2 = struct_pb2.ListValue() - list2.add_list() - empty_list = list2[0] - self.assertEqual([], list(empty_list.items())) - - # Tests empty struct. - struct.get_or_create_struct('empty_struct') - empty_struct = struct['empty_struct'] - self.assertEqual({}, dict(empty_struct.fields)) - list2.add_struct() - empty_struct = list2[1] - self.assertEqual({}, dict(empty_struct.fields)) - - self.assertEqual(9, len(struct)) - del struct['key3'] - del struct['key4'] - self.assertEqual(7, len(struct)) - self.assertEqual(6, len(struct['key5'])) - del struct['key5'][1] - self.assertEqual(5, len(struct['key5'])) - self.assertEqual([6, True, False, None, inner_struct], - list(struct['key5'].items())) - - def testStructAssignment(self): - # Tests struct assignment from another struct - s1 = struct_pb2.Struct() - s2 = struct_pb2.Struct() - for value in [1, 'a', [1], ['a'], {'a': 'b'}]: - s1['x'] = value - s2['x'] = s1['x'] - self.assertEqual(s1['x'], s2['x']) - - def testMergeFrom(self): - struct = struct_pb2.Struct() - struct_class = struct.__class__ - - dictionary = { - 'key1': 5, - 'key2': 'abc', - 'key3': True, - 'key4': {'subkey': 11.0}, - 'key5': [6, 'seven', True, False, None, {'subkey2': 9}], - 'key6': [['nested_list', True]], - 'empty_struct': {}, - 'empty_list': [] - } - struct.update(dictionary) - self.assertEqual(5, struct['key1']) - self.assertEqual('abc', struct['key2']) - self.assertIs(True, struct['key3']) - self.assertEqual(11, struct['key4']['subkey']) - inner_struct = struct_class() - inner_struct['subkey2'] = 9 - self.assertEqual([6, 'seven', True, False, None, inner_struct], - list(struct['key5'].items())) - self.assertEqual(2, len(struct['key6'][0].values)) - self.assertEqual('nested_list', struct['key6'][0][0]) - self.assertEqual(True, struct['key6'][0][1]) - empty_list = struct['empty_list'] - self.assertEqual([], list(empty_list.items())) - empty_struct = struct['empty_struct'] - self.assertEqual({}, dict(empty_struct.fields)) - - # According to documentation: "When parsing from the wire or when merging, - # if there are duplicate map keys the last key seen is used". - duplicate = { - 'key4': {'replace': 20}, - 'key5': [[False, 5]] - } - struct.update(duplicate) - self.assertEqual(1, len(struct['key4'].fields)) - self.assertEqual(20, struct['key4']['replace']) - self.assertEqual(1, len(struct['key5'].values)) - self.assertEqual(False, struct['key5'][0][0]) - self.assertEqual(5, struct['key5'][0][1]) - - -class AnyTest(unittest.TestCase): - - def testAnyMessage(self): - # Creates and sets message. - msg = any_test_pb2.TestAny() - msg_descriptor = msg.DESCRIPTOR - all_types = unittest_pb2.TestAllTypes() - all_descriptor = all_types.DESCRIPTOR - all_types.repeated_string.append(u'\u00fc\ua71f') - # Packs to Any. - msg.value.Pack(all_types) - self.assertEqual(msg.value.type_url, - 'type.googleapis.com/%s' % all_descriptor.full_name) - self.assertEqual(msg.value.value, - all_types.SerializeToString()) - # Tests Is() method. - self.assertTrue(msg.value.Is(all_descriptor)) - self.assertFalse(msg.value.Is(msg_descriptor)) - # Unpacks Any. - unpacked_message = unittest_pb2.TestAllTypes() - self.assertTrue(msg.value.Unpack(unpacked_message)) - self.assertEqual(all_types, unpacked_message) - # Unpacks to different type. - self.assertFalse(msg.value.Unpack(msg)) - # Only Any messages have Pack method. - try: - msg.Pack(all_types) - except AttributeError: - pass - else: - raise AttributeError('%s should not have Pack method.' % - msg_descriptor.full_name) - - def testUnpackWithNoSlashInTypeUrl(self): - msg = any_test_pb2.TestAny() - all_types = unittest_pb2.TestAllTypes() - all_descriptor = all_types.DESCRIPTOR - msg.value.Pack(all_types) - # Reset type_url to part of type_url after '/' - msg.value.type_url = msg.value.TypeName() - self.assertFalse(msg.value.Is(all_descriptor)) - unpacked_message = unittest_pb2.TestAllTypes() - self.assertFalse(msg.value.Unpack(unpacked_message)) - - def testMessageName(self): - # Creates and sets message. - submessage = any_test_pb2.TestAny() - submessage.int_value = 12345 - msg = any_pb2.Any() - msg.Pack(submessage) - self.assertEqual(msg.TypeName(), 'google.protobuf.internal.TestAny') - - def testPackWithCustomTypeUrl(self): - submessage = any_test_pb2.TestAny() - submessage.int_value = 12345 - msg = any_pb2.Any() - # Pack with a custom type URL prefix. - msg.Pack(submessage, 'type.myservice.com') - self.assertEqual(msg.type_url, - 'type.myservice.com/%s' % submessage.DESCRIPTOR.full_name) - # Pack with a custom type URL prefix ending with '/'. - msg.Pack(submessage, 'type.myservice.com/') - self.assertEqual(msg.type_url, - 'type.myservice.com/%s' % submessage.DESCRIPTOR.full_name) - # Pack with an empty type URL prefix. - msg.Pack(submessage, '') - self.assertEqual(msg.type_url, - '/%s' % submessage.DESCRIPTOR.full_name) - # Test unpacking the type. - unpacked_message = any_test_pb2.TestAny() - self.assertTrue(msg.Unpack(unpacked_message)) - self.assertEqual(submessage, unpacked_message) - - def testPackDeterministic(self): - submessage = any_test_pb2.TestAny() - for i in range(10): - submessage.map_value[str(i)] = i * 2 - msg = any_pb2.Any() - msg.Pack(submessage, deterministic=True) - serialized = msg.SerializeToString(deterministic=True) - golden = (b'\n4type.googleapis.com/google.protobuf.internal.TestAny\x12F' - b'\x1a\x05\n\x010\x10\x00\x1a\x05\n\x011\x10\x02\x1a\x05\n\x01' - b'2\x10\x04\x1a\x05\n\x013\x10\x06\x1a\x05\n\x014\x10\x08\x1a' - b'\x05\n\x015\x10\n\x1a\x05\n\x016\x10\x0c\x1a\x05\n\x017\x10' - b'\x0e\x1a\x05\n\x018\x10\x10\x1a\x05\n\x019\x10\x12') - self.assertEqual(golden, serialized) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/internal/wire_format.py b/ext/protobuf/Python/google/protobuf/internal/wire_format.py index 883f52558..1f54414b1 100644 --- a/ext/protobuf/Python/google/protobuf/internal/wire_format.py +++ b/ext/protobuf/Python/google/protobuf/internal/wire_format.py @@ -43,7 +43,7 @@ # These numbers identify the wire type of a protocol buffer value. # We use the least-significant TAG_TYPE_BITS bits of the varint-encoded # tag-and-type to store one of these WIRETYPE_* constants. -# These values must match WireType enum in google/protobuf/wire_format.h. +# These values must match WireType enum in //google/protobuf/wire_format.h. WIRETYPE_VARINT = 0 WIRETYPE_FIXED64 = 1 WIRETYPE_LENGTH_DELIMITED = 2 diff --git a/ext/protobuf/Python/google/protobuf/internal/wire_format_test.py b/ext/protobuf/Python/google/protobuf/internal/wire_format_test.py deleted file mode 100644 index f7ad0c79a..000000000 --- a/ext/protobuf/Python/google/protobuf/internal/wire_format_test.py +++ /dev/null @@ -1,252 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test for google.protobuf.internal.wire_format.""" - -__author__ = 'robinson@google.com (Will Robinson)' - -import unittest - -from google.protobuf import message -from google.protobuf.internal import wire_format - - -class WireFormatTest(unittest.TestCase): - - def testPackTag(self): - field_number = 0xabc - tag_type = 2 - self.assertEqual((field_number << 3) | tag_type, - wire_format.PackTag(field_number, tag_type)) - PackTag = wire_format.PackTag - # Number too high. - self.assertRaises(message.EncodeError, PackTag, field_number, 6) - # Number too low. - self.assertRaises(message.EncodeError, PackTag, field_number, -1) - - def testUnpackTag(self): - # Test field numbers that will require various varint sizes. - for expected_field_number in (1, 15, 16, 2047, 2048): - for expected_wire_type in range(6): # Highest-numbered wiretype is 5. - field_number, wire_type = wire_format.UnpackTag( - wire_format.PackTag(expected_field_number, expected_wire_type)) - self.assertEqual(expected_field_number, field_number) - self.assertEqual(expected_wire_type, wire_type) - - self.assertRaises(TypeError, wire_format.UnpackTag, None) - self.assertRaises(TypeError, wire_format.UnpackTag, 'abc') - self.assertRaises(TypeError, wire_format.UnpackTag, 0.0) - self.assertRaises(TypeError, wire_format.UnpackTag, object()) - - def testZigZagEncode(self): - Z = wire_format.ZigZagEncode - self.assertEqual(0, Z(0)) - self.assertEqual(1, Z(-1)) - self.assertEqual(2, Z(1)) - self.assertEqual(3, Z(-2)) - self.assertEqual(4, Z(2)) - self.assertEqual(0xfffffffe, Z(0x7fffffff)) - self.assertEqual(0xffffffff, Z(-0x80000000)) - self.assertEqual(0xfffffffffffffffe, Z(0x7fffffffffffffff)) - self.assertEqual(0xffffffffffffffff, Z(-0x8000000000000000)) - - self.assertRaises(TypeError, Z, None) - self.assertRaises(TypeError, Z, 'abcd') - self.assertRaises(TypeError, Z, 0.0) - self.assertRaises(TypeError, Z, object()) - - def testZigZagDecode(self): - Z = wire_format.ZigZagDecode - self.assertEqual(0, Z(0)) - self.assertEqual(-1, Z(1)) - self.assertEqual(1, Z(2)) - self.assertEqual(-2, Z(3)) - self.assertEqual(2, Z(4)) - self.assertEqual(0x7fffffff, Z(0xfffffffe)) - self.assertEqual(-0x80000000, Z(0xffffffff)) - self.assertEqual(0x7fffffffffffffff, Z(0xfffffffffffffffe)) - self.assertEqual(-0x8000000000000000, Z(0xffffffffffffffff)) - - self.assertRaises(TypeError, Z, None) - self.assertRaises(TypeError, Z, 'abcd') - self.assertRaises(TypeError, Z, 0.0) - self.assertRaises(TypeError, Z, object()) - - def NumericByteSizeTestHelper(self, byte_size_fn, value, expected_value_size): - # Use field numbers that cause various byte sizes for the tag information. - for field_number, tag_bytes in ((15, 1), (16, 2), (2047, 2), (2048, 3)): - expected_size = expected_value_size + tag_bytes - actual_size = byte_size_fn(field_number, value) - self.assertEqual(expected_size, actual_size, - 'byte_size_fn: %s, field_number: %d, value: %r\n' - 'Expected: %d, Actual: %d'% ( - byte_size_fn, field_number, value, expected_size, actual_size)) - - def testByteSizeFunctions(self): - # Test all numeric *ByteSize() functions. - NUMERIC_ARGS = [ - # Int32ByteSize(). - [wire_format.Int32ByteSize, 0, 1], - [wire_format.Int32ByteSize, 127, 1], - [wire_format.Int32ByteSize, 128, 2], - [wire_format.Int32ByteSize, -1, 10], - # Int64ByteSize(). - [wire_format.Int64ByteSize, 0, 1], - [wire_format.Int64ByteSize, 127, 1], - [wire_format.Int64ByteSize, 128, 2], - [wire_format.Int64ByteSize, -1, 10], - # UInt32ByteSize(). - [wire_format.UInt32ByteSize, 0, 1], - [wire_format.UInt32ByteSize, 127, 1], - [wire_format.UInt32ByteSize, 128, 2], - [wire_format.UInt32ByteSize, wire_format.UINT32_MAX, 5], - # UInt64ByteSize(). - [wire_format.UInt64ByteSize, 0, 1], - [wire_format.UInt64ByteSize, 127, 1], - [wire_format.UInt64ByteSize, 128, 2], - [wire_format.UInt64ByteSize, wire_format.UINT64_MAX, 10], - # SInt32ByteSize(). - [wire_format.SInt32ByteSize, 0, 1], - [wire_format.SInt32ByteSize, -1, 1], - [wire_format.SInt32ByteSize, 1, 1], - [wire_format.SInt32ByteSize, -63, 1], - [wire_format.SInt32ByteSize, 63, 1], - [wire_format.SInt32ByteSize, -64, 1], - [wire_format.SInt32ByteSize, 64, 2], - # SInt64ByteSize(). - [wire_format.SInt64ByteSize, 0, 1], - [wire_format.SInt64ByteSize, -1, 1], - [wire_format.SInt64ByteSize, 1, 1], - [wire_format.SInt64ByteSize, -63, 1], - [wire_format.SInt64ByteSize, 63, 1], - [wire_format.SInt64ByteSize, -64, 1], - [wire_format.SInt64ByteSize, 64, 2], - # Fixed32ByteSize(). - [wire_format.Fixed32ByteSize, 0, 4], - [wire_format.Fixed32ByteSize, wire_format.UINT32_MAX, 4], - # Fixed64ByteSize(). - [wire_format.Fixed64ByteSize, 0, 8], - [wire_format.Fixed64ByteSize, wire_format.UINT64_MAX, 8], - # SFixed32ByteSize(). - [wire_format.SFixed32ByteSize, 0, 4], - [wire_format.SFixed32ByteSize, wire_format.INT32_MIN, 4], - [wire_format.SFixed32ByteSize, wire_format.INT32_MAX, 4], - # SFixed64ByteSize(). - [wire_format.SFixed64ByteSize, 0, 8], - [wire_format.SFixed64ByteSize, wire_format.INT64_MIN, 8], - [wire_format.SFixed64ByteSize, wire_format.INT64_MAX, 8], - # FloatByteSize(). - [wire_format.FloatByteSize, 0.0, 4], - [wire_format.FloatByteSize, 1000000000.0, 4], - [wire_format.FloatByteSize, -1000000000.0, 4], - # DoubleByteSize(). - [wire_format.DoubleByteSize, 0.0, 8], - [wire_format.DoubleByteSize, 1000000000.0, 8], - [wire_format.DoubleByteSize, -1000000000.0, 8], - # BoolByteSize(). - [wire_format.BoolByteSize, False, 1], - [wire_format.BoolByteSize, True, 1], - # EnumByteSize(). - [wire_format.EnumByteSize, 0, 1], - [wire_format.EnumByteSize, 127, 1], - [wire_format.EnumByteSize, 128, 2], - [wire_format.EnumByteSize, wire_format.UINT32_MAX, 5], - ] - for args in NUMERIC_ARGS: - self.NumericByteSizeTestHelper(*args) - - # Test strings and bytes. - for byte_size_fn in (wire_format.StringByteSize, wire_format.BytesByteSize): - # 1 byte for tag, 1 byte for length, 3 bytes for contents. - self.assertEqual(5, byte_size_fn(10, 'abc')) - # 2 bytes for tag, 1 byte for length, 3 bytes for contents. - self.assertEqual(6, byte_size_fn(16, 'abc')) - # 2 bytes for tag, 2 bytes for length, 128 bytes for contents. - self.assertEqual(132, byte_size_fn(16, 'a' * 128)) - - # Test UTF-8 string byte size calculation. - # 1 byte for tag, 1 byte for length, 8 bytes for content. - self.assertEqual(10, wire_format.StringByteSize( - 5, b'\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82'.decode('utf-8'))) - - class MockMessage(object): - def __init__(self, byte_size): - self.byte_size = byte_size - def ByteSize(self): - return self.byte_size - - message_byte_size = 10 - mock_message = MockMessage(byte_size=message_byte_size) - # Test groups. - # (2 * 1) bytes for begin and end tags, plus message_byte_size. - self.assertEqual(2 + message_byte_size, - wire_format.GroupByteSize(1, mock_message)) - # (2 * 2) bytes for begin and end tags, plus message_byte_size. - self.assertEqual(4 + message_byte_size, - wire_format.GroupByteSize(16, mock_message)) - - # Test messages. - # 1 byte for tag, plus 1 byte for length, plus contents. - self.assertEqual(2 + mock_message.byte_size, - wire_format.MessageByteSize(1, mock_message)) - # 2 bytes for tag, plus 1 byte for length, plus contents. - self.assertEqual(3 + mock_message.byte_size, - wire_format.MessageByteSize(16, mock_message)) - # 2 bytes for tag, plus 2 bytes for length, plus contents. - mock_message.byte_size = 128 - self.assertEqual(4 + mock_message.byte_size, - wire_format.MessageByteSize(16, mock_message)) - - - # Test message set item byte size. - # 4 bytes for tags, plus 1 byte for length, plus 1 byte for type_id, - # plus contents. - mock_message.byte_size = 10 - self.assertEqual(mock_message.byte_size + 6, - wire_format.MessageSetItemByteSize(1, mock_message)) - - # 4 bytes for tags, plus 2 bytes for length, plus 1 byte for type_id, - # plus contents. - mock_message.byte_size = 128 - self.assertEqual(mock_message.byte_size + 7, - wire_format.MessageSetItemByteSize(1, mock_message)) - - # 4 bytes for tags, plus 2 bytes for length, plus 2 byte for type_id, - # plus contents. - self.assertEqual(mock_message.byte_size + 8, - wire_format.MessageSetItemByteSize(128, mock_message)) - - # Too-long varint. - self.assertRaises(message.EncodeError, - wire_format.UInt64ByteSize, 1, 1 << 128) - - -if __name__ == '__main__': - unittest.main() diff --git a/ext/protobuf/Python/google/protobuf/json_format.py b/ext/protobuf/Python/google/protobuf/json_format.py index 5024ed89d..a04e8aef1 100644 --- a/ext/protobuf/Python/google/protobuf/json_format.py +++ b/ext/protobuf/Python/google/protobuf/json_format.py @@ -53,6 +53,7 @@ from google.protobuf.internal import type_checkers from google.protobuf import descriptor +from google.protobuf import message_factory from google.protobuf import symbol_database @@ -109,7 +110,8 @@ def MessageToJson( names as defined in the .proto file. If False, convert the field names to lowerCamelCase. indent: The JSON object will be pretty-printed with this indent level. - An indent level of 0 or negative will only insert newlines. + An indent level of 0 or negative will only insert newlines. If the + indent level is None, no newlines will be inserted. sort_keys: If True, then the output will be sorted by field names. use_integers_for_enums: If true, print integers instead of enum names. descriptor_pool: A Descriptor Pool for resolving types. If None use the @@ -269,7 +271,7 @@ def _RegularMessageToJsonObject(self, message, js): except ValueError as e: raise SerializeToJsonError( - 'Failed to serialize {0} field: {1}.'.format(field.name, e)) + 'Failed to serialize {0} field: {1}.'.format(field.name, e)) from e return js @@ -286,10 +288,11 @@ def _FieldToJsonObject(self, field, value): if enum_value is not None: return enum_value.name else: - if field.file.syntax == 'proto3': + if field.enum_type.is_closed: + raise SerializeToJsonError('Enum field contains an integer value ' + 'which can not mapped to an enum value.') + else: return value - raise SerializeToJsonError('Enum field contains an integer value ' - 'which can not mapped to an enum value.') elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: if field.type == descriptor.FieldDescriptor.TYPE_BYTES: # Use base64 Data encoding for bytes @@ -352,8 +355,14 @@ def _ValueMessageToJsonObject(self, message): return None if which == 'list_value': return self._ListValueMessageToJsonObject(message.list_value) - if which == 'struct_value': - value = message.struct_value + if which == 'number_value': + value = message.number_value + if math.isinf(value): + raise ValueError('Fail to serialize Infinity for Value.number_value, ' + 'which would parse as string_value') + if math.isnan(value): + raise ValueError('Fail to serialize NaN for Value.number_value, ' + 'which would parse as string_value') else: value = getattr(message, which) oneof_descriptor = message.DESCRIPTOR.fields_by_name[which] @@ -397,10 +406,11 @@ def _CreateMessageFromTypeUrl(type_url, descriptor_pool): type_name = type_url.split('/')[-1] try: message_descriptor = pool.FindMessageTypeByName(type_name) - except KeyError: + except KeyError as e: raise TypeError( - 'Can not find message descriptor by type_url: {0}'.format(type_url)) - message_class = db.GetPrototype(message_descriptor) + 'Can not find message descriptor by type_url: {0}'.format(type_url) + ) from e + message_class = message_factory.GetMessageClass(message_descriptor) return message_class() @@ -432,7 +442,7 @@ def Parse(text, try: js = json.loads(text, object_pairs_hook=_DuplicateChecker) except ValueError as e: - raise ParseError('Failed to load JSON: {0}.'.format(str(e))) + raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e return ParseDict(js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth) @@ -624,13 +634,19 @@ def _ConvertFieldValuePair(self, js, message, path): '{0}.{1}'.format(path, name))) except ParseError as e: if field and field.containing_oneof is None: - raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) + raise ParseError( + 'Failed to parse {0} field: {1}.'.format(name, e) + ) from e else: - raise ParseError(str(e)) + raise ParseError(str(e)) from e except ValueError as e: - raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) + raise ParseError( + 'Failed to parse {0} field: {1}.'.format(name, e) + ) from e except TypeError as e: - raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) + raise ParseError( + 'Failed to parse {0} field: {1}.'.format(name, e) + ) from e def _ConvertAnyMessage(self, value, message, path): """Convert a JSON representation into Any message.""" @@ -638,14 +654,15 @@ def _ConvertAnyMessage(self, value, message, path): return try: type_url = value['@type'] - except KeyError: + except KeyError as e: raise ParseError( - '@type is missing when parsing any message at {0}'.format(path)) + '@type is missing when parsing any message at {0}'.format(path) + ) from e try: sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool) except TypeError as e: - raise ParseError('{0} at {1}'.format(e, path)) + raise ParseError('{0} at {1}'.format(e, path)) from e message_descriptor = sub_message.DESCRIPTOR full_name = message_descriptor.full_name if _IsWrapperMessage(message_descriptor): @@ -670,7 +687,7 @@ def _ConvertGenericMessage(self, value, message, path): try: message.FromJsonString(value) except ValueError as e: - raise ParseError('{0} at {1}'.format(e, path)) + raise ParseError('{0} at {1}'.format(e, path)) from e def _ConvertValueMessage(self, value, message, path): """Convert a JSON representation into Value message.""" @@ -794,18 +811,18 @@ def _ConvertScalarFieldValue(value, field, path, require_str=False): try: number = int(value) enum_value = field.enum_type.values_by_number.get(number, None) - except ValueError: + except ValueError as e: raise ParseError('Invalid enum value {0} for enum type {1}'.format( - value, field.enum_type.full_name)) + value, field.enum_type.full_name)) from e if enum_value is None: - if field.file.syntax == 'proto3': - # Proto3 accepts unknown enums. + if field.enum_type.is_closed: + raise ParseError('Invalid enum value {0} for enum type {1}'.format( + value, field.enum_type.full_name)) + else: return number - raise ParseError('Invalid enum value {0} for enum type {1}'.format( - value, field.enum_type.full_name)) return enum_value.number except ParseError as e: - raise ParseError('{0} at {1}'.format(e, path)) + raise ParseError('{0} at {1}'.format(e, path)) from e def _ConvertInteger(value): @@ -857,7 +874,7 @@ def _ConvertFloat(value, field): try: # Assume Python compatible syntax. return float(value) - except ValueError: + except ValueError as e: # Check alternative spellings. if value == _NEG_INFINITY: return float('-inf') @@ -866,7 +883,7 @@ def _ConvertFloat(value, field): elif value == _NAN: return float('nan') else: - raise ParseError('Couldn\'t parse float: {0}'.format(value)) + raise ParseError('Couldn\'t parse float: {0}'.format(value)) from e def _ConvertBool(value, require_str): diff --git a/ext/protobuf/Python/google/protobuf/message.py b/ext/protobuf/Python/google/protobuf/message.py index 76c6802f7..37b9c4054 100644 --- a/ext/protobuf/Python/google/protobuf/message.py +++ b/ext/protobuf/Python/google/protobuf/message.py @@ -74,7 +74,8 @@ class Message(object): __slots__ = [] - #: The :class:`google.protobuf.descriptor.Descriptor` for this message type. + #: The :class:`google.protobuf.Descriptor` + # for this message type. DESCRIPTOR = None def __deepcopy__(self, memo=None): @@ -191,7 +192,7 @@ def MergeFromString(self, serialized): raise NotImplementedError def ParseFromString(self, serialized): - """Parse serialized protocol buffer data into this message. + """Parse serialized protocol buffer data in binary form into this message. Like :func:`MergeFromString()`, except we clear the object first. @@ -311,13 +312,13 @@ def WhichOneof(self, oneof_group): """ raise NotImplementedError - def HasExtension(self, extension_handle): + def HasExtension(self, field_descriptor): """Checks if a certain extension is present for this message. Extensions are retrieved using the :attr:`Extensions` mapping (if present). Args: - extension_handle: The handle for the extension to check. + field_descriptor: The field descriptor for the extension to check. Returns: bool: Whether the extension is present for this message. @@ -329,11 +330,11 @@ def HasExtension(self, extension_handle): """ raise NotImplementedError - def ClearExtension(self, extension_handle): + def ClearExtension(self, field_descriptor): """Clears the contents of a given extension. Args: - extension_handle: The handle for the extension to clear. + field_descriptor: The field descriptor for the extension to clear. """ raise NotImplementedError @@ -367,7 +368,7 @@ def FromString(cls, s): raise NotImplementedError @staticmethod - def RegisterExtension(extension_handle): + def RegisterExtension(field_descriptor): raise NotImplementedError def _SetListener(self, message_listener): diff --git a/ext/protobuf/Python/google/protobuf/message_factory.py b/ext/protobuf/Python/google/protobuf/message_factory.py index 8d6520458..fac1165c5 100644 --- a/ext/protobuf/Python/google/protobuf/message_factory.py +++ b/ext/protobuf/Python/google/protobuf/message_factory.py @@ -39,6 +39,8 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' +import warnings + from google.protobuf.internal import api_implementation from google.protobuf import descriptor_pool from google.protobuf import message @@ -53,6 +55,95 @@ _GENERATED_PROTOCOL_MESSAGE_TYPE = message_impl.GeneratedProtocolMessageType +def GetMessageClass(descriptor): + """Obtains a proto2 message class based on the passed in descriptor. + + Passing a descriptor with a fully qualified name matching a previous + invocation will cause the same class to be returned. + + Args: + descriptor: The descriptor to build from. + + Returns: + A class describing the passed in descriptor. + """ + concrete_class = getattr(descriptor, '_concrete_class', None) + if concrete_class: + return concrete_class + return _InternalCreateMessageClass(descriptor) + + +def GetMessageClassesForFiles(files, pool): + """Gets all the messages from specified files. + + This will find and resolve dependencies, failing if the descriptor + pool cannot satisfy them. + + Args: + files: The file names to extract messages from. + pool: The descriptor pool to find the files including the dependent + files. + + Returns: + A dictionary mapping proto names to the message classes. + """ + result = {} + for file_name in files: + file_desc = pool.FindFileByName(file_name) + for desc in file_desc.message_types_by_name.values(): + result[desc.full_name] = GetMessageClass(desc) + + # While the extension FieldDescriptors are created by the descriptor pool, + # the python classes created in the factory need them to be registered + # explicitly, which is done below. + # + # The call to RegisterExtension will specifically check if the + # extension was already registered on the object and either + # ignore the registration if the original was the same, or raise + # an error if they were different. + + for extension in file_desc.extensions_by_name.values(): + extended_class = GetMessageClass(extension.containing_type) + extended_class.RegisterExtension(extension) + # Recursively load protos for extension field, in order to be able to + # fully represent the extension. This matches the behavior for regular + # fields too. + if extension.message_type: + GetMessageClass(extension.message_type) + return result + + +def _InternalCreateMessageClass(descriptor): + """Builds a proto2 message class based on the passed in descriptor. + + Args: + descriptor: The descriptor to build from. + + Returns: + A class describing the passed in descriptor. + """ + descriptor_name = descriptor.name + result_class = _GENERATED_PROTOCOL_MESSAGE_TYPE( + descriptor_name, + (message.Message,), + { + 'DESCRIPTOR': descriptor, + # If module not set, it wrongly points to message_factory module. + '__module__': None, + }) + for field in descriptor.fields: + if field.message_type: + GetMessageClass(field.message_type) + for extension in result_class.DESCRIPTOR.extensions: + extended_class = GetMessageClass(extension.containing_type) + extended_class.RegisterExtension(extension) + if extension.message_type: + GetMessageClass(extension.message_type) + return result_class + + +# Deprecated. Please use GetMessageClass() or GetMessageClassesForFiles() +# method above instead. class MessageFactory(object): """Factory for creating Proto2 messages from descriptors in a pool.""" @@ -60,9 +151,6 @@ def __init__(self, pool=None): """Initializes a new factory.""" self.pool = pool or descriptor_pool.DescriptorPool() - # local cache of all classes built from protobuf descriptors - self._classes = {} - def GetPrototype(self, descriptor): """Obtains a proto2 message class based on the passed in descriptor. @@ -75,21 +163,17 @@ def GetPrototype(self, descriptor): Returns: A class describing the passed in descriptor. """ - if descriptor not in self._classes: - result_class = self.CreatePrototype(descriptor) - # The assignment to _classes is redundant for the base implementation, but - # might avoid confusion in cases where CreatePrototype gets overridden and - # does not call the base implementation. - self._classes[descriptor] = result_class - return result_class - return self._classes[descriptor] + # TODO(b/258832141): add this warning + # warnings.warn('MessageFactory class is deprecated. Please use ' + # 'GetMessageClass() instead of MessageFactory.GetPrototype. ' + # 'MessageFactory class will be removed after 2024.') + return GetMessageClass(descriptor) def CreatePrototype(self, descriptor): """Builds a proto2 message class based on the passed in descriptor. Don't call this function directly, it always creates a new class. Call - GetPrototype() instead. This method is meant to be overridden in subblasses - to perform additional operations on the newly constructed class. + GetMessageClass() instead. Args: descriptor: The descriptor to build from. @@ -97,30 +181,11 @@ def CreatePrototype(self, descriptor): Returns: A class describing the passed in descriptor. """ - descriptor_name = descriptor.name - result_class = _GENERATED_PROTOCOL_MESSAGE_TYPE( - descriptor_name, - (message.Message,), - { - 'DESCRIPTOR': descriptor, - # If module not set, it wrongly points to message_factory module. - '__module__': None, - }) - result_class._FACTORY = self # pylint: disable=protected-access - # Assign in _classes before doing recursive calls to avoid infinite - # recursion. - self._classes[descriptor] = result_class - for field in descriptor.fields: - if field.message_type: - self.GetPrototype(field.message_type) - for extension in result_class.DESCRIPTOR.extensions: - if extension.containing_type not in self._classes: - self.GetPrototype(extension.containing_type) - extended_class = self._classes[extension.containing_type] - extended_class.RegisterExtension(extension) - if extension.message_type: - self.GetPrototype(extension.message_type) - return result_class + # TODO(b/258832141): add this warning + # warnings.warn('Directly call CreatePrototype is wrong. Please use ' + # 'GetMessageClass() method instead. Directly use ' + # 'CreatePrototype will raise error after July 2023.') + return _InternalCreateMessageClass(descriptor) def GetMessages(self, files): """Gets all the messages from a specified file. @@ -136,39 +201,20 @@ def GetMessages(self, files): any dependent messages as well as any messages defined in the same file as a specified message. """ - result = {} - for file_name in files: - file_desc = self.pool.FindFileByName(file_name) - for desc in file_desc.message_types_by_name.values(): - result[desc.full_name] = self.GetPrototype(desc) - - # While the extension FieldDescriptors are created by the descriptor pool, - # the python classes created in the factory need them to be registered - # explicitly, which is done below. - # - # The call to RegisterExtension will specifically check if the - # extension was already registered on the object and either - # ignore the registration if the original was the same, or raise - # an error if they were different. - - for extension in file_desc.extensions_by_name.values(): - if extension.containing_type not in self._classes: - self.GetPrototype(extension.containing_type) - extended_class = self._classes[extension.containing_type] - extended_class.RegisterExtension(extension) - if extension.message_type: - self.GetPrototype(extension.message_type) - return result - - -_FACTORY = MessageFactory() - - -def GetMessages(file_protos): + # TODO(b/258832141): add this warning + # warnings.warn('MessageFactory class is deprecated. Please use ' + # 'GetMessageClassesForFiles() instead of ' + # 'MessageFactory.GetMessages(). MessageFactory class ' + # 'will be removed after 2024.') + return GetMessageClassesForFiles(files, self.pool) + + +def GetMessages(file_protos, pool=None): """Builds a dictionary of all the messages available in a set of files. Args: file_protos: Iterable of FileDescriptorProto to build messages out of. + pool: The descriptor pool to add the file protos. Returns: A dictionary mapping proto names to the message classes. This will include @@ -177,13 +223,15 @@ def GetMessages(file_protos): """ # The cpp implementation of the protocol buffer library requires to add the # message in topological order of the dependency graph. + des_pool = pool or descriptor_pool.DescriptorPool() file_by_name = {file_proto.name: file_proto for file_proto in file_protos} def _AddFile(file_proto): for dependency in file_proto.dependency: if dependency in file_by_name: # Remove from elements to be visited, in order to cut cycles. _AddFile(file_by_name.pop(dependency)) - _FACTORY.pool.Add(file_proto) + des_pool.Add(file_proto) while file_by_name: _AddFile(file_by_name.popitem()[1]) - return _FACTORY.GetMessages([file_proto.name for file_proto in file_protos]) + return GetMessageClassesForFiles( + [file_proto.name for file_proto in file_protos], des_pool) diff --git a/ext/protobuf/Python/google/protobuf/proto_builder.py b/ext/protobuf/Python/google/protobuf/proto_builder.py index a4667ce63..8dab8b3ee 100644 --- a/ext/protobuf/Python/google/protobuf/proto_builder.py +++ b/ext/protobuf/Python/google/protobuf/proto_builder.py @@ -36,22 +36,23 @@ from google.protobuf import descriptor_pb2 from google.protobuf import descriptor +from google.protobuf import descriptor_pool from google.protobuf import message_factory -def _GetMessageFromFactory(factory, full_name): +def _GetMessageFromFactory(pool, full_name): """Get a proto class from the MessageFactory by name. Args: - factory: a MessageFactory instance. + pool: a descriptor pool. full_name: str, the fully qualified name of the proto type. Returns: A class, for the type identified by full_name. Raises: KeyError, if the proto is not found in the factory's descriptor pool. """ - proto_descriptor = factory.pool.FindMessageTypeByName(full_name) - proto_cls = factory.GetPrototype(proto_descriptor) + proto_descriptor = pool.FindMessageTypeByName(full_name) + proto_cls = message_factory.GetMessageClass(proto_descriptor) return proto_cls @@ -69,11 +70,10 @@ def MakeSimpleProtoClass(fields, full_name=None, pool=None): Returns: a class, the new protobuf class with a FileDescriptor. """ - factory = message_factory.MessageFactory(pool=pool) - + pool_instance = pool or descriptor_pool.DescriptorPool() if full_name is not None: try: - proto_cls = _GetMessageFromFactory(factory, full_name) + proto_cls = _GetMessageFromFactory(pool_instance, full_name) return proto_cls except KeyError: # The factory's DescriptorPool doesn't know about this class yet. @@ -99,16 +99,16 @@ def MakeSimpleProtoClass(fields, full_name=None, pool=None): full_name = ('net.proto2.python.public.proto_builder.AnonymousProto_' + fields_hash.hexdigest()) try: - proto_cls = _GetMessageFromFactory(factory, full_name) + proto_cls = _GetMessageFromFactory(pool_instance, full_name) return proto_cls except KeyError: # The factory's DescriptorPool doesn't know about this class yet. pass # This is the first time we see this proto: add a new descriptor to the pool. - factory.pool.Add( + pool_instance.Add( _MakeFileDescriptorProto(proto_file_name, full_name, field_items)) - return _GetMessageFromFactory(factory, full_name) + return _GetMessageFromFactory(pool_instance, full_name) def _MakeFileDescriptorProto(proto_file_name, full_name, field_items): diff --git a/ext/protobuf/Python/google/protobuf/reflection.py b/ext/protobuf/Python/google/protobuf/reflection.py index 81e18859a..1627669b9 100644 --- a/ext/protobuf/Python/google/protobuf/reflection.py +++ b/ext/protobuf/Python/google/protobuf/reflection.py @@ -92,4 +92,4 @@ def MakeClass(descriptor): # Original implementation leads to duplicate message classes, which won't play # well with extensions. Message factory info is also missing. # Redirect to message_factory. - return symbol_database.Default().GetPrototype(descriptor) + return message_factory.GetMessageClass(descriptor) diff --git a/ext/protobuf/Python/google/protobuf/source_context_pb2.py b/ext/protobuf/Python/google/protobuf/source_context_pb2.py index 7a6d0c1aa..51d91e7fd 100644 --- a/ext/protobuf/Python/google/protobuf/source_context_pb2.py +++ b/ext/protobuf/Python/google/protobuf/source_context_pb2.py @@ -15,12 +15,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$google/protobuf/source_context.proto\x12\x0fgoogle.protobuf\",\n\rSourceContext\x12\x1b\n\tfile_name\x18\x01 \x01(\tR\x08\x66ileNameB\x8a\x01\n\x13\x63om.google.protobufB\x12SourceContextProtoP\x01Z6google.golang.org/protobuf/types/known/sourcecontextpb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.source_context_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.source_context_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\022SourceContextProtoP\001Z6google.golang.org/protobuf/types/known/sourcecontextpb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _SOURCECONTEXT._serialized_start=57 - _SOURCECONTEXT._serialized_end=101 + _globals['_SOURCECONTEXT']._serialized_start=57 + _globals['_SOURCECONTEXT']._serialized_end=101 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/struct_pb2.py b/ext/protobuf/Python/google/protobuf/struct_pb2.py index 981de9614..84ad3bbde 100644 --- a/ext/protobuf/Python/google/protobuf/struct_pb2.py +++ b/ext/protobuf/Python/google/protobuf/struct_pb2.py @@ -15,22 +15,23 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cgoogle/protobuf/struct.proto\x12\x0fgoogle.protobuf\"\x98\x01\n\x06Struct\x12;\n\x06\x66ields\x18\x01 \x03(\x0b\x32#.google.protobuf.Struct.FieldsEntryR\x06\x66ields\x1aQ\n\x0b\x46ieldsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.ValueR\x05value:\x02\x38\x01\"\xb2\x02\n\x05Value\x12;\n\nnull_value\x18\x01 \x01(\x0e\x32\x1a.google.protobuf.NullValueH\x00R\tnullValue\x12#\n\x0cnumber_value\x18\x02 \x01(\x01H\x00R\x0bnumberValue\x12#\n\x0cstring_value\x18\x03 \x01(\tH\x00R\x0bstringValue\x12\x1f\n\nbool_value\x18\x04 \x01(\x08H\x00R\tboolValue\x12<\n\x0cstruct_value\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\x0bstructValue\x12;\n\nlist_value\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.ListValueH\x00R\tlistValueB\x06\n\x04kind\";\n\tListValue\x12.\n\x06values\x18\x01 \x03(\x0b\x32\x16.google.protobuf.ValueR\x06values*\x1b\n\tNullValue\x12\x0e\n\nNULL_VALUE\x10\x00\x42\x7f\n\x13\x63om.google.protobufB\x0bStructProtoP\x01Z/google.golang.org/protobuf/types/known/structpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.struct_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.struct_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\013StructProtoP\001Z/google.golang.org/protobuf/types/known/structpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' _STRUCT_FIELDSENTRY._options = None _STRUCT_FIELDSENTRY._serialized_options = b'8\001' - _NULLVALUE._serialized_start=574 - _NULLVALUE._serialized_end=601 - _STRUCT._serialized_start=50 - _STRUCT._serialized_end=202 - _STRUCT_FIELDSENTRY._serialized_start=121 - _STRUCT_FIELDSENTRY._serialized_end=202 - _VALUE._serialized_start=205 - _VALUE._serialized_end=511 - _LISTVALUE._serialized_start=513 - _LISTVALUE._serialized_end=572 + _globals['_NULLVALUE']._serialized_start=574 + _globals['_NULLVALUE']._serialized_end=601 + _globals['_STRUCT']._serialized_start=50 + _globals['_STRUCT']._serialized_end=202 + _globals['_STRUCT_FIELDSENTRY']._serialized_start=121 + _globals['_STRUCT_FIELDSENTRY']._serialized_end=202 + _globals['_VALUE']._serialized_start=205 + _globals['_VALUE']._serialized_end=511 + _globals['_LISTVALUE']._serialized_start=513 + _globals['_LISTVALUE']._serialized_end=572 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/symbol_database.py b/ext/protobuf/Python/google/protobuf/symbol_database.py index fdcf8cf06..390c49810 100644 --- a/ext/protobuf/Python/google/protobuf/symbol_database.py +++ b/ext/protobuf/Python/google/protobuf/symbol_database.py @@ -57,15 +57,41 @@ my_message_instance = db.GetSymbol('MyMessage')() """ +import warnings from google.protobuf.internal import api_implementation from google.protobuf import descriptor_pool from google.protobuf import message_factory -class SymbolDatabase(message_factory.MessageFactory): +class SymbolDatabase(): """A database of Python generated symbols.""" + # local cache of registered classes. + _classes = {} + + def __init__(self, pool=None): + """Initializes a new SymbolDatabase.""" + self.pool = pool or descriptor_pool.DescriptorPool() + + def GetPrototype(self, descriptor): + warnings.warn('SymbolDatabase.GetPrototype() is deprecated. Please ' + 'use message_factory.GetMessageClass() instead. ' + 'SymbolDatabase.GetPrototype() will be removed soon.') + return message_factory.GetMessageClass(descriptor) + + def CreatePrototype(self, descriptor): + warnings.warn('Directly call CreatePrototype() is wrong. Please use ' + 'message_factory.GetMessageClass() instead. ' + 'SymbolDatabase.CreatePrototype() will be removed soon.') + return message_factory._InternalCreateMessageClass(descriptor) + + def GetMessages(self, files): + warnings.warn('SymbolDatabase.GetMessages() is deprecated. Please use ' + 'message_factory.GetMessageClassedForFiles() instead. ' + 'SymbolDatabase.GetMessages() will be removed soon.') + return message_factory.GetMessageClassedForFiles(files, self.pool) + def RegisterMessage(self, message): """Registers the given message type in the local database. diff --git a/ext/protobuf/Python/google/protobuf/text_encoding.py b/ext/protobuf/Python/google/protobuf/text_encoding.py index 759cf11f6..1955b6a3c 100644 --- a/ext/protobuf/Python/google/protobuf/text_encoding.py +++ b/ext/protobuf/Python/google/protobuf/text_encoding.py @@ -53,8 +53,7 @@ del byte, string -def CEscape(text, as_utf8): - # type: (...) -> str +def CEscape(text, as_utf8) -> str: """Escape a bytes string for use in an text protocol buffer. Args: @@ -83,8 +82,7 @@ def CEscape(text, as_utf8): _CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])') -def CUnescape(text): - # type: (str) -> bytes +def CUnescape(text: str) -> bytes: """Unescape a text string with C-style escape sequences to UTF-8 bytes. Args: diff --git a/ext/protobuf/Python/google/protobuf/text_format.py b/ext/protobuf/Python/google/protobuf/text_format.py index a6d8bcf64..e1a5ad544 100644 --- a/ext/protobuf/Python/google/protobuf/text_format.py +++ b/ext/protobuf/Python/google/protobuf/text_format.py @@ -67,6 +67,7 @@ _FLOAT_NAN = re.compile('nanf?$', re.IGNORECASE) _QUOTES = frozenset(("'", '"')) _ANY_FULL_TYPE_NAME = 'google.protobuf.Any' +_DEBUG_STRING_SILENT_MARKER = '\t ' class Error(Exception): @@ -125,8 +126,7 @@ def MessageToString( indent=0, message_formatter=None, print_unknown_fields=False, - force_colon=False): - # type: (...) -> str + force_colon=False) -> str: """Convert protobuf message to text format. Double values can be formatted compactly with 15 digits of @@ -191,8 +191,7 @@ def MessageToString( return result -def MessageToBytes(message, **kwargs): - # type: (...) -> bytes +def MessageToBytes(message, **kwargs) -> bytes: """Convert protobuf message to encoded text format. See MessageToString.""" text = MessageToString(message, **kwargs) if isinstance(text, bytes): @@ -331,17 +330,16 @@ def _BuildMessageFromTypeName(type_name, descriptor_pool): if descriptor_pool is None: from google.protobuf import descriptor_pool as pool_mod descriptor_pool = pool_mod.Default() - from google.protobuf import symbol_database - database = symbol_database.Default() + from google.protobuf import message_factory try: message_descriptor = descriptor_pool.FindMessageTypeByName(type_name) except KeyError: return None - message_type = database.GetPrototype(message_descriptor) + message_type = message_factory.GetMessageClass(message_descriptor) return message_type() -# These values must match WireType enum in google/protobuf/wire_format.h. +# These values must match WireType enum in //google/protobuf/wire_format.h. WIRETYPE_LENGTH_DELIMITED = 2 WIRETYPE_START_GROUP = 3 @@ -558,7 +556,7 @@ def _PrintFieldName(self, field): # For groups, use the capitalized name. out.write(field.message_type.name) else: - out.write(field.name) + out.write(field.name) if (self.force_colon or field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE): @@ -856,10 +854,15 @@ def _ParseOrMerge(self, lines, message): ParseError: On text parsing problems. """ # Tokenize expects native str lines. - str_lines = ( - line if isinstance(line, str) else line.decode('utf-8') - for line in lines) - tokenizer = Tokenizer(str_lines) + try: + str_lines = ( + line if isinstance(line, str) else line.decode('utf-8') + for line in lines) + tokenizer = Tokenizer(str_lines) + except UnicodeDecodeError as e: + raise ParseError from e + if message: + self.root_type = message.DESCRIPTOR.full_name while not tokenizer.AtEnd(): self._MergeField(tokenizer, message) @@ -879,6 +882,8 @@ def _MergeField(self, tokenizer, message): type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) tokenizer.Consume(']') tokenizer.TryConsume(':') + self._DetectSilentMarker(tokenizer, message_descriptor.full_name, + type_url_prefix + '/' + packed_type_name) if tokenizer.TryConsume('<'): expanded_any_end_token = '>' else: @@ -917,8 +922,6 @@ def _MergeField(self, tokenizer, message): # pylint: disable=protected-access field = message.Extensions._FindExtensionByName(name) # pylint: enable=protected-access - - if not field: if self.allow_unknown_extension: field = None @@ -978,9 +981,13 @@ def _MergeField(self, tokenizer, message): if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: tokenizer.TryConsume(':') + self._DetectSilentMarker(tokenizer, message_descriptor.full_name, + field.full_name) merger = self._MergeMessageField else: tokenizer.Consume(':') + self._DetectSilentMarker(tokenizer, message_descriptor.full_name, + field.full_name) merger = self._MergeScalarField if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and @@ -998,13 +1005,19 @@ def _MergeField(self, tokenizer, message): else: # Proto field is unknown. assert (self.allow_unknown_extension or self.allow_unknown_field) - _SkipFieldContents(tokenizer) + self._SkipFieldContents(tokenizer, name, message_descriptor.full_name) # For historical reasons, fields may optionally be separated by commas or # semicolons. if not tokenizer.TryConsume(','): tokenizer.TryConsume(';') + def _LogSilentMarker(self, immediate_message_type, field_name): + pass + + def _DetectSilentMarker(self, tokenizer, immediate_message_type, field_name): + if tokenizer.contains_silent_marker_before_current_token: + self._LogSilentMarker(immediate_message_type, field_name) def _ConsumeAnyTypeUrl(self, tokenizer): """Consumes a google.protobuf.Any type URL and returns the type name.""" @@ -1079,12 +1092,6 @@ def _MergeMessageField(self, tokenizer, message, field): else: getattr(message, field.name)[sub_message.key] = sub_message.value - @staticmethod - def _IsProto3Syntax(message): - message_descriptor = message.DESCRIPTOR - return (hasattr(message_descriptor, 'syntax') and - message_descriptor.syntax == 'proto3') - def _MergeScalarField(self, tokenizer, message, field): """Merges a single scalar field into a message. @@ -1136,7 +1143,7 @@ def _MergeScalarField(self, tokenizer, message, field): else: if field.is_extension: if (not self._allow_multiple_scalars and - not self._IsProto3Syntax(message) and + field.has_presence and message.HasExtension(field)): raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" should not have multiple "%s" extensions.' % @@ -1146,12 +1153,12 @@ def _MergeScalarField(self, tokenizer, message, field): else: duplicate_error = False if not self._allow_multiple_scalars: - if self._IsProto3Syntax(message): - # Proto3 doesn't represent presence so we try best effort to check - # multiple scalars by compare to default values. - duplicate_error = bool(getattr(message, field.name)) - else: + if field.has_presence: duplicate_error = message.HasField(field.name) + else: + # For field that doesn't represent presence, try best effort to + # check multiple scalars by compare to default values. + duplicate_error = bool(getattr(message, field.name)) if duplicate_error: raise tokenizer.ParseErrorPreviousToken( @@ -1160,105 +1167,117 @@ def _MergeScalarField(self, tokenizer, message, field): else: setattr(message, field.name, value) + def _SkipFieldContents(self, tokenizer, field_name, immediate_message_type): + """Skips over contents (value or message) of a field. -def _SkipFieldContents(tokenizer): - """Skips over contents (value or message) of a field. - - Args: - tokenizer: A tokenizer to parse the field name and values. - """ - # Try to guess the type of this field. - # If this field is not a message, there should be a ":" between the - # field name and the field value and also the field value should not - # start with "{" or "<" which indicates the beginning of a message body. - # If there is no ":" or there is a "{" or "<" after ":", this field has - # to be a message or the input is ill-formed. - if tokenizer.TryConsume( - ':') and not tokenizer.LookingAt('{') and not tokenizer.LookingAt('<'): - if tokenizer.LookingAt('['): - _SkipRepeatedFieldValue(tokenizer) + Args: + tokenizer: A tokenizer to parse the field name and values. + field_name: The field name currently being parsed. + immediate_message_type: The type of the message immediately containing + the silent marker. + """ + # Try to guess the type of this field. + # If this field is not a message, there should be a ":" between the + # field name and the field value and also the field value should not + # start with "{" or "<" which indicates the beginning of a message body. + # If there is no ":" or there is a "{" or "<" after ":", this field has + # to be a message or the input is ill-formed. + if tokenizer.TryConsume( + ':') and not tokenizer.LookingAt('{') and not tokenizer.LookingAt('<'): + self._DetectSilentMarker(tokenizer, immediate_message_type, field_name) + if tokenizer.LookingAt('['): + self._SkipRepeatedFieldValue(tokenizer) + else: + self._SkipFieldValue(tokenizer) else: - _SkipFieldValue(tokenizer) - else: - _SkipFieldMessage(tokenizer) - - -def _SkipField(tokenizer): - """Skips over a complete field (name and value/message). - - Args: - tokenizer: A tokenizer to parse the field name and values. - """ - if tokenizer.TryConsume('['): - # Consume extension name. - tokenizer.ConsumeIdentifier() - while tokenizer.TryConsume('.'): - tokenizer.ConsumeIdentifier() - tokenizer.Consume(']') - else: - tokenizer.ConsumeIdentifierOrNumber() - - _SkipFieldContents(tokenizer) + self._DetectSilentMarker(tokenizer, immediate_message_type, field_name) + self._SkipFieldMessage(tokenizer, immediate_message_type) - # For historical reasons, fields may optionally be separated by commas or - # semicolons. - if not tokenizer.TryConsume(','): - tokenizer.TryConsume(';') + def _SkipField(self, tokenizer, immediate_message_type): + """Skips over a complete field (name and value/message). + Args: + tokenizer: A tokenizer to parse the field name and values. + immediate_message_type: The type of the message immediately containing + the silent marker. + """ + field_name = '' + if tokenizer.TryConsume('['): + # Consume extension or google.protobuf.Any type URL + field_name += '[' + tokenizer.ConsumeIdentifier() + num_identifiers = 1 + while tokenizer.TryConsume('.'): + field_name += '.' + tokenizer.ConsumeIdentifier() + num_identifiers += 1 + # This is possibly a type URL for an Any message. + if num_identifiers == 3 and tokenizer.TryConsume('/'): + field_name += '/' + tokenizer.ConsumeIdentifier() + while tokenizer.TryConsume('.'): + field_name += '.' + tokenizer.ConsumeIdentifier() + tokenizer.Consume(']') + field_name += ']' + else: + field_name += tokenizer.ConsumeIdentifierOrNumber() -def _SkipFieldMessage(tokenizer): - """Skips over a field message. - - Args: - tokenizer: A tokenizer to parse the field name and values. - """ + self._SkipFieldContents(tokenizer, field_name, immediate_message_type) - if tokenizer.TryConsume('<'): - delimiter = '>' - else: - tokenizer.Consume('{') - delimiter = '}' + # For historical reasons, fields may optionally be separated by commas or + # semicolons. + if not tokenizer.TryConsume(','): + tokenizer.TryConsume(';') - while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'): - _SkipField(tokenizer) + def _SkipFieldMessage(self, tokenizer, immediate_message_type): + """Skips over a field message. - tokenizer.Consume(delimiter) + Args: + tokenizer: A tokenizer to parse the field name and values. + immediate_message_type: The type of the message immediately containing + the silent marker + """ + if tokenizer.TryConsume('<'): + delimiter = '>' + else: + tokenizer.Consume('{') + delimiter = '}' + while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'): + self._SkipField(tokenizer, immediate_message_type) -def _SkipFieldValue(tokenizer): - """Skips over a field value. + tokenizer.Consume(delimiter) - Args: - tokenizer: A tokenizer to parse the field name and values. + def _SkipFieldValue(self, tokenizer): + """Skips over a field value. - Raises: - ParseError: In case an invalid field value is found. - """ - # String/bytes tokens can come in multiple adjacent string literals. - # If we can consume one, consume as many as we can. - if tokenizer.TryConsumeByteString(): - while tokenizer.TryConsumeByteString(): - pass - return + Args: + tokenizer: A tokenizer to parse the field name and values. - if (not tokenizer.TryConsumeIdentifier() and - not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and - not tokenizer.TryConsumeFloat()): - raise ParseError('Invalid field value: ' + tokenizer.token) + Raises: + ParseError: In case an invalid field value is found. + """ + # String/bytes tokens can come in multiple adjacent string literals. + # If we can consume one, consume as many as we can. + if tokenizer.TryConsumeByteString(): + while tokenizer.TryConsumeByteString(): + pass + return + if (not tokenizer.TryConsumeIdentifier() and + not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and + not tokenizer.TryConsumeFloat()): + raise ParseError('Invalid field value: ' + tokenizer.token) -def _SkipRepeatedFieldValue(tokenizer): - """Skips over a repeated field value. + def _SkipRepeatedFieldValue(self, tokenizer): + """Skips over a repeated field value. - Args: - tokenizer: A tokenizer to parse the field value. - """ - tokenizer.Consume('[') - if not tokenizer.LookingAt(']'): - _SkipFieldValue(tokenizer) - while tokenizer.TryConsume(','): - _SkipFieldValue(tokenizer) - tokenizer.Consume(']') + Args: + tokenizer: A tokenizer to parse the field value. + """ + tokenizer.Consume('[') + if not tokenizer.LookingAt(']'): + self._SkipFieldValue(tokenizer) + while tokenizer.TryConsume(','): + self._SkipFieldValue(tokenizer) + tokenizer.Consume(']') class Tokenizer(object): @@ -1299,6 +1318,8 @@ def __init__(self, lines, skip_comments=True): self._skip_comments = skip_comments self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT or self._WHITESPACE) + self.contains_silent_marker_before_current_token = False + self._SkipWhitespace() self.NextToken() @@ -1331,6 +1352,8 @@ def _SkipWhitespace(self): match = self._whitespace_pattern.match(self._current_line, self._column) if not match: break + self.contains_silent_marker_before_current_token = match.group(0) == ( + ' ' + _DEBUG_STRING_SILENT_MARKER) length = len(match.group(0)) self._column += length @@ -1583,6 +1606,7 @@ def NextToken(self): """Reads the next meaningful token.""" self._previous_line = self._line self._previous_column = self._column + self.contains_silent_marker_before_current_token = False self._column += len(self.token) self._SkipWhitespace() @@ -1829,12 +1853,8 @@ def ParseEnum(field, value): raise ValueError('Enum type "%s" has no value named %s.' % (enum_descriptor.full_name, value)) else: - # Numeric value. - if hasattr(field.file, 'syntax'): - # Attribute is checked for compatibility. - if field.file.syntax == 'proto3': - # Proto3 accept numeric unknown enums. - return number + if not field.enum_type.is_closed: + return number enum_value = enum_descriptor.values_by_number.get(number, None) if enum_value is None: raise ValueError('Enum type "%s" has no value with number %d.' % diff --git a/ext/protobuf/Python/google/protobuf/timestamp_pb2.py b/ext/protobuf/Python/google/protobuf/timestamp_pb2.py index 1def5d33b..81aec8776 100644 --- a/ext/protobuf/Python/google/protobuf/timestamp_pb2.py +++ b/ext/protobuf/Python/google/protobuf/timestamp_pb2.py @@ -15,12 +15,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1fgoogle/protobuf/timestamp.proto\x12\x0fgoogle.protobuf\";\n\tTimestamp\x12\x18\n\x07seconds\x18\x01 \x01(\x03R\x07seconds\x12\x14\n\x05nanos\x18\x02 \x01(\x05R\x05nanosB\x85\x01\n\x13\x63om.google.protobufB\x0eTimestampProtoP\x01Z2google.golang.org/protobuf/types/known/timestamppb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.timestamp_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.timestamp_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\016TimestampProtoP\001Z2google.golang.org/protobuf/types/known/timestamppb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _TIMESTAMP._serialized_start=52 - _TIMESTAMP._serialized_end=111 + _globals['_TIMESTAMP']._serialized_start=52 + _globals['_TIMESTAMP']._serialized_end=111 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/type_pb2.py b/ext/protobuf/Python/google/protobuf/type_pb2.py index 0764ca2a0..ee7ee569a 100644 --- a/ext/protobuf/Python/google/protobuf/type_pb2.py +++ b/ext/protobuf/Python/google/protobuf/type_pb2.py @@ -17,26 +17,27 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1agoogle/protobuf/type.proto\x12\x0fgoogle.protobuf\x1a\x19google/protobuf/any.proto\x1a$google/protobuf/source_context.proto\"\x8d\x02\n\x04Type\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12.\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x16.google.protobuf.FieldR\x06\x66ields\x12\x16\n\x06oneofs\x18\x03 \x03(\tR\x06oneofs\x12\x31\n\x07options\x18\x04 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x45\n\x0esource_context\x18\x05 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12/\n\x06syntax\x18\x06 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\"\xb4\x06\n\x05\x46ield\x12/\n\x04kind\x18\x01 \x01(\x0e\x32\x1b.google.protobuf.Field.KindR\x04kind\x12\x44\n\x0b\x63\x61rdinality\x18\x02 \x01(\x0e\x32\".google.protobuf.Field.CardinalityR\x0b\x63\x61rdinality\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x19\n\x08type_url\x18\x06 \x01(\tR\x07typeUrl\x12\x1f\n\x0boneof_index\x18\x07 \x01(\x05R\noneofIndex\x12\x16\n\x06packed\x18\x08 \x01(\x08R\x06packed\x12\x31\n\x07options\x18\t \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12#\n\rdefault_value\x18\x0b \x01(\tR\x0c\x64\x65\x66\x61ultValue\"\xc8\x02\n\x04Kind\x12\x10\n\x0cTYPE_UNKNOWN\x10\x00\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"t\n\x0b\x43\x61rdinality\x12\x17\n\x13\x43\x41RDINALITY_UNKNOWN\x10\x00\x12\x18\n\x14\x43\x41RDINALITY_OPTIONAL\x10\x01\x12\x18\n\x14\x43\x41RDINALITY_REQUIRED\x10\x02\x12\x18\n\x14\x43\x41RDINALITY_REPEATED\x10\x03\"\xff\x01\n\x04\x45num\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x38\n\tenumvalue\x18\x02 \x03(\x0b\x32\x1a.google.protobuf.EnumValueR\tenumvalue\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x45\n\x0esource_context\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12/\n\x06syntax\x18\x05 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\"j\n\tEnumValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\"H\n\x06Option\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.google.protobuf.AnyR\x05value*.\n\x06Syntax\x12\x11\n\rSYNTAX_PROTO2\x10\x00\x12\x11\n\rSYNTAX_PROTO3\x10\x01\x42{\n\x13\x63om.google.protobufB\tTypeProtoP\x01Z-google.golang.org/protobuf/types/known/typepb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.type_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.type_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\tTypeProtoP\001Z-google.golang.org/protobuf/types/known/typepb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _SYNTAX._serialized_start=1647 - _SYNTAX._serialized_end=1693 - _TYPE._serialized_start=113 - _TYPE._serialized_end=382 - _FIELD._serialized_start=385 - _FIELD._serialized_end=1205 - _FIELD_KIND._serialized_start=759 - _FIELD_KIND._serialized_end=1087 - _FIELD_CARDINALITY._serialized_start=1089 - _FIELD_CARDINALITY._serialized_end=1205 - _ENUM._serialized_start=1208 - _ENUM._serialized_end=1463 - _ENUMVALUE._serialized_start=1465 - _ENUMVALUE._serialized_end=1571 - _OPTION._serialized_start=1573 - _OPTION._serialized_end=1645 + _globals['_SYNTAX']._serialized_start=1647 + _globals['_SYNTAX']._serialized_end=1693 + _globals['_TYPE']._serialized_start=113 + _globals['_TYPE']._serialized_end=382 + _globals['_FIELD']._serialized_start=385 + _globals['_FIELD']._serialized_end=1205 + _globals['_FIELD_KIND']._serialized_start=759 + _globals['_FIELD_KIND']._serialized_end=1087 + _globals['_FIELD_CARDINALITY']._serialized_start=1089 + _globals['_FIELD_CARDINALITY']._serialized_end=1205 + _globals['_ENUM']._serialized_start=1208 + _globals['_ENUM']._serialized_end=1463 + _globals['_ENUMVALUE']._serialized_start=1465 + _globals['_ENUMVALUE']._serialized_end=1571 + _globals['_OPTION']._serialized_start=1573 + _globals['_OPTION']._serialized_end=1645 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/google/protobuf/wrappers_pb2.py b/ext/protobuf/Python/google/protobuf/wrappers_pb2.py index 6f850dc79..b11eddf27 100644 --- a/ext/protobuf/Python/google/protobuf/wrappers_pb2.py +++ b/ext/protobuf/Python/google/protobuf/wrappers_pb2.py @@ -15,28 +15,29 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1egoogle/protobuf/wrappers.proto\x12\x0fgoogle.protobuf\"#\n\x0b\x44oubleValue\x12\x14\n\x05value\x18\x01 \x01(\x01R\x05value\"\"\n\nFloatValue\x12\x14\n\x05value\x18\x01 \x01(\x02R\x05value\"\"\n\nInt64Value\x12\x14\n\x05value\x18\x01 \x01(\x03R\x05value\"#\n\x0bUInt64Value\x12\x14\n\x05value\x18\x01 \x01(\x04R\x05value\"\"\n\nInt32Value\x12\x14\n\x05value\x18\x01 \x01(\x05R\x05value\"#\n\x0bUInt32Value\x12\x14\n\x05value\x18\x01 \x01(\rR\x05value\"!\n\tBoolValue\x12\x14\n\x05value\x18\x01 \x01(\x08R\x05value\"#\n\x0bStringValue\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\"\"\n\nBytesValue\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05valueB\x83\x01\n\x13\x63om.google.protobufB\rWrappersProtoP\x01Z1google.golang.org/protobuf/types/known/wrapperspb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.wrappers_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.wrappers_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\023com.google.protobufB\rWrappersProtoP\001Z1google.golang.org/protobuf/types/known/wrapperspb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' - _DOUBLEVALUE._serialized_start=51 - _DOUBLEVALUE._serialized_end=86 - _FLOATVALUE._serialized_start=88 - _FLOATVALUE._serialized_end=122 - _INT64VALUE._serialized_start=124 - _INT64VALUE._serialized_end=158 - _UINT64VALUE._serialized_start=160 - _UINT64VALUE._serialized_end=195 - _INT32VALUE._serialized_start=197 - _INT32VALUE._serialized_end=231 - _UINT32VALUE._serialized_start=233 - _UINT32VALUE._serialized_end=268 - _BOOLVALUE._serialized_start=270 - _BOOLVALUE._serialized_end=303 - _STRINGVALUE._serialized_start=305 - _STRINGVALUE._serialized_end=340 - _BYTESVALUE._serialized_start=342 - _BYTESVALUE._serialized_end=376 + _globals['_DOUBLEVALUE']._serialized_start=51 + _globals['_DOUBLEVALUE']._serialized_end=86 + _globals['_FLOATVALUE']._serialized_start=88 + _globals['_FLOATVALUE']._serialized_end=122 + _globals['_INT64VALUE']._serialized_start=124 + _globals['_INT64VALUE']._serialized_end=158 + _globals['_UINT64VALUE']._serialized_start=160 + _globals['_UINT64VALUE']._serialized_end=195 + _globals['_INT32VALUE']._serialized_start=197 + _globals['_INT32VALUE']._serialized_end=231 + _globals['_UINT32VALUE']._serialized_start=233 + _globals['_UINT32VALUE']._serialized_end=268 + _globals['_BOOLVALUE']._serialized_start=270 + _globals['_BOOLVALUE']._serialized_end=303 + _globals['_STRINGVALUE']._serialized_start=305 + _globals['_STRINGVALUE']._serialized_end=340 + _globals['_BYTESVALUE']._serialized_start=342 + _globals['_BYTESVALUE']._serialized_end=376 # @@protoc_insertion_point(module_scope) diff --git a/ext/protobuf/Python/six.py b/ext/protobuf/Python/six.py index 83f69783d..4e15675d8 100644 --- a/ext/protobuf/Python/six.py +++ b/ext/protobuf/Python/six.py @@ -29,7 +29,7 @@ import types __author__ = "Benjamin Peterson " -__version__ = "1.15.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ def __len__(self): MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -186,6 +191,11 @@ def find_module(self, fullname, path=None): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -223,6 +233,12 @@ def get_code(self, fullname): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) diff --git a/src/Base/Core/MemoryCache.cs b/src/Base/Core/MemoryCache.cs index 8c111b40e..a5d40387e 100644 --- a/src/Base/Core/MemoryCache.cs +++ b/src/Base/Core/MemoryCache.cs @@ -1,15 +1,92 @@ using Microsoft.Extensions.Caching.Memory; using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; namespace Fusee.Base.Core { + /// + /// Extensions for Microsoft.Extensions.Caching.Memory MemoryCache + /// Source: https://stackoverflow.com/questions/45597057/how-to-retrieve-a-list-of-memory-cache-keys-in-asp-net-core + /// + public static class MemoryCacheExtensions + { + #region Microsoft.Extensions.Caching.Memory_6_OR_OLDER + + private static readonly Lazy> GetEntries6 = + new Lazy>(() => (Func)Delegate.CreateDelegate( + typeof(Func), + typeof(MemoryCache).GetProperty("EntriesCollection", BindingFlags.NonPublic | BindingFlags.Instance).GetGetMethod(true), + throwOnBindFailure: true)); + + #endregion + + #region Microsoft.Extensions.Caching.Memory_7_OR_NEWER + + private static readonly Lazy> GetCoherentState = + new Lazy>(() => + CreateGetter(typeof(MemoryCache) + .GetField("_coherentState", BindingFlags.NonPublic | BindingFlags.Instance))); + + private static readonly Lazy> GetEntries7 = + new Lazy>(() => + CreateGetter(typeof(MemoryCache) + .GetNestedType("CoherentState", BindingFlags.NonPublic) + .GetField("_entries", BindingFlags.NonPublic | BindingFlags.Instance))); + + private static Func CreateGetter(FieldInfo field) + { + var methodName = $"{field.ReflectedType.FullName}.get_{field.Name}"; + var method = new DynamicMethod(methodName, typeof(TReturn), new[] { typeof(TParam) }, typeof(TParam), true); + var ilGen = method.GetILGenerator(); + ilGen.Emit(OpCodes.Ldarg_0); + ilGen.Emit(OpCodes.Ldfld, field); + ilGen.Emit(OpCodes.Ret); + return (Func)method.CreateDelegate(typeof(Func)); + } + + #endregion + + private static readonly Func GetEntries = + Assembly.GetAssembly(typeof(MemoryCache)).GetName().Version.Major < 7 + ? (cache => (IDictionary)GetEntries6.Value(cache)) + : cache => GetEntries7.Value(GetCoherentState.Value(cache)); + + /// + /// Returns all currently cached keys as . + /// + /// The source cache. + public static ICollection GetKeys(this IMemoryCache memoryCache) => + GetEntries((MemoryCache)memoryCache).Keys; + + /// + /// Returns all currently cached keys as . + /// + /// The source cache. + public static IEnumerable GetKeys(this IMemoryCache memoryCache) => + memoryCache.GetKeys().OfType(); + } + /// /// Generic implementation of . /// The type of the key. /// The type of the cached item. /// - public class MemoryCache + public class MemoryCache : IDisposable { + /// + /// Snapshot of the caches keys as . + /// + public IEnumerable GetKeys => _cache.GetKeys(); + + /// + /// Gets the number of items in the cache for diagnostic purposes. + /// + public int Count => _cache.Count; + /// /// Sets how long a cache entry can be inactive (not accessed) before it will be removed. /// @@ -27,8 +104,10 @@ public class MemoryCache private readonly MemoryCache _cache; + private bool _disposed = false; + /// - /// Creates a new instance and initializes the internal with the gi + /// Creates a new instance and initializes the internal . /// public MemoryCache() { @@ -52,11 +131,11 @@ public bool TryGetValue(TKey key, out TItem item) } /// - /// Adds the given item to the cache. Will not check if the item is already in the cache! + /// Adds the given item to the cache or updates it if the item is already present. /// /// The key of the cache item. /// The cache item. - public void Add(TKey key, TItem cacheEntry) + public void AddOrUpdate(TKey key, TItem cacheEntry) { var cacheEntryOptions = new MemoryCacheEntryOptions() .SetPriority(CacheItemPriority.High) @@ -73,27 +152,64 @@ public void Add(TKey key, TItem cacheEntry) } /// - /// If the item isn't in the cache, add it, otherwise override the value in the cache. + /// Removes the object associated with the given key. /// - /// The key of the cache item. - /// The cache item. - public void AddOrUpdate(TKey key, TItem cacheEntry) + /// The key. + public void Remove(TKey key) { - if (!_cache.TryGetValue(key, out cacheEntry)) - { - var cacheEntryOptions = new MemoryCacheEntryOptions() - .SetPriority(CacheItemPriority.High) - // Keep in cache for this time, reset time if accessed. - .SetSlidingExpiration(TimeSpan.FromSeconds(SlidingExpiration)); + _cache.Remove(key); + } + + /// + /// Implement IDisposable. + /// Do not make this method virtual. + /// A derived class should not be able to override this method. + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } - cacheEntryOptions.RegisterPostEvictionCallback((subkey, subValue, reason, state) => + /// + /// Dispose(bool disposing) executes in two distinct scenarios. + /// If disposing equals true, the method has been called directly + /// or indirectly by a user's code. Managed and unmanaged resources + /// can be disposed. + /// If disposing equals false, the method has been called by the + /// runtime from inside the finalizer and you should not reference + /// other objects. Only unmanaged resources can be disposed. + /// + protected virtual void Dispose(bool disposing) + { + // Check to see if Dispose has already been called. + if (!_disposed) + { + // If disposing equals true, dispose all managed + // and unmanaged resources. + if (disposing) { - HandleEvictedItem?.Invoke(subkey, subValue, reason, state); - }); + // Dispose managed resources. + } - // Key not in cache, so get data. - _cache.Set(key, cacheEntry, cacheEntryOptions); + _cache.Compact(100); + _cache.Dispose(); + + // Note disposing has been done. + _disposed = true; } } + + /// + /// Use C# finalizer syntax for finalization code. + /// This finalizer will run only if the Dispose method + /// does not get called. + /// It gives your base class the opportunity to finalize. + /// Do not provide finalizer in types derived from this class. + /// + ~MemoryCache() + { + Dispose(disposing: false); + } } } \ No newline at end of file diff --git a/src/Base/Imp/Blazor/Fusee.Base.Imp.Blazor.csproj b/src/Base/Imp/Blazor/Fusee.Base.Imp.Blazor.csproj index f7c9d818b..cf5e59553 100644 --- a/src/Base/Imp/Blazor/Fusee.Base.Imp.Blazor.csproj +++ b/src/Base/Imp/Blazor/Fusee.Base.Imp.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 @@ -16,6 +16,6 @@ analyzers - + diff --git a/src/Base/Imp/Desktop/EmbeddedResourcesDllHandler.cs b/src/Base/Imp/Desktop/EmbeddedResourcesDllHandler.cs index c525e1670..0a41bfcc6 100644 --- a/src/Base/Imp/Desktop/EmbeddedResourcesDllHandler.cs +++ b/src/Base/Imp/Desktop/EmbeddedResourcesDllHandler.cs @@ -33,7 +33,7 @@ public static void LoadEmbeddedDll(string dllName, string resourceName) { // The temporary folder holds one or more of the temporary DLLs // It is made "unique" to avoid different versions of the DLL or architectures. - var tempFolder = String.Format("{0}.{1}.{2}", an.Name, an.ProcessorArchitecture, an.Version); + var tempFolder = string.Format("{0}.{1}.{2}", an.Name, RuntimeInformation.ProcessArchitecture, an.Version); string dirName = Path.Combine(Path.GetTempPath(), tempFolder); if (!Directory.Exists(dirName)) @@ -67,4 +67,4 @@ public static void LoadEmbeddedDll(string dllName, string resourceName) [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Unicode)] static extern IntPtr LoadLibrary(string lpFileName); } -} \ No newline at end of file +} diff --git a/src/Base/Imp/Desktop/FileAssetProvider.cs b/src/Base/Imp/Desktop/FileAssetProvider.cs index 503bb03c7..46fb05a32 100644 --- a/src/Base/Imp/Desktop/FileAssetProvider.cs +++ b/src/Base/Imp/Desktop/FileAssetProvider.cs @@ -1,6 +1,5 @@ using Fusee.Base.Common; using Fusee.Base.Core; -using SixLabors.ImageSharp.Drawing; using System; using System.Collections.Generic; using System.IO; diff --git a/src/Engine/Common/IRenderCanvasImp.cs b/src/Engine/Common/IRenderCanvasImp.cs index 1a32ab71e..009ea1c6c 100644 --- a/src/Engine/Common/IRenderCanvasImp.cs +++ b/src/Engine/Common/IRenderCanvasImp.cs @@ -1,4 +1,5 @@ using System; +using System.ComponentModel; namespace Fusee.Engine.Common { @@ -63,7 +64,7 @@ public interface IRenderCanvasImp /// /// Implementation Tasks: Gets and sets a value indicating whether this is in fullscreen mode. - /// This option can not be applied to all plattforms. + /// This option can not be applied to all plattforms. /// /// /// true if fullscreen; otherwise, false. @@ -166,5 +167,9 @@ public interface IRenderCanvasImp /// Occurs when [Resize] is called. /// event EventHandler Resize; + /// + /// Occurs when [Close] is called. + /// + event EventHandler Closing; } } \ No newline at end of file diff --git a/src/Engine/Common/IRenderContextImp.cs b/src/Engine/Common/IRenderContextImp.cs index bfe354d72..d6e043476 100644 --- a/src/Engine/Common/IRenderContextImp.cs +++ b/src/Engine/Common/IRenderContextImp.cs @@ -59,6 +59,19 @@ public interface IRenderContextImp /// void DisableDepthClamp(); + /// + /// Retrieve pixels from bound framebuffer + /// + /// x pixel position + /// y pixel position + /// format to retrieve, this has to match the current bound FBO! + /// how many pixel in x direction + /// how many pixel in y direction + /// with pixel content + /// Does usually not throw on error (e. g. wrong pixel format, out of bounds, etc), uses GL.GetError() to retrieve + /// potential error + public ReadOnlySpan ReadPixels(int x, int y, ImagePixelFormat pixelFormat, int width, int height); + /// /// Creates a shader object from vertex shader source code and pixel shader source code. /// @@ -890,6 +903,11 @@ public enum PrimitiveType /// /// Relates to OpenGl GL_PATCHES. /// - Patches + Patches, + + /// + /// Relates to OpenGl GL_LINES_ADJACENCY. + /// + LineAdjacency } } \ No newline at end of file diff --git a/src/Engine/Common/RCEnums.cs b/src/Engine/Common/RCEnums.cs index 5b12d0fd0..7b7514287 100644 --- a/src/Engine/Common/RCEnums.cs +++ b/src/Engine/Common/RCEnums.cs @@ -491,6 +491,8 @@ public enum CursorType : int #pragma warning disable 1591 Standard, Hand, + HResize, + VResize #pragma warning restore 1591 } diff --git a/src/Engine/Core/Assets/lineAdjacency.geom b/src/Engine/Core/Assets/lineAdjacency.geom new file mode 100644 index 000000000..f134918ab --- /dev/null +++ b/src/Engine/Core/Assets/lineAdjacency.geom @@ -0,0 +1,95 @@ +#version 460 core + +layout (lines_adjacency) in;// enables access to four vertices (line segment vertices, predecessor, successor) +layout (triangle_strip, max_vertices = 256) out; + +in vec4 vColor0[]; +out vec4 gColor; + +uniform float Thickness = 4;// just a test default +uniform ivec2 FUSEE_ViewportPx; +uniform mat4 FUSEE_MVP; +uniform bool EnableVertexColors = false; + +void main() +{ + float u_width = float(FUSEE_ViewportPx.x); + float u_height = float(FUSEE_ViewportPx.y); + float u_aspect_ratio = u_height / u_width; + vec2 Viewport = vec2(u_width, u_height); + float line_width = max(1.0, Thickness); + vec4 pos0 = gl_in[0].gl_Position; + vec4 pos1 = gl_in[1].gl_Position; + vec4 pos2 = gl_in[2].gl_Position; + vec4 pos3 = gl_in[3].gl_Position; + + //ndc + vec2 ndc0 = gl_in[0].gl_Position.xy / gl_in[0].gl_Position.w; + vec2 ndc1 = gl_in[1].gl_Position.xy / gl_in[1].gl_Position.w; + vec2 ndc2 = gl_in[2].gl_Position.xy / gl_in[2].gl_Position.w; + vec2 ndc3 = gl_in[3].gl_Position.xy / gl_in[3].gl_Position.w; + + //direction of the three segments (previous, current, next) */ + vec2 line_vector0 = ndc1 - ndc0; + vec2 line_vector1 = ndc2 - ndc1; + vec2 line_vector2 = ndc3 - ndc2; + vec2 dir0 = normalize(vec2(line_vector0.x, line_vector0.y * u_aspect_ratio)); + vec2 dir1 = normalize(vec2(line_vector1.x, line_vector1.y * u_aspect_ratio)); + vec2 dir2 = normalize(vec2(line_vector2.x, line_vector2.y * u_aspect_ratio)); + + //normals of the three segments (previous, current, next) + vec2 n0 = vec2( -dir0.y, dir0.x ); + vec2 n1 = vec2( -dir1.y, dir1.x ); + vec2 n2 = vec2( -dir2.y, dir2.x ); + + // determine miter lines by averaging the normals of the 2 segments + vec2 miter_a = normalize( n0 + n1 );// miter at start of current segment + vec2 miter_b = normalize( n1 + n2 );// miter at end of current segment + + // determine the length of the miter by projecting it onto normal and then inverse it + float an1 = dot(miter_a, n1); + float bn1 = dot(miter_b, n2); + if (an1==0) an1 = 1; + if (bn1==0) bn1 = 1; + + float length_a = line_width / an1; + if( dot(dir0, dir1 ) < -0.1/*MiterLimit*/) + { + miter_a = n1; + length_a = Thickness; + } + + float length_b = line_width / bn1; + if( dot(dir1, dir2) < -0.1/*MiterLimit*/) { + miter_b = n1; + length_b = Thickness; + } + + n0 = vec2(line_width/u_width, line_width/u_height) * n0; + n1 = vec2(line_width/u_width, line_width/u_height) * n1; + n2 = vec2(line_width/u_width, line_width/u_height) * n2; + miter_a = vec2(length_a/u_width, length_a/u_height) * miter_a; + miter_b = vec2(length_b/u_width, length_b/u_height) * miter_b; + + if(EnableVertexColors) + gColor = vColor0[0]; + gl_Position = vec4((ndc1 + miter_a) * pos1.w, pos1.zw); + EmitVertex(); + + if(EnableVertexColors) + gColor = vColor0[0]; + gl_Position = vec4((ndc1 - miter_a) * pos1.w, pos1.zw); + EmitVertex(); + + if(EnableVertexColors) + gColor = vColor0[1]; + gl_Position = vec4((ndc2 + miter_b) * pos2.w, pos2.zw); + EmitVertex(); + + if(EnableVertexColors) + gColor = vColor0[1]; + gl_Position = vec4((ndc2 - miter_b) * pos2.w, pos2.zw); + EmitVertex(); + + EndPrimitive(); +} \ No newline at end of file diff --git a/src/Engine/Core/CameraResult.cs b/src/Engine/Core/CameraResult.cs index 50fd5c5a5..47655ade2 100644 --- a/src/Engine/Core/CameraResult.cs +++ b/src/Engine/Core/CameraResult.cs @@ -4,7 +4,7 @@ namespace Fusee.Engine.Core { - internal struct CameraResult : IEquatable + public struct CameraResult : IEquatable { public Camera Camera { get; private set; } diff --git a/src/Engine/Core/FusSceneConverter.cs b/src/Engine/Core/FusSceneConverter.cs index 490b2d53a..8d6bc62a5 100644 --- a/src/Engine/Core/FusSceneConverter.cs +++ b/src/Engine/Core/FusSceneConverter.cs @@ -1,3 +1,4 @@ +using CommunityToolkit.Diagnostics; using Fusee.Base.Core; using Fusee.Engine.Common; using Fusee.Engine.Core.Effects; @@ -47,7 +48,7 @@ public static async Task ConvertFromAsync(FusFile fus, string id } // try to cast, if this fails the content is empty or null - if (!(fus.Contents is FusScene)) + if (fus.Contents is not FusScene) { Diagnostics.Error($"Could not read content of scene from {fus.Header.CreationDate} created by {fus.Header.CreatedBy} with {fus.Header.Generator}"); return new SceneContainer(); @@ -60,7 +61,12 @@ public static async Task ConvertFromAsync(FusFile fus, string id fus.Header.LoadPath = Path.GetDirectoryName(id); } - var instance = new FusFileToSceneConvertV1(); + var instance = fus.Header.FileVersion switch + { + 2 => new FusFileToSceneConvertV2(), + _ => new FusFileToSceneConvertV1(), + }; + var payload = (FusScene)fus.Contents; @@ -116,7 +122,8 @@ public static async Task ConvertFromAsync(FusFile fus, string id /// Traverses the given SceneContainer and creates new high low level graph by converting and/or splitting its components into the low level equivalents. /// /// The Scene to convert. - public static FusFile ConvertTo(SceneContainer sc) + /// The scene version, default = 2, currently there are V1 and V2 implemented + public static FusFile ConvertTo(SceneContainer sc, int fileVersion = 2) { if (sc == null) { @@ -124,7 +131,15 @@ public static FusFile ConvertTo(SceneContainer sc) return new FusFile(); } - var instance = new SceneToFusFileConvertV1(); + Guard.IsInRange(fileVersion, 1, 3); + + + var instance = fileVersion switch + { + 2 => new SceneToFusFileConvertV2(), + _ => new SceneToFusFileConvertV1() + }; + var converted = instance.Convert(sc); converted.Header = new FusHeader @@ -132,26 +147,88 @@ public static FusFile ConvertTo(SceneContainer sc) CreatedBy = sc.Header.CreatedBy, CreationDate = sc.Header.CreationDate, Generator = sc.Header.Generator, - FileVersion = 1 + FileVersion = fileVersion }; return converted; } } + internal class FusFileToSceneConvertV2 : FusFileToSceneConvertV1 + { + private readonly Dictionary _meshMap; + + internal FusFileToSceneConvertV2() + { + _meshMap = new Dictionary(); + } + + /// + /// Converts the PickComponent. + /// + /// The mesh to convert. + [VisitMethod] + public void ConvPickComp(Serialization.V2.FusPickComponent pc) + { + _currentNode.Components.Add(new PickComponent + { + Active = pc.Active, + Name = pc.Name, + PickLayer = pc.PickLayer + }); + } + + /// + /// Converts the mesh. + /// + /// The mesh to convert. + [VisitMethod] + public void ConvMesh(Serialization.V2.FusMesh m) + { + if (_currentNode.Components == null) + { + _currentNode.Components = new List(); + } + + if (_meshMap.TryGetValue(m, out var mesh)) + { + _currentNode.Components.Add(mesh); + return; + } + + // convert mesh + mesh = new Mesh(m.Triangles, m.Vertices, m.Normals, m.UVs, m.BoneWeights, m.BoneIndices, m.Tangents, m.BiTangents, + m.Colors) + { + MeshType = (PrimitiveType)m.MeshType, + Active = true, + Name = m.Name + }; + + if (_currentNode.Components == null) + { + _currentNode.Components = new List(); + } + + _currentNode.Components.Add(mesh); + + _meshMap.Add(m, mesh); + } + } + internal class FusFileToSceneConvertV1 : Visitor { - private FusScene _fusScene; - private readonly SceneContainer _convertedScene; - private readonly Stack _predecessors; - private SceneNode _currentNode; + protected FusScene _fusScene; + protected readonly SceneContainer _convertedScene; + protected readonly Stack _predecessors; + protected SceneNode _currentNode; - private readonly Dictionary _matMap; + protected readonly Dictionary _matMap; private readonly Dictionary _meshMap; - private readonly ConcurrentDictionary _texMap; - private readonly Stack _boneContainers; + protected readonly ConcurrentDictionary _texMap; + protected readonly Stack _boneContainers; - private readonly Dictionary> _allEffects; + protected readonly Dictionary> _allEffects; /// /// Method is called when going up one hierarchy level while traversing. Override this method to perform pop on any self-defined state. @@ -551,12 +628,14 @@ public void ConvMesh(FusMesh m) } // convert mesh - mesh = new Mesh(m.Triangles, m.Vertices, m.Normals, m.UVs, m.BoneWeights, m.BoneIndices, m.Tangents, m.BiTangents, - m.Colors); - - mesh.MeshType = (PrimitiveType)m.MeshType; - mesh.Active = true; - mesh.Name = m.Name; + var triangles = m.Triangles.Select(x => (uint)x).ToArray(); + mesh = new Mesh(triangles, m.Vertices, m.Normals, m.UVs, m.BoneWeights, m.BoneIndices, m.Tangents, m.BiTangents, + m.Colors) + { + MeshType = (PrimitiveType)m.MeshType, + Active = true, + Name = m.Name + }; if (_currentNode.Components == null) { @@ -954,13 +1033,48 @@ private async Task GetEffectForMat(FusMaterialBase m, ShadingModel light #endregion } + internal class SceneToFusFileConvertV2 : SceneToFusFileConvertV1 + { + internal SceneToFusFileConvertV2() + { + + } + + /// + /// Converts the mesh. + /// + /// The mesh to convert. + [VisitMethod] + public void ConvMesh(Mesh m) + { + // convert mesh + var mesh = new Serialization.V2.FusMesh + { + MeshType = (int)m.MeshType, + BiTangents = m.BiTangents?.ToArray(), + BoneIndices = m.BoneIndices?.ToArray(), + BoundingBox = m.BoundingBox, + BoneWeights = m.BoneWeights?.ToArray(), + Colors = m.Colors0?.ToArray(), + Name = m.Name, + Normals = m.Normals?.ToArray(), + Tangents = m.Tangents?.ToArray(), + Triangles = m.Triangles?.ToArray(), + UVs = m.UVs?.ToArray(), + Vertices = m.Vertices?.ToArray() + }; + + _currentNode.AddComponent(mesh); + } + } + internal class SceneToFusFileConvertV1 : Visitor { - private readonly FusFile _convertedScene; - private readonly Stack _predecessors; - private FusNode _currentNode; + protected readonly FusFile _convertedScene; + protected readonly Stack _predecessors; + protected FusNode _currentNode; - private readonly Stack _boneContainers; + protected readonly Stack _boneContainers; /// /// Method is called when going up one hierarchy level while traversing. Override this method to perform pop on any self-defined state. @@ -1255,7 +1369,7 @@ public void ConvMesh(Mesh m) Name = m.Name, Normals = m.Normals?.ToArray(), Tangents = m.Tangents?.ToArray(), - Triangles = m.Triangles.ToArray(), + Triangles = m.Triangles?.ToArray().Select(x => (ushort)x).ToArray(), UVs = m.UVs?.ToArray(), Vertices = m.Vertices?.ToArray() }; diff --git a/src/Engine/Core/Fusee.Engine.Core.csproj b/src/Engine/Core/Fusee.Engine.Core.csproj index 33b1c9a3a..5fe919f11 100644 --- a/src/Engine/Core/Fusee.Engine.Core.csproj +++ b/src/Engine/Core/Fusee.Engine.Core.csproj @@ -1,4 +1,4 @@ - + netstandard2.1;net7.0 @@ -9,7 +9,9 @@ - + + + @@ -40,15 +42,18 @@ - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + \ No newline at end of file diff --git a/src/Engine/Core/GpuMesh.cs b/src/Engine/Core/GpuMesh.cs index 533e8b30b..e2b81d7d2 100644 --- a/src/Engine/Core/GpuMesh.cs +++ b/src/Engine/Core/GpuMesh.cs @@ -1,4 +1,4 @@ -using Fusee.Engine.Common; +using Fusee.Engine.Common; using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using System; @@ -20,10 +20,11 @@ namespace Fusee.Engine.Core /// The bitangents of the mesh. /// The boneIndices of the mesh. /// The vertiboneWeightsces of the mesh. + /// The flags of the mesh. /// - public delegate GpuMesh CreateGpuMesh(PrimitiveType primitiveType, float3[] vertices, uint[] triangles = null, - float3[] normals = null, uint[] colors = null, uint[] colors1 = null, uint[] colors2 = null, float2[] uvs = null, - float4[] tangents = null, float3[] bitangents = null, float4[] boneIndices = null, float4[] boneWeights = null, uint[] flags = null); + public delegate GpuMesh CreateGpuMesh(PrimitiveType primitiveType, float3[] vertices, uint[]? triangles = null, + float3[]? normals = null, uint[]? colors = null, uint[]? colors1 = null, uint[]? colors2 = null, float2[]? uvs = null, + float4[]? tangents = null, float3[]? bitangents = null, float4[]? boneIndices = null, float4[]? boneWeights = null, uint[]? flags = null); /// /// This type of mesh doesn't create a copy of the mesh data in the RAM. @@ -38,7 +39,7 @@ public class GpuMesh : SceneComponent, IManagedMesh /// /// MeshChanged event notifies observing MeshManager the Mesh's disposal. /// - public event EventHandler DisposeData; + public event EventHandler? DisposeData; /// /// SessionUniqueIdentifier is used to verify a Mesh's uniqueness in the current session. diff --git a/src/Engine/Core/IPickerModule.cs b/src/Engine/Core/IPickerModule.cs new file mode 100644 index 000000000..692d41726 --- /dev/null +++ b/src/Engine/Core/IPickerModule.cs @@ -0,0 +1,17 @@ +using Fusee.Engine.Common; +using System.Collections.Generic; +using static Fusee.Engine.Core.ScenePicker; + +namespace Fusee.Engine.Core +{ + public interface IPickerModule : IVisitorModule + { + /// + /// Sets the for this module. Pass the state from the base renderer. + /// + /// The state to set. + public void SetState(PickerState state); + + public List PickResults { get; set; } + } +} \ No newline at end of file diff --git a/src/Engine/Core/MakeEffect.cs b/src/Engine/Core/MakeEffect.cs index f6b0bc902..9e2b210ff 100644 --- a/src/Engine/Core/MakeEffect.cs +++ b/src/Engine/Core/MakeEffect.cs @@ -55,6 +55,37 @@ public static Effect LineEffect(float lineThickness, float4 albedoColor, bool en return new ShaderEffect(uniformParameters, RenderStateSet.Default, vs, ps, gs); } + + /// + /// Generates a line shader which can be used with a with set to . /// Loads shader files via + /// For an asynchronous version use + /// + /// + /// + /// + /// + public static Effect LineEffectAdjacency(float lineThickness, float4 albedoColor, bool enableVertexColors = false) + { + var vs = AssetStorage.Get("line.vert"); + var gs = AssetStorage.Get("lineAdjacency.geom"); + var ps = AssetStorage.Get("line.frag"); + var uniformParameters = new List + { + new FxParamDeclaration + { Name = UniformNameDeclarations.ModelViewProjection, Value = float4x4.Identity }, + new FxParamDeclaration + { Name = UniformNameDeclarations.ModelView, Value = float4x4.Identity }, + new FxParamDeclaration + { Name = UniformNameDeclarations.Projection, Value = float4x4.Identity }, + new FxParamDeclaration { Name = "Thickness", Value = lineThickness }, + new FxParamDeclaration { Name = UniformNameDeclarations.ViewportPx, Value = int2.Zero }, + new FxParamDeclaration { Name = "Albedo", Value = albedoColor }, + new FxParamDeclaration { Name = "EnableVertexColors", Value = enableVertexColors } + }; + + return new ShaderEffect(uniformParameters, RenderStateSet.Default, vs, ps, gs); + } + /// /// Generates a line shader which can be used with a with set to . /// Loads shader files via diff --git a/src/Engine/Core/PrePassVisitor.cs b/src/Engine/Core/PrePassVisitor.cs index 2215a7c5e..af12afdc4 100644 --- a/src/Engine/Core/PrePassVisitor.cs +++ b/src/Engine/Core/PrePassVisitor.cs @@ -5,10 +5,21 @@ namespace Fusee.Engine.Core { - internal class PrePassVisitor : Visitor + /// + /// Visitor is inside before rendering the scene. + /// Collects s and s for rendering, picking, etc. + /// + public class PrePassVisitor : Visitor { - public List LightPrepassResuls; - public List CameraPrepassResults; + /// + /// Collection of s found while traversing the scene. + /// + public List LightPrepassResults { get; private set; } + + /// + /// Collection of s found while traversing the scene. + /// + public List CameraPrepassResults { get; private set; } /// /// Holds the status of the model matrices and other information we need while traversing up and down the scene graph. @@ -16,18 +27,26 @@ internal class PrePassVisitor : Visitor private readonly RendererState _state; private int _currentLight; + /// + /// which traverses the scene and yields and . + /// public PrePassVisitor() { _state = new RendererState(); IgnoreInactiveComponents = true; - LightPrepassResuls = new List(); + LightPrepassResults = new List(); CameraPrepassResults = new List(); } + /// + /// Call this method to initialize the traversal process. + /// + /// The to traverse. public void PrePassTraverse(SceneContainer sc) { _currentLight = 0; CameraPrepassResults.Clear(); + LightPrepassResults.Clear(); Traverse(sc.Children); } @@ -60,7 +79,7 @@ protected override void PopState() /// /// If a TransformComponent is visited the model matrix of the and is updated. /// It additionally updates the view matrix of the RenderContext. - /// + /// /// The TransformComponent. [VisitMethod] public void RenderTransform(Transform transform) @@ -71,7 +90,7 @@ public void RenderTransform(Transform transform) [VisitMethod] public void OnLight(Light lightComponent) { - if (LightPrepassResuls.Count - 1 < _currentLight) + if (LightPrepassResults.Count - 1 < _currentLight) { var lightResult = new LightResult(lightComponent) { @@ -79,11 +98,11 @@ public void OnLight(Light lightComponent) WorldSpacePos = new float3(_state.Model.M14, _state.Model.M24, _state.Model.M34) }; - LightPrepassResuls.Add(lightResult); + LightPrepassResults.Add(lightResult); } else { - var currentRes = LightPrepassResuls[_currentLight]; + var currentRes = LightPrepassResults[_currentLight]; currentRes.Rotation = _state.Model.RotationComponent(); currentRes.WorldSpacePos = new float3(_state.Model.M14, _state.Model.M24, _state.Model.M34); } diff --git a/src/Engine/Core/Primitives/Plane.cs b/src/Engine/Core/Primitives/Plane.cs index d2cb3e87f..7d8fe9b86 100644 --- a/src/Engine/Core/Primitives/Plane.cs +++ b/src/Engine/Core/Primitives/Plane.cs @@ -1,4 +1,5 @@ -using Fusee.Engine.Core.Scene; +using CommunityToolkit.Diagnostics; +using Fusee.Engine.Core.Scene; using Fusee.Math.Core; namespace Fusee.Engine.Core.Primitives @@ -47,6 +48,9 @@ public Plane() new float2(1, 0), }); + + Guard.IsNotNull(Vertices); + BoundingBox = new AABBf(Vertices.AsReadOnlySpan); #endregion } diff --git a/src/Engine/Core/Primitives/Sphere.cs b/src/Engine/Core/Primitives/Sphere.cs index 45578d0b2..22acf4ffe 100644 --- a/src/Engine/Core/Primitives/Sphere.cs +++ b/src/Engine/Core/Primitives/Sphere.cs @@ -1,4 +1,5 @@ -using Fusee.Engine.Core.Scene; +using CommunityToolkit.Diagnostics; +using Fusee.Engine.Core.Scene; using Fusee.Math.Core; namespace Fusee.Engine.Core.Primitives @@ -15,11 +16,13 @@ public class Sphere : Mesh public Sphere(int segments, int rings) { BuildSphere(segments, rings); + Guard.IsNotNull(Vertices); + BoundingBox = new AABBf(Vertices.AsReadOnlySpan); } private void BuildSphere(int segments, int rings)// segments: Longitude ||| - rings: Latitude --- { - const float radius = 1f; + const float radius = 0.5f; const double pi = System.Math.PI; const double twoPi = pi * 2f; diff --git a/src/Engine/Core/RenderCanvas.cs b/src/Engine/Core/RenderCanvas.cs index d655673b6..a261909c8 100644 --- a/src/Engine/Core/RenderCanvas.cs +++ b/src/Engine/Core/RenderCanvas.cs @@ -2,6 +2,7 @@ using Fusee.Base.Core; using Fusee.Engine.Common; using System; +using System.ComponentModel; using System.Threading.Tasks; namespace Fusee.Engine.Core @@ -72,7 +73,7 @@ public class RenderCanvas /// /// Used to inject functionality that is meant to be executed when the application is shutting down. /// - public event EventHandler? ApplicationIsShuttingDown; + public event EventHandler? ApplicationIsShuttingDown; /// /// Used to inject functionality that is meant to execute at the end of each frame. E.g. if components of the SceneGraph need to be changed. @@ -90,8 +91,7 @@ public bool IsShuttingDown private set { _isShuttingDown = value; - if (_isShuttingDown) - ApplicationIsShuttingDown?.Invoke(this, new EventArgs()); + } } private bool _isShuttingDown; @@ -191,6 +191,12 @@ public void InitApp() VideoManager.Instance.VideoManagerImp = VideoManagerImplementor; + CanvasImplementor.Closing += (s, e) => + { + ApplicationIsShuttingDown?.Invoke(this, e); + IsShuttingDown = !e.Cancel; // only set shutdown if not canceled + }; + CanvasImplementor.Init += async delegate { Init(); @@ -222,11 +228,13 @@ public void InitApp() // pre-rendering Time.Instance.DeltaTimeIncrement = CanvasImplementor.DeltaTime; + // update all meshes (changed values like position, normals, etc.) before rendering them + RC.UpdateAllMeshes(); + // rendering if (Width != 0 || Height != 0) RenderAFrame(); - RC.UpdateAllMeshes(); RC.CleanupResourceManagers(); EndOfFrame?.Invoke(this, EventArgs.Empty); diff --git a/src/Engine/Core/RenderContext.cs b/src/Engine/Core/RenderContext.cs index 3e1d3c176..f946fb411 100644 --- a/src/Engine/Core/RenderContext.cs +++ b/src/Engine/Core/RenderContext.cs @@ -1,3 +1,4 @@ +using Fusee.Base.Common; using Fusee.Base.Core; using Fusee.Engine.Common; using Fusee.Engine.Core.Effects; @@ -769,6 +770,15 @@ public float4x4 InvTransModelViewProjection /// public LightResult[] ForwardLights = new LightResult[ModuleExtensionPoint.NumberOfLightsForward]; + /// + /// Render meshes even if is . + /// If all meshes are guaranted to have valid and up-to-date values each frame. + /// Usually this is not necessary, however if there is caching and/or visibility testing on a mesh level involved + /// we want to ensure the data validity of each frame. + /// + public bool AllowDirtyMeshs { get; set; } = true; + + /// /// Initializes a new instance of the class. /// @@ -1745,6 +1755,23 @@ public void DisableDepthClamp() _rci.DisableDepthClamp(); } + + /// + /// Retrieve pixels from bound framebuffer + /// + /// x pixel position + /// y pixel position + /// format to retrieve, this has to match the current bound FBO! + /// how many pixel in x direction + /// how many pixel in y direction + /// with pixel content + /// Does usually not throw on error (e. g. wrong pixel format, out of bounds, etc), uses GL.GetError() to retrieve + /// potential error + public ReadOnlySpan ReadPixels(int x, int y, ImagePixelFormat pixelFormat, int width, int height) + { + return _rci.ReadPixels(x, y, pixelFormat, width, height); + } + /// /// Returns the hardware capabilities. /// @@ -1908,7 +1935,7 @@ public void SetRenderTarget(IWritableArrayTexture tex, int layer) /// Renders into the given texture. /// /// The render texture. - public void SetRenderTarget(IWritableTexture tex) + public void SetRenderTarget(IWritableTexture? tex) { if (tex == null) SetRenderTarget(); @@ -2003,6 +2030,14 @@ public void Render(Mesh mesh, InstanceData instanceData = null, bool doRenderFor UpdateAllActiveFxParams(cFx); var meshImp = _meshManager.GetImpFromMesh(mesh); + + // The dirty index functionality works after the initial call to the MeshManager + // This is therefore the first possible place to catch und discard (pointcloud)-meshes with a dirty index + if (!AllowDirtyMeshs && mesh != null && mesh.HasDirtyIndices) + { + return; + } + if (instanceData != null) { var instanceDataImp = _meshManager.GetImpFromInstanceData(mesh, instanceData); diff --git a/src/Engine/Core/Scene/Camera.cs b/src/Engine/Core/Scene/Camera.cs index c38febd5b..3f313dbf6 100644 --- a/src/Engine/Core/Scene/Camera.cs +++ b/src/Engine/Core/Scene/Camera.cs @@ -91,7 +91,7 @@ public class Camera : SceneComponent /// The texture given here will be used as render target. /// If this is not null the output gets rendered into the texture, otherwise to the screen. /// - public IWritableTexture RenderTexture; + public IWritableTexture? RenderTexture; /// /// Allows to overwrite the calculation of the projection matrix. @@ -102,7 +102,7 @@ public class Camera : SceneComponent /// but if this delegate is not null its out values (Projection matrix and Viewport) /// will overwrite the ones calculated from the other camera parameters. /// - public CustomCameraUpdate CustomCameraUpdate; + public CustomCameraUpdate? CustomCameraUpdate; /// /// Sets the RenderLayer for this camera. diff --git a/src/Engine/Core/Scene/ChildList.cs b/src/Engine/Core/Scene/ChildList.cs index c3dd1b0a5..4ee918059 100644 --- a/src/Engine/Core/Scene/ChildList.cs +++ b/src/Engine/Core/Scene/ChildList.cs @@ -75,7 +75,8 @@ private void AddSceneNode(SceneNode snc) else { //remove from old parent's child list - snc.Parent.Children.Remove(snc); + if (this != snc.Parent.Children) + snc.Parent.Children.Remove(snc); OnAdd?.Invoke(this, new AddChildEventArgs(snc)); } } diff --git a/src/Engine/Core/Scene/InstanceData.cs b/src/Engine/Core/Scene/InstanceData.cs index a3231c7e1..30dff6ed7 100644 --- a/src/Engine/Core/Scene/InstanceData.cs +++ b/src/Engine/Core/Scene/InstanceData.cs @@ -38,50 +38,50 @@ public float3[] Positions /// /// The rotation of each instance. This array needs to be as long as . /// - public float3[] Rotations + public float3[]? Rotations { get => _rotations; set { - if (Amount != value.Length) + if (Amount != value?.Length) throw new ArgumentOutOfRangeException(); _rotations = value; DataChanged?.Invoke(this, new InstanceDataChangedEventArgs(this, InstanceDataChangedEnum.Transform)); } } - private float3[] _rotations; + private float3[]? _rotations; /// /// The scale of each instance. This array needs to be as long as . /// - public float3[] Scales + public float3[]? Scales { get => _scales; set { - if (Amount != value.Length) + if (Amount != value?.Length) throw new ArgumentOutOfRangeException(); _scales = value; DataChanged?.Invoke(this, new InstanceDataChangedEventArgs(this, InstanceDataChangedEnum.Transform)); } } - private float3[] _scales; + private float3[]? _scales; /// /// The color of each instance. This array needs to be as long as . /// - public float4[] Colors + public float4[]? Colors { get => _colors; set { - if (Amount != value.Length) + if (Amount != value?.Length) throw new ArgumentOutOfRangeException(); _colors = value; DataChanged?.Invoke(this, new InstanceDataChangedEventArgs(this, InstanceDataChangedEnum.Colors)); } } - private float4[] _colors; + private float4[]? _colors; /// /// The amount of instances that will be rendered. @@ -102,7 +102,7 @@ public float4[] Colors /// The scale of each instance. /// The color of each instance. /// - public InstanceData(int amount, float3[] positions, float3[] rotations = null, float3[] scales = null, float4[] colors = null) + public InstanceData(int amount, float3[] positions, float3[]? rotations = null, float3[]? scales = null, float4[]? colors = null) { Amount = amount; if (Amount != positions.Length) diff --git a/src/Engine/Core/Scene/Mesh.cs b/src/Engine/Core/Scene/Mesh.cs index 757d98c42..a984faa7f 100644 --- a/src/Engine/Core/Scene/Mesh.cs +++ b/src/Engine/Core/Scene/Mesh.cs @@ -175,15 +175,55 @@ public class Mesh : SceneComponent, IManagedMesh public Suid SessionUniqueIdentifier { get; } = Suid.GenerateSuid(); /// - /// Update all changed data after each frame? + /// Update all changed data before each frame? /// public bool UpdatePerFrame { set; get; } = true; + /// + /// Gather all values in one property. + /// + /// if any of the is true. + public bool HasDirtyIndices + { +#pragma warning disable CS8602 // Dereference of a possibly null reference. + get + { + var returnVal = Vertices.DirtyIndex && Triangles.DirtyIndex; + + if (NormalsSet) + returnVal &= Normals.DirtyIndex; + if (UVsSet) + returnVal &= UVs.DirtyIndex; + if (TangentsSet) + returnVal &= Tangents.DirtyIndex; + if (BiTangentsSet) + returnVal &= BiTangents.DirtyIndex; + if (BoneIndicesSet) + returnVal &= BoneIndices.DirtyIndex; + if (BoneWeightsSet) + returnVal &= BoneWeights.DirtyIndex; + if (Colors0Set) + returnVal &= Colors0.DirtyIndex; + if (Colors1Set) + returnVal &= Colors1.DirtyIndex; + if (Colors2Set) + returnVal &= Colors2.DirtyIndex; + if (FlagsSet) + returnVal &= Flags.DirtyIndex; + + return returnVal; + } +#pragma warning restore CS8602 // Dereference of a possibly null reference. + + } + /// /// Reset all dirty flags /// public void ResetIndexLists() { +#pragma warning disable CS8602 // Dereference of a possibly null reference. + Vertices.DirtyIndex = false; Triangles.DirtyIndex = false; @@ -205,6 +245,8 @@ public void ResetIndexLists() Colors1.DirtyIndex = false; if (Colors2Set) Colors2.DirtyIndex = false; + if (FlagsSet) + Flags.DirtyIndex = false; @@ -221,6 +263,8 @@ public void ResetIndexLists() //Colors0?.DirtyIndices.ResetList(); //Colors1?.DirtyIndices.ResetList(); //Colors2?.DirtyIndices.ResetList(); + +#pragma warning restore CS8602 // Dereference of a possibly null reference. } #endregion diff --git a/src/Engine/Core/Scene/PickComponent.cs b/src/Engine/Core/Scene/PickComponent.cs new file mode 100644 index 000000000..2746f70a0 --- /dev/null +++ b/src/Engine/Core/Scene/PickComponent.cs @@ -0,0 +1,25 @@ +using Fusee.Math.Core; +using System; + +namespace Fusee.Engine.Core.Scene +{ + /// + /// PickComponent + /// + public class PickComponent : SceneComponent + { + /// + /// Pick layer, on picking the result with the higher layer will be preferred + /// + public int PickLayer { get; set; } + + /// + /// Possibility to deposit a custom method on how to pick the following mesh(es) + /// Check visitor module for new mesh types + /// Passes current , + /// , + /// Pick position in clip space + /// + public Func? CustomPickMethod; + } +} \ No newline at end of file diff --git a/src/Engine/Core/Scene/SceneExtensions.cs b/src/Engine/Core/Scene/SceneExtensions.cs index 39cdffd74..6dd626f90 100644 --- a/src/Engine/Core/Scene/SceneExtensions.cs +++ b/src/Engine/Core/Scene/SceneExtensions.cs @@ -509,7 +509,7 @@ public static void Rotate(this Transform tc, float4x4 rotationMtx) /// Global (accumulated) rotation of the parent node. public static void RotateGlobal(this Transform tc, QuaternionF rotation, QuaternionF parentGlobalRot) { - tc.RotationQuaternion *= parentGlobalRot * rotation * QuaternionF.Invert(parentGlobalRot); + tc.RotationQuaternion *= QuaternionF.Invert(parentGlobalRot) * rotation * parentGlobalRot; } /// diff --git a/src/Engine/Core/Scene/SceneNode.cs b/src/Engine/Core/Scene/SceneNode.cs index 907ec5c34..bdca5ab70 100644 --- a/src/Engine/Core/Scene/SceneNode.cs +++ b/src/Engine/Core/Scene/SceneNode.cs @@ -39,16 +39,16 @@ public class SceneNode : Xene.INode /// /// The components this node is made of. - /// + /// public List Components; /// - /// This SceneNodeContainer's snc. + /// This SceneNodeContainer's snc. /// public SceneNode Parent; /// - /// Creates a new instance of this SceneNode class. + /// Creates a new instance of this SceneNode class. /// public SceneNode() { @@ -57,8 +57,8 @@ public SceneNode() } /// - /// Possible children. - /// + /// Possible children. + /// public ChildList Children { get => _children; diff --git a/src/Engine/Core/Scene/Transform.cs b/src/Engine/Core/Scene/Transform.cs index afcb5ce81..4407d0ec2 100644 --- a/src/Engine/Core/Scene/Transform.cs +++ b/src/Engine/Core/Scene/Transform.cs @@ -11,11 +11,11 @@ public class Transform : SceneComponent #region Fields private bool _matrixDirty = true; - private float4x4 _matrix; + private float4x4 _matrix = float4x4.Identity; - private float4x4 _translationMtx; - private float4x4 _rotationMtx; - private float4x4 _scaleMtx; + private float4x4 _translationMtx = float4x4.Identity; + private float4x4 _rotationMtx = float4x4.Identity; + private float4x4 _scaleMtx = float4x4.Identity; //cached Values private bool _translationVecDirty = true; @@ -35,7 +35,7 @@ public class Transform : SceneComponent /// /// Creates a Transform component /// - public Transform() : this(float4x4.Identity, float4x4.Identity, float4x4.Identity) + public Transform() { } diff --git a/src/Engine/Core/ScenePicker.cs b/src/Engine/Core/ScenePicker.cs index 4ae31e3ad..3f756a6b1 100644 --- a/src/Engine/Core/ScenePicker.cs +++ b/src/Engine/Core/ScenePicker.cs @@ -1,4 +1,5 @@ -using Fusee.Base.Core; +using CommunityToolkit.Diagnostics; +using Fusee.Base.Core; using Fusee.Engine.Common; using Fusee.Engine.Core.Effects; using Fusee.Engine.Core.Scene; @@ -6,6 +7,7 @@ using Fusee.Xene; using System; using System.Collections.Generic; +using System.Linq; namespace Fusee.Engine.Core { @@ -18,22 +20,7 @@ public class PickResult /// /// The scene code container. /// - public SceneNode Node; - - /// - /// The mesh. - /// - public Mesh Mesh; - - /// - /// The index of the triangle that was picked. - /// - public int Triangle; - - /// - /// The barycentric u, v coordinates within the picked triangle. - /// - public float U, V; + public SceneNode? Node; /// /// The model matrix. @@ -50,120 +37,50 @@ public class PickResult /// public float4x4 Projection; - // Convenience /// - /// Gets the triangles of the picked mesh. + /// The clip position /// - /// - /// - /// - public void GetTriangle(out float3 a, out float3 b, out float3 c) - { - a = Mesh.Vertices[(int)Mesh.Triangles[Triangle + 0]]; - b = Mesh.Vertices[(int)Mesh.Triangles[Triangle + 1]]; - c = Mesh.MeshType == PrimitiveType.Triangles ? Mesh.Vertices[(int)Mesh.Triangles[Triangle + 2]] : float3.Zero; - } + public float3 ClipPos; + } + /// + /// A possible line . + /// Contains specific information for a line geometry. + /// + public class LinePickResult : PickResult + { /// - /// Returns the center of the picked triangle. - /// - public float3 TriangleCenter - { - get - { - GetTriangle(out var a, out var b, out var c); - return (a + b + c) / 3; - } - } - /// - /// Returns the barycentric triangle coordinates. - /// - public float3 TriangleBarycentric - { - get - { - GetTriangle(out var a, out var b, out var c); - return float3.Barycentric(a, b, c, U, V); - } - } - /// - /// Gets the normals at the picked triangle. - /// - /// - /// - /// - public void GetNormals(out float3 a, out float3 b, out float3 c) - { - a = Mesh.Normals[(int)Mesh.Triangles[Triangle + 0]]; - b = Mesh.Normals[(int)Mesh.Triangles[Triangle + 1]]; - c = Mesh.MeshType == PrimitiveType.Triangles ? Mesh.Normals[(int)Mesh.Triangles[Triangle + 2]] : float3.Zero; ; - } - /// - /// Returns the normal at the center of the picked triangle. - /// - public float3 NormalCenter - { - get - { - GetNormals(out var a, out var b, out var c); - return (a + b + c) / 3; - } - } - /// - /// Returns the barycentric normal coordinates. + /// The mesh. /// - public float3 NormalBarycentric - { - get - { - GetNormals(out var a, out var b, out var c); - return float3.Barycentric(a, b, c, U, V); - } - } + public Mesh? Mesh; + } + + /// + /// A possible mesh . + /// Contains specific information for a triangle geometry. + /// + public class MeshPickResult : PickResult + { /// - /// Returns the model position. + /// The mesh. /// - public float3 ModelPos => TriangleBarycentric; + public Mesh? Mesh; /// - /// Returns the clipping position of the model. + /// The hit triangle. /// - public float3 ClipPos - { - get - { - var mat = Projection * View * Model; - return float4x4.TransformPerspective(mat, ModelPos); - } - } + public int Triangle; /// - /// Returns the world position of the model. + /// U coordinate. /// - public float3 WorldPos => float4x4.TransformPerspective(Model, ModelPos); + public float U; /// - /// Returns the camera position. + /// V coordinate. /// - public float3 CameraPos - { - get - { - var mat = View * Model; - return float4x4.TransformPerspective(mat, ModelPos); - } - } + public float V; /// - /// + /// The distance from the (mouse position) to the . /// - public float2 UV - { - get - { - float2 uva = Mesh.UVs[(int)Mesh.Triangles[Triangle]]; - float2 uvb = Mesh.UVs[(int)Mesh.Triangles[Triangle + 1]]; - float2 uvc = Mesh.MeshType == PrimitiveType.Triangles ? Mesh.UVs[(int)Mesh.Triangles[Triangle + 2]] : float2.Zero; - - return float2.Barycentric(uva, uvb, uvc, U, V); - } - } + public float DistanceFromOrigin; } /// @@ -171,12 +88,14 @@ public float2 UV /// public class ScenePicker : Viserator { - private CanvasTransform _ctc; - private RenderContext _rc; + private CanvasTransform? _ctc; private bool isCtcInitialized = false; private MinMaxRect _parentRect; + private int _canvasWidth; + private int _canvasHeight; + #region State /// /// The picker state upon scene traversal. @@ -187,6 +106,22 @@ public class PickerState : VisitorState private readonly CollapsingStateStack _model = new(); private readonly CollapsingStateStack _uiRect = new(); private readonly CollapsingStateStack _cullMode = new(); + private readonly CollapsingStateStack _shaderFX = new(); + + /// + /// The current pick position in clip coordinate space. + /// + public float2 PickPosClip { get; internal set; } + + /// + /// The current camera used for picking + /// + public CameraResult CurrentCameraResult { get; internal set; } + + /// + /// The current canvas screen size + /// + public int2 ScreenSize { get; internal set; } /// /// The registered model. @@ -224,6 +159,11 @@ public Cull CullMode set => _cullMode.Tos = value; } + /// + /// The currently bound optional which can be used for storing custom pick methods as well as a . + /// + public PickComponent? CurrentPickComp; + /// /// The default constructor for the class, which registers state stacks for model, UI rectangle, and canvas transform, as well as cull mode. /// @@ -233,31 +173,57 @@ public PickerState() RegisterState(_uiRect); RegisterState(_canvasXForm); RegisterState(_cullMode); + RegisterState(_shaderFX); } - }; + } /// - /// The current view matrix. + /// The pick position on the screen. /// - public float4x4 View { get; private set; } + public float2 PickPosClip { get; set; } - /// - /// The current projection matrix. - /// - public float4x4 Projection { get; private set; } + private float4x4 _view; + private float4x4 _invView; + private float4x4 _projection; + private float4x4 _invProj; + + private CameraResult _currentCameraResult; + + internal CameraResult CurrentCameraResult + { + get => _currentCameraResult; + private set + { + Guard.IsGreaterThan(_canvasWidth, 0); + Guard.IsGreaterThan(_canvasHeight, 0); + + _currentCameraResult = value; + _projection = _currentCameraResult.Camera == null ? float4x4.Identity : _currentCameraResult.Camera.GetProjectionMat(_canvasWidth, _canvasHeight, out _); + _view = _currentCameraResult.View; + _invView = _view.Invert(); + _invProj = _projection.Invert(); + } + } + + private readonly IEnumerable _prePassResults; #endregion + /// /// The constructor to initialize a new ScenePicker. /// /// The to pick from. - public ScenePicker(SceneContainer scene) - : base(scene.Children) + /// The collected from the functionality. + /// The 's mode. + /// Any custom s, e. g. a point cloud picker module. Has to be registered from "external" like Desktop.Core. + public ScenePicker(SceneContainer scene, IEnumerable prePassCameraResults, Cull cullMode = Cull.None, IEnumerable? customPickModule = null) + : base(scene.Children, customPickModule) { IgnoreInactiveComponents = true; - View = float4x4.Identity; - Projection = float4x4.Identity; + State.CullMode = cullMode; + _prePassResults = prePassCameraResults; + } /// @@ -268,22 +234,87 @@ protected override void InitState() base.InitState(); State.Model = float4x4.Identity; State.CanvasXForm = float4x4.Identity; - State.CullMode = _rc != null ? (Cull)_rc.GetRenderState(RenderState.CullMode) : Cull.None; } /// /// Returns a collection of objects that fall in the area of the pick position and that can be iterated over. /// - /// - /// The pick position. + /// The pick position in canvas coordinates (e.g. [1270x720]), usually .Position. + /// The width of the current canvas, gets overwrite if a is bound + /// The height of the current canvas, gets overwrite if a is bound /// - public IEnumerable Pick(RenderContext rc, float2 pickPos) + public IEnumerable? Pick(float2 pickPos, int canvasWidth, int canvasHeight) + { + _canvasWidth = canvasWidth; + _canvasHeight = canvasHeight; + + float2 pickPosClip; + if (_prePassResults.Count() == 0) + { + Diagnostics.Error("No camera from a PrePassVisitor found. Picking not possible!"); + return null; + } + + CameraResult pickCam = default; + Rectangle pickCamRect = new(); + + foreach (var camRes in _prePassResults) + { + Rectangle camRect = new() + { + Left = (int)(camRes.Camera.Viewport.x * _canvasWidth / 100), + Top = (int)(camRes.Camera.Viewport.y * _canvasHeight / 100) + }; + camRect.Right = ((int)(camRes.Camera.Viewport.z * _canvasWidth) / 100) + camRect.Left; + camRect.Bottom = ((int)(camRes.Camera.Viewport.w * _canvasHeight) / 100) + camRect.Top; + + if (!float2.PointInRectangle(new float2(camRect.Left, camRect.Top), new float2(camRect.Right, camRect.Bottom), pickPos)) + continue; + + if (pickCam == default || camRes.Camera.Layer > pickCam.Camera.Layer) + { + pickCam = camRes; + pickCamRect = camRect; + } + } + + CurrentCameraResult = pickCam; + + pickPosClip = ((pickPos - new float2(pickCamRect.Left, pickCamRect.Top)) * new float2(2.0f / pickCamRect.Width, -2.0f / pickCamRect.Height)) + new float2(-1, 1); + PickPosClip = pickPosClip; + State.PickPosClip = pickPosClip; + State.CurrentCameraResult = pickCam; + State.ScreenSize = new int2(pickCamRect.Width, pickCamRect.Height); + + SetState(); + var res = Viserate().ToList(); + res.AddRange(CheckVisitorModuleResults()); + return res; + } + + private IEnumerable CheckVisitorModuleResults() { - _rc = rc; - PickPosClip = pickPos; - View = _rc.View; - Projection = _rc.Projection; - return Viserate(); + var res = new List(); + foreach (var module in VisitorModules) + { + var m = (IPickerModule)module; + if (m.PickResults.Count != 0) + res.AddRange(m.PickResults); + } + + return res; + } + + /// + /// Wire state (call by ref) to visitor module + /// + private void SetState() + { + foreach (var module in VisitorModules) + { + var m = (IPickerModule)module; + m.SetState(State); + } } #region Visitors @@ -313,14 +344,12 @@ public void RenderCanvasTransform(CanvasTransform ctc) } else if (ctc.CanvasRenderMode == CanvasRenderMode.Screen) { - var invProj = float4x4.Invert(_rc.Projection); - var frustumCorners = new float4[4]; - frustumCorners[0] = invProj * new float4(-1, -1, -1, 1); //nbl - frustumCorners[1] = invProj * new float4(1, -1, -1, 1); //nbr - frustumCorners[2] = invProj * new float4(-1, 1, -1, 1); //ntl - frustumCorners[3] = invProj * new float4(1, 1, -1, 1); //ntr + frustumCorners[0] = _invProj * new float4(-1, -1, -1, 1); //nbl + frustumCorners[1] = _invProj * new float4(1, -1, -1, 1); //nbr + frustumCorners[2] = _invProj * new float4(-1, 1, -1, 1); //ntl + frustumCorners[3] = _invProj * new float4(1, 1, -1, 1); //ntr for (var i = 0; i < frustumCorners.Length; i++) { @@ -333,7 +362,7 @@ public void RenderCanvasTransform(CanvasTransform ctc) var height = (frustumCorners[0] - frustumCorners[2]).Length; var zNear = frustumCorners[0].z; - var canvasPos = new float3(_rc.InvView.M14, _rc.InvView.M24, _rc.InvView.M34 + zNear); + var canvasPos = new float3(_invView.M14, _invView.M24, _invView.M34 + zNear); ctc.ScreenSpaceSize = new MinMaxRect { @@ -356,7 +385,7 @@ public void RenderCanvasTransform(CanvasTransform ctc) isCtcInitialized = true; } - State.CanvasXForm *= _rc.InvModel * _rc.InvView * float4x4.CreateTranslation(0, 0, zNear + (zNear * 0.01f)); + State.CanvasXForm *= State.Model.Invert() * _invView * float4x4.CreateTranslation(0, 0, zNear + (zNear * 0.01f)); State.Model *= State.CanvasXForm; _parentRect = newRect; @@ -372,7 +401,7 @@ public void RenderCanvasTransform(CanvasTransform ctc) public void RenderRectTransform(RectTransform rtc) { MinMaxRect newRect; - if (_ctc.CanvasRenderMode == CanvasRenderMode.Screen) + if (_ctc?.CanvasRenderMode == CanvasRenderMode.Screen) { newRect = new MinMaxRect { @@ -416,17 +445,14 @@ public void RenderXForm(XForm xfc) var scaleY = State.UiRect.Size.y / _parentRect.Size.y; scale = float4x4.CreateScale(scaleX, scaleY, 1); } - else if (State.UiRect.Size == _parentRect.Size && xfc.Name.Contains("Canvas")) - { - scale = float4x4.CreateScale(State.UiRect.Size.x, State.UiRect.Size.y, 1); - } else { - scale = float4x4.CreateScale(1, 1, 1); + scale = State.UiRect.Size == _parentRect.Size && xfc.Name.Contains("Canvas") + ? float4x4.CreateScale(State.UiRect.Size.x, State.UiRect.Size.y, 1) + : float4x4.CreateScale(1, 1, 1); } State.Model *= scale; - _rc.Model = State.Model; } /// @@ -436,7 +462,7 @@ public void RenderXForm(XForm xfc) [VisitMethod] public void RenderXFormText(XFormText xfc) { - var zNear = (_rc.InvProjection * new float4(-1, -1, -1, 1)).z; + var zNear = (_invProj * new float4(-1, -1, -1, 1)).z; var scaleFactor = zNear / 100; var invScaleFactor = 1 / scaleFactor; @@ -446,7 +472,7 @@ public void RenderXFormText(XFormText xfc) float scaleX; float scaleY; - if (_ctc.CanvasRenderMode == CanvasRenderMode.Screen) + if (_ctc?.CanvasRenderMode == CanvasRenderMode.Screen) { //Undo parent scale scaleX = 1 / State.UiRect.Size.x; @@ -496,7 +522,6 @@ public void RenderXFormText(XFormText xfc) State.Model *= scale; State.Model *= translation; - _rc.Model = State.Model; } /// @@ -508,7 +533,17 @@ public void RenderXFormText(XFormText xfc) public void RenderTransform(Transform transform) { State.Model *= transform.Matrix; - _rc.Model = State.Model; + } + + /// + /// Handles custom pick component with pick layer and custom picking methods. + /// If is not active, the picking is being skipped + /// + /// + [VisitMethod] + public void HandlePickComponent(PickComponent comp) + { + State.CurrentPickComp = comp; } /// @@ -516,104 +551,288 @@ public void RenderTransform(Transform transform) /// /// The given Mesh. [VisitMethod] - public void PickMesh(Mesh mesh) + public void HandleMesh(Mesh mesh) { + if (State.CurrentPickComp?.Active == false) return; + if (!mesh.Active) return; + if (mesh == null) return; + + if (State?.CurrentPickComp != null) + { + if (State?.CurrentPickComp?.CustomPickMethod != null) + { + var res = State?.CurrentPickComp?.CustomPickMethod(mesh, CurrentNode, State.Model, _view, _projection, PickPosClip); + if (res != null) + { + YieldItem(res); + return; + } + } + } + + switch (mesh.MeshType) + { + // we should be able to pick all Triangle* with Raycast + case PrimitiveType.Triangles: + case PrimitiveType.TriangleFan: + case PrimitiveType.TriangleStrip: + if (State != null) + PickTriangleGeometry(mesh); + break; + case PrimitiveType.Lines: + PickLineGeometry(mesh); + break; + case PrimitiveType.LineAdjacency: + PickLineAdjacencyGeometry(mesh); + break; + // point cloud will be picked via PointCloudPicker Module + case PrimitiveType.Points: + Diagnostics.Warn($"Picking of points not possible! Use the PointCloudPicker module!"); + break; + default: + Diagnostics.Warn($"Picking failed, unknown primitive type {mesh.MeshType}. Use PickComponent.CustomPickMethod!"); + break; + } + } + + private void PickLineAdjacencyGeometry(Mesh mesh) + { + + var mvp = _projection * _view * State.Model; + var matOfNode = CurrentNode.GetComponent(); + if (matOfNode == null) + { + Diagnostics.Debug("No shader effect for line renderer found!"); + return; + } + var thicknessFromShader = matOfNode.GetFxParam("Thickness"); + + if (mesh.Triangles == null) return; + if (mesh.Vertices == null) return; + if (CurrentCameraResult == null) + { + Diagnostics.Warn("No camera found in SceneGraph, no picking possible!"); + return; + } + + if (CurrentCameraResult.Camera == default) + return; - var mvp = Projection * View * State.Model; + var size = CurrentCameraResult.Camera.GetViewportInPx(_canvasWidth, _canvasHeight); + var viewportHeight = size.w; + var viewportWidth = size.z; + var aspect = viewportHeight / viewportWidth; + var line_width = M.Max(1.0f, thicknessFromShader); - if (mesh.MeshType == PrimitiveType.Triangles || - mesh.MeshType == PrimitiveType.TriangleFan || - mesh.MeshType == PrimitiveType.TriangleStrip) + for (var i = 1; i < mesh.Triangles.Length - 1; i += 2) { - for (var i = 0; i < mesh.Triangles.Length; i += 3) + // recreate the mesh of the geometry shader, pt in triangle check for three vertices + // not very perfomant, however this is currently the only way to yield the resulting vertices. + // The GLSL.TransformFeedback is cluttered with problems and GLSL performance warnings + var ndc0 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i - 1]]); + var ndc1 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 0]]); + var ndc2 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 1]]); + var ndc3 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 2]]); + + //direction of the three segments (previous, current, next) */ + var line_vector0 = ndc1 - ndc0; + var line_vector1 = ndc2 - ndc1; + var line_vector2 = ndc3 - ndc2; + var dir0 = new float2(line_vector0.x, line_vector0.y * aspect).Normalize(); + var dir1 = new float2(line_vector1.x, line_vector1.y * aspect).Normalize(); + var dir2 = new float2(line_vector2.x, line_vector2.y * aspect).Normalize(); + + //normals of the three segments (previous, current, next) + var n0 = new float2(-dir0.y, dir0.x); + var n1 = new float2(-dir1.y, dir1.x); + var n2 = new float2(-dir2.y, dir2.x); + + // determine miter lines by averaging the normals of the 2 segments + var miter_a = (n0 + n1).Normalize();// miter at start of current segment + var miter_b = (n1 + n2).Normalize();// miter at end of current segment + + // determine the length of the miter by projecting it onto normal and then inverse it + float an1 = float2.Dot(miter_a, n1); + float bn1 = float2.Dot(miter_b, n2); + if (an1 == 0) an1 = 1; + if (bn1 == 0) bn1 = 1; + + float length_a = line_width / an1; + if (float2.Dot(dir0, dir1) < -0.1) { - // a, b c: current triangle's vertices in clip coordinates - var a = new float4(mesh.Vertices[(int)mesh.Triangles[i + 0]], 1); - a = float4x4.TransformPerspective(mvp, a); + miter_a = n1; + length_a = line_width; + } - var b = new float4(mesh.Vertices[(int)mesh.Triangles[i + 1]], 1); - b = float4x4.TransformPerspective(mvp, b); + float length_b = line_width / bn1; + if (float2.Dot(dir1, dir2) < -0.1) + { + miter_b = n1; + length_b = line_width; + } - var c = new float4(mesh.Vertices[(int)mesh.Triangles[i + 2]], 1); - c = float4x4.TransformPerspective(mvp, c); + miter_a = new float2(length_a / viewportWidth, length_a / viewportHeight) * miter_a; + miter_b = new float2(length_b / viewportWidth, length_b / viewportHeight) * miter_b; - // Point-in-Triangle-Test - if (float2.PointInTriangle(a.xy, b.xy, c.xy, PickPosClip, out var u, out var v)) - { - var pickPos = float3.Barycentric(a.xyz, b.xyz, c.xyz, u, v); + var vert0 = new float3(ndc1.x + miter_a.x, ndc1.y + miter_a.y, ndc1.z); + var vert1 = new float3(ndc1.x - miter_a.x, ndc1.y - miter_a.y, ndc1.z); + var vert2 = new float3(ndc2.x + miter_b.x, ndc2.y + miter_b.y, ndc2.z); + var vert3 = new float3(ndc2.x - miter_b.x, ndc2.y - miter_b.y, ndc2.z); - if (pickPos.z >= -1 && pickPos.z <= 1) - { - if (State.CullMode == Cull.None || float2.IsTriangleCW(a.xy, b.xy, c.xy) == (State.CullMode == Cull.Clockwise)) - { - YieldItem(new PickResult - { - Mesh = mesh, - Node = CurrentNode, - Triangle = i, - Model = State.Model, - View = View, - Projection = Projection, - U = u, - V = v - }); - } - } - } + if (float2.PointInTriangle(vert0.xy, vert1.xy, vert2.xy, PickPosClip, out _, out _) || + float2.PointInTriangle(vert2.xy, vert1.xy, vert3.xy, PickPosClip, out _, out _)) + { + YieldItem(new LinePickResult + { + Mesh = mesh, + Node = CurrentNode, + Model = State.Model, + ClipPos = float4x4.TransformPerspective(_projection * _view, State.Model.Translation()), + View = _view, + Projection = _projection + }); } } - else if (mesh.MeshType == PrimitiveType.Lines) + } + + private void PickLineGeometry(Mesh mesh) + { + + var mvp = _projection * _view * State.Model; + + var matOfNode = CurrentNode.GetComponent(); + if (matOfNode == null) { - var matOfNode = CurrentNode.GetComponent(); - if (matOfNode == null) + Diagnostics.Debug("No shader effect for line renderer found!"); + return; + } + var thicknessFromShader = matOfNode.GetFxParam("Thickness"); + + if (mesh.Triangles == null) return; + if (mesh.Vertices == null) return; + if (CurrentCameraResult == null) + { + Diagnostics.Warn("No camera found in SceneGraph, no picking possible!"); + return; + } + + if (CurrentCameraResult.Camera == default) + return; + + var size = CurrentCameraResult.Camera.GetViewportInPx(_canvasWidth, _canvasHeight); + var viewportHeight = size.w; + var viewportWidth = size.z; + var aspect = viewportHeight / viewportWidth; + var line_width = M.Max(1.0f, thicknessFromShader); + + for (var i = 0; i < mesh.Triangles.Length; i += 2) + { + var pt0 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 0]]).xy; + var pt1 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 1]]).xy; + + var lineVector = pt1 - pt0; + var dir = new float2(lineVector.x, lineVector.y * aspect).Normalize(); + + var normal = new float2(-dir.y, dir.x); + var normal_a = new float2(line_width / viewportWidth, line_width / viewportHeight) * normal; + var normal_b = new float2(line_width / viewportWidth, line_width / viewportHeight) * normal; + + var vert0 = pt0 + normal_a; + var vert1 = pt0 - normal_a; + var vert2 = pt1 + normal_b; + var vert3 = pt1 - normal_b; + + if (float2.PointInTriangle(vert0.xy, vert1.xy, vert2.xy, PickPosClip, out _, out _) || + float2.PointInTriangle(vert2.xy, vert1.xy, vert3.xy, PickPosClip, out _, out _)) { - Diagnostics.Debug("No shader effect for line renderer found!"); - return; + YieldItem(new LinePickResult + { + Mesh = mesh, + Node = CurrentNode, + Model = State.Model, + ClipPos = float4x4.TransformPerspective(_projection * _view, State.Model.Translation()), + View = _view, + Projection = _projection + }); } - var thicknessFromShader = matOfNode.GetFxParam("Thickness"); + } + } - for (var i = 0; i < mesh.Triangles.Length; i += 2) - { - var thickness = (thicknessFromShader / _rc.ViewportHeight); + /// + /// Pick triangle geometry via ray cast. + /// + /// + private void PickTriangleGeometry(Mesh mesh) + { + if (mesh == null) return; + if (mesh.Triangles == null) return; + if (mesh.Vertices == null) return; - var pt1 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 0]]).xy; - var pt2 = float4x4.TransformPerspective(mvp, mesh.Vertices[(int)mesh.Triangles[i + 1]]).xy; - var pt0 = PickPosClip; + if (mesh.BoundingBox == default) + { + //Diagnostics.Warn($"Current bounding box of {mesh} is default while mesh is being picked. Generating box ..."); + mesh.BoundingBox = new(mesh.Vertices.AsReadOnlySpan); + } + + if (mesh.GetType() != typeof(Primitives.Plane) && (mesh.BoundingBox.Size.x <= 0f || mesh.BoundingBox.Size.y <= 0f || mesh.BoundingBox.Size.z <= 0f)) + { + Diagnostics.Warn($"Size of current bounding box is 0 for one or more dimensions. Picking not possible."); + return; + } - // Line Eq = ax + by + c = 0 - // A = (y1 - y2) - // B = (x2 - x1) - // C = (x1 * y2 - x2 * y1) + var ray = new RayF(PickPosClip, _view, _projection); - // dist(line, pt) = |Ax + By + C| / A² + B² - var a = pt1.y - pt2.y; - var b = pt2.x - pt1.x; - var c = pt1.x * pt2.y - pt2.x * pt1.y; + var box = State.Model * mesh.BoundingBox; + if (!box.IntersectRay(ray)) + return; - var d = (MathF.Abs(a * pt0.x + b * pt0.y + c) / (a * a + b * b)); + for (int i = 0; i < mesh.Triangles.Length; i += 3) + { + // Vertices of the picked triangle in world space + var a = new float3(mesh.Vertices[(int)mesh.Triangles[i + 0]]); + a = float4x4.Transform(State.Model, a); - if (d <= thickness) + var b = new float3(mesh.Vertices[(int)mesh.Triangles[i + 1]]); + b = float4x4.Transform(State.Model, b); + + var c = new float3(mesh.Vertices[(int)mesh.Triangles[i + 2]]); + c = float4x4.Transform(State.Model, c); + + // Normal of the plane defined by a, b, and c. + var n = float3.Normalize(float3.Cross(a - c, b - c)); + + // Distance between "Origin" and the plane abc when following the Direction. + var distance = -float3.Dot(ray.Origin - a, n) / float3.Dot(ray.Direction, n); + + if (distance < 0) + continue; + + // Position of the intersection point between ray and plane. + var point = ray.Origin + (ray.Direction * distance); + + if (float3.PointInTriangle(a, b, c, point, out float u, out float v)) + { + if (State.CullMode == Cull.None || (State.CullMode == Cull.Clockwise) == (float3.Dot(n, ray.Direction) < 0)) { - YieldItem(new PickResult + YieldItem(new MeshPickResult { Mesh = mesh, Node = CurrentNode, Triangle = i, Model = State.Model, - View = View, - Projection = Projection + U = u, + V = v, + ClipPos = float4x4.TransformPerspective(_projection * _view, State.Model.Translation()), + Projection = _projection, + View = _view, + DistanceFromOrigin = distance }); } } } } - /// - /// The pick position on the screen. - /// - public float2 PickPosClip { get; set; } - #endregion } diff --git a/src/Engine/Core/SceneRayCaster.cs b/src/Engine/Core/SceneRayCaster.cs index 3d4c21d32..75daa5c59 100644 --- a/src/Engine/Core/SceneRayCaster.cs +++ b/src/Engine/Core/SceneRayCaster.cs @@ -9,83 +9,12 @@ namespace Fusee.Engine.Core /// /// This class contains information about the scene of the picked point. /// - public class RayCastResult + public class RayCastResult : PickResult { /// - /// The scene node container of the result. + /// The mesh. /// - public SceneNode Node; - - /// - /// The picked mesh. - /// - public Mesh Mesh; - - /// - /// The index of the triangle in which the intersection of ray and mesh happened. - /// - public int Triangle; - - /// - /// The barycentric u, v coordinates within the picked triangle. - /// - public float U, V; - - /// - /// The (texture-) UV coordinates of the picked point. - /// - public float2 UV - { - get - { - float2 uva = Mesh.UVs[(int)Mesh.Triangles[Triangle]]; - float2 uvb = Mesh.UVs[(int)Mesh.Triangles[Triangle + 1]]; - float2 uvc = Mesh.UVs[(int)Mesh.Triangles[Triangle + 2]]; - - return float2.Barycentric(uva, uvb, uvc, U, V); - } - } - - /// - /// The model matrix. - /// - public float4x4 Model; - - /// - /// Gets the triangles of the picked mesh. - /// - /// - /// - /// - public void GetTriangle(out float3 a, out float3 b, out float3 c) - { - a = Mesh.Vertices[(int)Mesh.Triangles[Triangle + 0]]; - b = Mesh.Vertices[(int)Mesh.Triangles[Triangle + 1]]; - c = Mesh.Vertices[(int)Mesh.Triangles[Triangle + 2]]; - } - - /// - /// Returns the barycentric triangle coordinates. - /// - public float3 TriangleBarycentric - { - get - { - GetTriangle(out var a, out var b, out var c); - return float3.Barycentric(a, b, c, U, V); - } - } - - /// - /// Returns the model position. - /// - public float3 ModelPos => TriangleBarycentric; - - /// - /// Returns the world position of the intersection. - /// - public float3 WorldPos => float4x4.TransformPerspective(Model, ModelPos); - + public Mesh? Mesh; /// /// Returns the distance between ray origin and the intersection. /// @@ -112,7 +41,7 @@ public class SceneRayCaster : Viserator protected SceneContainer _sc; - internal PrePassVisitor PrePassVisitor { get; private set; } + private readonly IEnumerable _prePassResults; #region State /// @@ -145,13 +74,13 @@ public RayCasterState() /// The constructor to initialize a new SceneRayCaster. /// /// The to use. + /// The collected from the functionality. /// The mode to use. - public SceneRayCaster(SceneContainer scene, Cull cullMode = Cull.None) + public SceneRayCaster(SceneContainer scene, IEnumerable prePassCameraResults, Cull cullMode = Cull.None) : base(scene.Children) { CullMode = cullMode; - - PrePassVisitor = new PrePassVisitor(); + _prePassResults = prePassCameraResults; _sc = scene; } @@ -178,47 +107,11 @@ public IEnumerable RayCast(RayF ray) /// /// Returns a collection of objects that are hit by the ray and that can be iterated over. /// - /// - /// + /// The which should travel through the scene /// - public IEnumerable RayPick(RenderContext rc, float2 pickPos) + public IEnumerable? Traverse(RayF ray) { - PrePassVisitor.PrePassTraverse(_sc); - var cams = PrePassVisitor.CameraPrepassResults; - - float2 pickPosClip; - - if (cams.Count == 0) - { - pickPosClip = (pickPos * new float2(2.0f / rc.ViewportWidth, -2.0f / rc.ViewportHeight)) + new float2(-1, 1); - Ray = new RayF(pickPosClip, rc.View, rc.Projection); - return Viserate(); - } - - CameraResult pickCam = default; - Rectangle pickCamRect = new(); - - foreach (var camRes in cams) - { - Rectangle camRect = new(); - camRect.Left = (int)(camRes.Camera.Viewport.x * rc.ViewportWidth / 100); - camRect.Top = (int)(camRes.Camera.Viewport.y * rc.ViewportHeight / 100); - camRect.Right = (int)(camRes.Camera.Viewport.z * rc.ViewportWidth) / 100 + camRect.Left; - camRect.Bottom = (int)(camRes.Camera.Viewport.w * rc.ViewportHeight) / 100 + camRect.Top; - - if (!float2.PointInRectangle(new float2(camRect.Left, camRect.Top), new float2(camRect.Right, camRect.Bottom), pickPos)) continue; - - if (pickCam == default || camRes.Camera.Layer > pickCam.Camera.Layer) - { - pickCam = camRes; - pickCamRect = camRect; - } - } - - // Calculate pickPosClip - pickPosClip = ((pickPos - new float2(pickCamRect.Left, pickCamRect.Top)) * new float2(2.0f / pickCamRect.Width, -2.0f / pickCamRect.Height)) + new float2(-1, 1); - Ray = new RayF(pickPosClip, pickCam.View, pickCam.Camera.GetProjectionMat(rc.ViewportWidth, rc.ViewportHeight, out _)); - + Ray = ray; return Viserate(); } @@ -241,7 +134,10 @@ public void RenderTransform(Transform transform) [VisitMethod] public void HitMesh(Mesh mesh) { + if (mesh == null) return; if (!mesh.Active) return; + if (mesh.Vertices == null) return; + if (mesh.Triangles == null) return; AABBf box = State.Model * mesh.BoundingBox; if (!box.IntersectRay(Ray)) return; @@ -278,10 +174,7 @@ public void HitMesh(Mesh mesh) { Mesh = mesh, Node = CurrentNode, - Triangle = i, Model = State.Model, - U = u, - V = v, DistanceFromOrigin = distance }); } diff --git a/src/Engine/Core/SceneRendererDeferred.cs b/src/Engine/Core/SceneRendererDeferred.cs index 965a5994b..10235285e 100644 --- a/src/Engine/Core/SceneRendererDeferred.cs +++ b/src/Engine/Core/SceneRendererDeferred.cs @@ -162,7 +162,7 @@ public void RenderShaderEffect(Effect effect) if (!RenderLayer.HasFlag(_state.RenderLayer.Layer) && !_state.RenderLayer.Layer.HasFlag(RenderLayer) || _state.RenderLayer.Layer.HasFlag(RenderLayers.None)) return; - if (DoFrumstumCulling) + if (DoFrustumCulling) { FrustumF frustum; if (_currentPass == RenderPasses.Shadow) @@ -201,7 +201,7 @@ public void RenderShaderEffect(Effect effect) if (!RenderLayer.HasFlag(_state.RenderLayer.Layer) && !_state.RenderLayer.Layer.HasFlag(RenderLayer) || _state.RenderLayer.Layer.HasFlag(RenderLayers.None)) return; - if (DoFrumstumCulling) + if (DoFrustumCulling) { FrustumF frustum; if (_currentPass == RenderPasses.Shadow) @@ -464,14 +464,14 @@ public override void Render(RenderContext rc) { PerCamClear(cam); NotifyCameraChanges(cam.Camera); - DoFrumstumCulling = cam.Camera.FrustumCullingOn; + DoFrustumCulling = cam.Camera.FrustumCullingOn; PerCamRender(cam, cam.Camera.RenderTexture); } } //Reset Viewport and frustum culling bool in case we have another scene, rendered without a camera _rc.Viewport(0, 0, rc.DefaultState.CanvasWidth, rc.DefaultState.CanvasHeight); - DoFrumstumCulling = true; + DoFrustumCulling = true; } else { @@ -479,7 +479,7 @@ public override void Render(RenderContext rc) } } - private void PerCamRender(CameraResult cam, IWritableTexture renderTex = null) + private void PerCamRender(CameraResult cam, IWritableTexture? renderTex = null) { RenderLayer = cam.Camera.RenderLayer; _rc.View = cam.View; @@ -507,7 +507,7 @@ private void PerCamRender(CameraResult cam, IWritableTexture renderTex = null) } } - private void RenderAllPasses(float4 lightingPassViewport, IWritableTexture renderTex = null) + private void RenderAllPasses(float4 lightingPassViewport, IWritableTexture? renderTex = null) { var preRenderStateSet = _rc.CurrentRenderState.Copy(); //"Snapshot" of the current render states as they came from the user code. var preRenderLockedStates = new Dictionary>(_rc.LockedStates); @@ -526,9 +526,9 @@ private void RenderAllPasses(float4 lightingPassViewport, IWritableTexture rende _rc.Viewport(0, 0, (int)ShadowMapRes, (int)ShadowMapRes); //Cache state because rendering the shadow maps will change the state eventually. - var doCulling = DoFrumstumCulling; + var doCulling = DoFrustumCulling; RenderShadowMaps(); - DoFrumstumCulling = doCulling; + DoFrustumCulling = doCulling; //Undo all user-made and shadow pass related render state changes to be able to work on a "white sheet" from here on. _rc.UnlockAllRenderStates(); @@ -570,7 +570,7 @@ private void RenderAllPasses(float4 lightingPassViewport, IWritableTexture rende /// Alternatively it would be possible to iterate the lights in the shader, but this would create a more complex shader. Additionally it would be more difficult to implement a dynamic number of lights. /// The iteration here should not prove critical, due to the scene only consisting of a single quad. /// - private void RenderLightPasses(IWritableTexture renderTex = null) + private void RenderLightPasses(IWritableTexture? renderTex = null) { _rc.SetRenderTarget(renderTex); _rc.Clear(ClearFlags.Color | ClearFlags.Depth); @@ -711,7 +711,7 @@ private void RenderShadowMaps() { case LightType.Point: { - DoFrumstumCulling = false; + DoFrustumCulling = false; if (_shadowCubeMapEffect == null) { @@ -779,7 +779,7 @@ private void RenderShadowMaps() } case LightType.Spot: { - DoFrumstumCulling = true; + DoFrustumCulling = true; _shadowEffect.SetFxParam(UniformNameDeclarations.LightSpaceMatrixHash, shadowParams.LightSpaceMatrices[0]); _rc.SetEffect(_shadowEffect); _rc.SetRenderTarget(shadowParams.ShadowMap); @@ -836,7 +836,7 @@ private void RenderSSAO() _gBufferRenderTarget.SetTexture(_blurRenderTex, RenderTargetTextureTypes.Ssao); } - private void RenderFXAA(IWritableTexture renderTex = null) + private void RenderFXAA(IWritableTexture? renderTex = null) { _currentPass = RenderPasses.Fxaa; if (_fxaaEffect == null) diff --git a/src/Engine/Core/SceneRendererForward.cs b/src/Engine/Core/SceneRendererForward.cs index 191ff2176..e956b4248 100644 --- a/src/Engine/Core/SceneRendererForward.cs +++ b/src/Engine/Core/SceneRendererForward.cs @@ -23,7 +23,7 @@ public class SceneRendererForward : Visitor /// Enables or disables Frustum Culling. /// If we render with one or more cameras this value will be overwritten by . /// - public bool DoFrumstumCulling = true; + public bool DoFrustumCulling = true; /// /// The RenderLayer this renderer should render. @@ -71,7 +71,7 @@ private set private MinMaxRect _parentRect; private int _numberOfLights; - internal PrePassVisitor PrePassVisitor { get; private set; } + public PrePassVisitor PrePassVisitor { get; private set; } /// /// Caches SceneNodes and their model matrices. Used when visiting a . @@ -330,7 +330,7 @@ public virtual void Render(RenderContext rc) { PerCamClear(cam); NotifyCameraChanges(cam.Camera); - DoFrumstumCulling = cam.Camera.FrustumCullingOn; + DoFrustumCulling = cam.Camera.FrustumCullingOn; PerCamRender(cam); } } @@ -338,7 +338,7 @@ public virtual void Render(RenderContext rc) //Reset Viewport and frustum culling bool in case we have another scene, rendered without a camera _rc.Viewport(0, 0, rc.DefaultState.CanvasWidth, rc.DefaultState.CanvasHeight); //Standard value: frustum culling is on. - DoFrumstumCulling = true; + DoFrustumCulling = true; } else { @@ -377,7 +377,7 @@ private void PerCamRender(CameraResult cam) _rc.SetRenderTarget(tex); _rc.Projection = tex != null - ? cam.Camera.GetProjectionMat(cam.Camera.RenderTexture.Width, cam.Camera.RenderTexture.Height, out float4 viewport) + ? cam.Camera.GetProjectionMat(tex.Width, tex.Height, out float4 viewport) : cam.Camera.GetProjectionMat(_rc.GetWindowWidth(), _rc.GetWindowHeight(), out viewport); _rc.Viewport((int)viewport.x, (int)viewport.y, (int)viewport.z, (int)viewport.w); @@ -399,7 +399,7 @@ private void PerCamRender(CameraResult cam) /// protected void AccumulateLight() { - LightViseratorResults = PrePassVisitor.LightPrepassResuls; + LightViseratorResults = PrePassVisitor.LightPrepassResults; if (LightViseratorResults.Count == 0) SetDefaultLight(); @@ -709,7 +709,7 @@ public void RenderMesh(Mesh mesh) if (!RenderLayer.HasFlag(_state.RenderLayer.Layer) && !_state.RenderLayer.Layer.HasFlag(RenderLayer) || _state.RenderLayer.Layer.HasFlag(RenderLayers.None)) return; - if (DoFrumstumCulling) + if (DoFrustumCulling) { //If the bounding box is zero in size, it is not initialized and we cannot perform the culling test. if (mesh.BoundingBox.Size.x > 0 && mesh.BoundingBox.Size.y > 0 && mesh.BoundingBox.Size.z > 0) @@ -739,7 +739,7 @@ public void RenderMesh(GpuMesh mesh) if (!RenderLayer.HasFlag(_state.RenderLayer.Layer) && !_state.RenderLayer.Layer.HasFlag(RenderLayer) || _state.RenderLayer.Layer.HasFlag(RenderLayers.None)) return; - if (DoFrumstumCulling) + if (DoFrustumCulling) { //If the bounding box is zero in size, it is not initialized and we cannot perform the culling test. if (mesh.BoundingBox.Size != float3.Zero) diff --git a/src/Engine/GUI/SceneInteractionHandler.cs b/src/Engine/GUI/SceneInteractionHandler.cs index 718774aa1..c6ad9599e 100644 --- a/src/Engine/GUI/SceneInteractionHandler.cs +++ b/src/Engine/GUI/SceneInteractionHandler.cs @@ -19,14 +19,16 @@ public class SceneInteractionHandler : Visitor private SceneNode _pickRes; private SceneNode _pickResCache; + /// /// Initializes a new instance of the class. /// /// The scene the interaction handler belongs to. - public SceneInteractionHandler(SceneContainer scene) + /// The of the operation. + public SceneInteractionHandler(SceneContainer scene, IEnumerable prePassCameraResults) { IgnoreInactiveComponents = true; - _scenePicker = new ScenePicker(scene); + _scenePicker = new ScenePicker(scene, prePassCameraResults); } private static SceneNode FindLeafNodeInPickRes(SceneNode firstPickRes, IList pickResults) @@ -60,12 +62,10 @@ private static SceneNode FindLeafNodeInPickRes(SceneNode firstPickRes, IListThe current mouse position. /// Canvas width - needed to determine the mouse position in clip space. /// Canvas height - needed to determine the mouse position in clip space. - public void CheckForInteractiveObjects(RenderContext rc, float2 mousePos, int canvasWidth, int canvasHeight) + public void CheckForInteractiveObjects(float2 mousePos, int canvasWidth, int canvasHeight) { - var pickPosClip = mousePos * new float2(2.0f / canvasWidth, -2.0f / canvasHeight) + new float2(-1, 1); - - var pickResults = _scenePicker.Pick(rc, pickPosClip).ToList().OrderBy(pr => pr.ClipPos.z).ToList(); - var pickResNodes = pickResults.Select(x => x.Node).ToList(); + var pickResults = _scenePicker.Pick(mousePos, canvasWidth, canvasHeight).ToList().OrderBy(pr => pr.ClipPos.z).ToList(); + var pickResNodes = pickResults.ConvertAll(x => x.Node); var firstPickRes = pickResults.FirstOrDefault(); _pickRes = null; diff --git a/src/Engine/Imp/Graphics/Android/RenderCanvasImp.cs b/src/Engine/Imp/Graphics/Android/RenderCanvasImp.cs index 06b25b941..3c1a9982a 100644 --- a/src/Engine/Imp/Graphics/Android/RenderCanvasImp.cs +++ b/src/Engine/Imp/Graphics/Android/RenderCanvasImp.cs @@ -7,6 +7,7 @@ using OpenTK.Graphics.ES30; using OpenTK.Platform.Android; using System; +using System.ComponentModel; using System.Diagnostics; using Uri = Android.Net.Uri; @@ -308,6 +309,11 @@ public void Run() /// public event EventHandler Resize; + /// + /// Occurs when [closing] - not wired, dummy impl. + /// + public event EventHandler Closing; + #endregion Events /// @@ -459,7 +465,7 @@ protected override void OnRenderFrame(FrameEventArgs e) _renderCanvasImp.DoRender(); } - // This method is called every time the context needs to be recreated. + // This method is called every time the context needs to be recreated. //Use it to set any egl-specific settings prior to context creation. protected override void CreateFrameBuffer() { diff --git a/src/Engine/Imp/Graphics/Android/RenderContextImp.cs b/src/Engine/Imp/Graphics/Android/RenderContextImp.cs index dec92429f..c639857b1 100644 --- a/src/Engine/Imp/Graphics/Android/RenderContextImp.cs +++ b/src/Engine/Imp/Graphics/Android/RenderContextImp.cs @@ -2786,6 +2786,33 @@ public void DeleteStorageBuffer(IBufferHandle storageBufferHandle) #region Picking related Members + /// + /// Retrieve pixels from bound framebuffer + /// + /// x pixel position + /// y pixel position + /// format to retrieve, this has to match the current bound FBO! + /// how many pixel in x direction + /// how many pixel in y direction + /// with pixel content + /// Does usually not throw on error (e. g. wrong pixel format, out of bounds, etc), uses GL.GetError() to retrieve + /// potential error + public ReadOnlySpan ReadPixels(int x, int y, ImagePixelFormat pixelFormat, int width, int height) + { + var format = GetTexturePixelInfo(pixelFormat); + var data = new byte[width * height * pixelFormat.BytesPerPixel]; + + GL.ReadPixels(x, y, 1, 1, format.Format, format.PxType, data); + + var err = GL.GetErrorCode(); + if (err != ErrorCode.NoError) + { + throw new Exception($"ReadPixel failed with error code {err}!"); + } + + return data; + } + /// /// Retrieves a sub-image of the given region. /// diff --git a/src/Engine/Imp/Graphics/Blazor/Fusee.Engine.Imp.Graphics.Blazor.csproj b/src/Engine/Imp/Graphics/Blazor/Fusee.Engine.Imp.Graphics.Blazor.csproj index 9d61f03f8..310c96278 100644 --- a/src/Engine/Imp/Graphics/Blazor/Fusee.Engine.Imp.Graphics.Blazor.csproj +++ b/src/Engine/Imp/Graphics/Blazor/Fusee.Engine.Imp.Graphics.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 @@ -23,7 +23,7 @@ analyzers - + diff --git a/src/Engine/Imp/Graphics/Blazor/RenderCanvasImp.cs b/src/Engine/Imp/Graphics/Blazor/RenderCanvasImp.cs index 065104bac..1473ae8c2 100644 --- a/src/Engine/Imp/Graphics/Blazor/RenderCanvasImp.cs +++ b/src/Engine/Imp/Graphics/Blazor/RenderCanvasImp.cs @@ -2,7 +2,7 @@ using Fusee.Engine.Common; using Microsoft.JSInterop; using System; - +using System.ComponentModel; namespace Fusee.Engine.Imp.Graphics.Blazor { @@ -103,6 +103,11 @@ public RenderCanvasImp(IJSObjectReference canvas, IJSRuntime runtime, WebGL2Rend /// public event EventHandler Update; + /// + /// Occurs when closing the application + /// + public event EventHandler Closing; + /// /// Closes the game window. /// @@ -110,6 +115,7 @@ public RenderCanvasImp(IJSObjectReference canvas, IJSRuntime runtime, WebGL2Rend public void CloseGameWindow() { UnLoad?.Invoke(this, null); + Closing.Invoke(this, null); } /// @@ -128,7 +134,7 @@ public void OpenLink(string link) /// public void Present() { - // Nothing to do in WebGL + // Nothing to do in WebGL } /// diff --git a/src/Engine/Imp/Graphics/Blazor/RenderContextImp.cs b/src/Engine/Imp/Graphics/Blazor/RenderContextImp.cs index 3c7bf2030..ecec100d0 100644 --- a/src/Engine/Imp/Graphics/Blazor/RenderContextImp.cs +++ b/src/Engine/Imp/Graphics/Blazor/RenderContextImp.cs @@ -2679,6 +2679,33 @@ public void DebugLine(float3 start, float3 end, float4 color) #region Picking related Members + /// + /// Retrieve pixels from bound framebuffer + /// + /// x pixel position + /// y pixel position + /// format to retrieve, this has to match the current bound FBO! + /// how many pixel in x direction + /// how many pixel in y direction + /// with pixel content + /// Does usually not throw on error (e. g. wrong pixel format, out of bounds, etc), uses GL.GetError() to retrieve + /// potential error + public ReadOnlySpan ReadPixels(int x, int y, ImagePixelFormat pixelFormat, int width, int height) + { + var format = GetTexturePixelInfo(pixelFormat); + var data = new byte[width * height * pixelFormat.BytesPerPixel]; + + gl2.ReadPixels(x, y, 1, 1, format.Format, format.PxType, data); + + var err = gl2.GetError(); + if (err != NO_ERROR) + { + throw new Exception($"ReadPixel failed with error code {err}!"); + } + + return data; + } + /// /// Retrieves a sub-image of the given region. /// diff --git a/src/Engine/Imp/Graphics/Desktop/Fusee.Engine.Imp.Graphics.Desktop.csproj b/src/Engine/Imp/Graphics/Desktop/Fusee.Engine.Imp.Graphics.Desktop.csproj index b2701eb43..4ff3c6fd5 100644 --- a/src/Engine/Imp/Graphics/Desktop/Fusee.Engine.Imp.Graphics.Desktop.csproj +++ b/src/Engine/Imp/Graphics/Desktop/Fusee.Engine.Imp.Graphics.Desktop.csproj @@ -1,4 +1,4 @@ - + net7.0 @@ -27,7 +27,7 @@ analyzers - + diff --git a/src/Engine/Imp/Graphics/Desktop/RenderCanvasGameWindow.cs b/src/Engine/Imp/Graphics/Desktop/RenderCanvasGameWindow.cs index bd6f153dc..82eff9b79 100644 --- a/src/Engine/Imp/Graphics/Desktop/RenderCanvasGameWindow.cs +++ b/src/Engine/Imp/Graphics/Desktop/RenderCanvasGameWindow.cs @@ -2,6 +2,7 @@ using OpenTK.Windowing.Desktop; using OpenTK.Windowing.GraphicsLibraryFramework; using System; +using System.ComponentModel; namespace Fusee.Engine.Imp.Graphics.Desktop { @@ -95,9 +96,9 @@ protected override void OnLoad() int major = version[0]; // int minor = (int)version[2]; - if (major < 2) + if (major < 3) { - throw new InvalidOperationException("You need at least OpenGL 2.0 to run this example. GLSL not supported."); + throw new InvalidOperationException("You need at least OpenGL 3.0 to run this application. GLSL not supported."); } // Use VSync! @@ -123,6 +124,12 @@ protected override void OnResize(OpenTK.Windowing.Common.ResizeEventArgs e) } } + protected override void OnClosing(CancelEventArgs e) + { + _renderCanvasImp.DoClose(e); + base.OnClosing(e); + } + protected override void OnUpdateFrame(OpenTK.Windowing.Common.FrameEventArgs args) { base.OnUpdateFrame(args); diff --git a/src/Engine/Imp/Graphics/Desktop/RenderCanvasImp.cs b/src/Engine/Imp/Graphics/Desktop/RenderCanvasImp.cs index 7564bcaa9..b09b5c73e 100644 --- a/src/Engine/Imp/Graphics/Desktop/RenderCanvasImp.cs +++ b/src/Engine/Imp/Graphics/Desktop/RenderCanvasImp.cs @@ -7,6 +7,7 @@ using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; using System; +using System.ComponentModel; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -313,6 +314,11 @@ public RenderCanvasImp(int width, int height) /// public event EventHandler Resize; + /// + /// Occurs when [close] is called. + /// + public event EventHandler Closing; + #endregion #region Members @@ -349,7 +355,6 @@ public virtual void DoUnLoad() { UnLoad?.Invoke(this, new InitEventArgs()); } - /// /// Does the update of this instance. /// @@ -366,6 +371,14 @@ public virtual void DoRender() Render?.Invoke(this, new RenderEventArgs()); } + /// + /// Does close the window + /// + public virtual void DoClose(CancelEventArgs e) + { + Closing?.Invoke(this, e); + } + /// /// Does the resize on this instance. /// @@ -461,7 +474,23 @@ public virtual void Present() /// The type of the cursor to set. public void SetCursor(CursorType cursorType) { - // Currently not supported by OpenTK... Too bad. + switch (cursorType) + { + case CursorType.Standard: + _gameWindow.Cursor = MouseCursor.Default; + break; + case CursorType.Hand: + _gameWindow.Cursor = MouseCursor.Hand; + break; + case CursorType.HResize: + _gameWindow.Cursor = MouseCursor.HResize; + break; + case CursorType.VResize: + _gameWindow.Cursor = MouseCursor.VResize; + break; + default: + break; + } } /// @@ -490,8 +519,6 @@ public virtual void Run() if (_gameWindow != null) { _gameWindow.UpdateFrequency = 60; - _gameWindow.RenderFrequency = 0; - _gameWindow.Run(); } } diff --git a/src/Engine/Imp/Graphics/Desktop/RenderContextImp.cs b/src/Engine/Imp/Graphics/Desktop/RenderContextImp.cs index 74792d954..1d6e3c1e2 100644 --- a/src/Engine/Imp/Graphics/Desktop/RenderContextImp.cs +++ b/src/Engine/Imp/Graphics/Desktop/RenderContextImp.cs @@ -1236,6 +1236,33 @@ public void EnableDepthClamp() GL.Enable(EnableCap.DepthClamp); } + /// + /// Retrieve pixels from bound framebuffer + /// + /// x pixel position + /// y pixel position + /// format to retrieve, this has to match the current bound FBO! + /// how many pixel in x direction + /// how many pixel in y direction + /// with pixel content + /// Does usually not throw on error (e. g. wrong pixel format, out of bounds, etc), uses GL.GetError() to retrieve + /// potential error + public ReadOnlySpan ReadPixels(int x, int y, ImagePixelFormat pixelFormat, int width, int height) + { + var format = GetTexturePixelInfo(pixelFormat); + var data = new byte[width * height * pixelFormat.BytesPerPixel]; + + GL.ReadPixels(x, y, 1, 1, format.Format, format.PxType, data); + + var err = GL.GetError(); + if (err != ErrorCode.NoError) + { + throw new Exception($"ReadPixel failed with error code {err}!"); + } + + return data; + } + /// /// Disables depths clamping. /// @@ -2274,6 +2301,7 @@ public void Render(IMeshImp mr, IInstanceDataImp instanceData = null) Common.PrimitiveType.TriangleFan => OpenTK.Graphics.OpenGL.PrimitiveType.TriangleFan, Common.PrimitiveType.TriangleStrip => OpenTK.Graphics.OpenGL.PrimitiveType.TriangleStrip, Common.PrimitiveType.Quads => OpenTK.Graphics.OpenGL.PrimitiveType.Quads, + Common.PrimitiveType.LineAdjacency => OpenTK.Graphics.OpenGL.PrimitiveType.LinesAdjacency, _ => OpenTK.Graphics.OpenGL.PrimitiveType.Triangles, }; diff --git a/src/Engine/Player/Android/Fusee.Engine.Player.Android.csproj b/src/Engine/Player/Android/Fusee.Engine.Player.Android.csproj index a0836eaf8..b9661dc3a 100644 --- a/src/Engine/Player/Android/Fusee.Engine.Player.Android.csproj +++ b/src/Engine/Player/Android/Fusee.Engine.Player.Android.csproj @@ -1,4 +1,4 @@ - + Fusee.Engine.Player.Android @@ -41,7 +41,7 @@ - pdbonly + portable True $(DefineConstants);PLATFORM_ANDROID prompt @@ -49,14 +49,13 @@ False Full true - armeabi-v7a;x86;x86_64;arm64-v8a - + @@ -88,7 +87,7 @@ {1228EB3F-8BCC-453F-8A2E-D9246495A118} Fusee.Engine.Core - + {26808b4a-9f15-47f0-b147-e744241b79d2} Fusee.Engine.Gui diff --git a/src/Engine/Player/Android/Properties/AndroidManifest.xml b/src/Engine/Player/Android/Properties/AndroidManifest.xml index 6b8422464..727d568a7 100644 --- a/src/Engine/Player/Android/Properties/AndroidManifest.xml +++ b/src/Engine/Player/Android/Properties/AndroidManifest.xml @@ -1,9 +1,6 @@  - - - - + \ No newline at end of file diff --git a/src/Engine/Player/Blazor/Fusee.Engine.Player.Blazor.csproj b/src/Engine/Player/Blazor/Fusee.Engine.Player.Blazor.csproj index 1de162868..94b6428ac 100644 --- a/src/Engine/Player/Blazor/Fusee.Engine.Player.Blazor.csproj +++ b/src/Engine/Player/Blazor/Fusee.Engine.Player.Blazor.csproj @@ -1,4 +1,4 @@ - + net7.0 @@ -23,9 +23,9 @@ analyzers - - - + + + diff --git a/src/Engine/Player/Core/Fusee.Engine.Player.Core.csproj b/src/Engine/Player/Core/Fusee.Engine.Player.Core.csproj index f198d7368..3b11b337f 100644 --- a/src/Engine/Player/Core/Fusee.Engine.Player.Core.csproj +++ b/src/Engine/Player/Core/Fusee.Engine.Player.Core.csproj @@ -1,5 +1,5 @@ - + netstandard2.1;net7.0 $(BaseOutputPath)\Player\Core\ @@ -32,5 +32,5 @@ analyzers - + \ No newline at end of file diff --git a/src/Engine/Player/Core/Player.cs b/src/Engine/Player/Core/Player.cs index 8cf827777..5ab67f2a1 100644 --- a/src/Engine/Player/Core/Player.cs +++ b/src/Engine/Player/Core/Player.cs @@ -52,8 +52,6 @@ public async Task LoadAssets() _scene = await AssetStorage.GetAsync(ModelFile); _gui = await FuseeGuiHelper.CreateDefaultGuiAsync(this, _canvasRenderMode, "FUSEE Player"); - // Create the interaction handler - _sih = new SceneInteractionHandler(_gui); AABBCalculator aabbc = new(_scene); var bbox = aabbc.GetBox(); @@ -111,6 +109,9 @@ public async Task LoadAssets() // Wrap a SceneRenderer around the model. _sceneRenderer = new SceneRendererForward(_scene); _guiRenderer = new SceneRendererForward(_gui); + + // Create the interaction handler + _sih = new SceneInteractionHandler(_gui, _guiRenderer.PrePassVisitor.CameraPrepassResults); } public override async Task InitAsync() @@ -215,11 +216,11 @@ public override void RenderAFrame() _guiRenderer.Render(RC); // Constantly check for interactive objects. - _sih.CheckForInteractiveObjects(RC, Mouse.Position, Width, Height); + _sih.CheckForInteractiveObjects(Mouse.Position, Width, Height); if (Touch != null && Touch.GetTouchActive(TouchPoints.Touchpoint_0) && !Touch.TwoPoint) { - _sih.CheckForInteractiveObjects(RC, Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); + _sih.CheckForInteractiveObjects(Touch.GetPosition(TouchPoints.Touchpoint_0), Width, Height); } // Swap buffers: Show the contents of the backbuffer (containing the currently rendered frame) on the front buffer. diff --git a/src/ImGui/Desktop/Fusee.ImGui.Desktop/Fusee.ImGuiImp.Desktop.csproj b/src/ImGui/Desktop/Fusee.ImGui.Desktop/Fusee.ImGuiImp.Desktop.csproj index 75cf8f47b..941da3066 100644 --- a/src/ImGui/Desktop/Fusee.ImGui.Desktop/Fusee.ImGuiImp.Desktop.csproj +++ b/src/ImGui/Desktop/Fusee.ImGui.Desktop/Fusee.ImGuiImp.Desktop.csproj @@ -1,4 +1,4 @@ - + net7.0 @@ -13,7 +13,7 @@ - + diff --git a/src/ImGui/Desktop/Fusee.ImGui.Desktop/ImGuiRenderCanvasImp.cs b/src/ImGui/Desktop/Fusee.ImGui.Desktop/ImGuiRenderCanvasImp.cs index 5e934b42c..b1bd4212f 100644 --- a/src/ImGui/Desktop/Fusee.ImGui.Desktop/ImGuiRenderCanvasImp.cs +++ b/src/ImGui/Desktop/Fusee.ImGui.Desktop/ImGuiRenderCanvasImp.cs @@ -45,8 +45,6 @@ public override void Run() { //MUST be 0, is handled internally by ImGui. Other values will lead to AccessViolation Exception. _gameWindow.UpdateFrequency = 0; - _gameWindow.RenderFrequency = 0; - _gameWindow.Run(); } } diff --git a/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFilePicker.cs b/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFilePicker.cs index 9286f82e7..c4091a388 100644 --- a/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFilePicker.cs +++ b/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFilePicker.cs @@ -11,13 +11,23 @@ public class ImGuiFilePicker /// /// Invoked on clicked "open". /// - public EventHandler? OnPicked; + public EventHandler? OnPicked; /// /// Invoked on cancel. /// public EventHandler? OnCancel; + /// + /// Allow resizing of file picker window + /// + public bool AllowFilePickerResize { get; set; } = true; + + /// + /// Allow resizing of new folder window + /// + public bool AllowNewFolderResize { get; set; } = true; + /// /// Title of window (visible in top bar). /// @@ -66,30 +76,90 @@ public class ImGuiFilePicker public string ParentFolderTxt = "Parent"; public string BackTxt = "Back"; - public readonly bool OnlyAllowFolders; - public readonly List? AllowedExtensions; + public List? AllowedExtensions; - public string? SelectedFile { get; protected set; } - public string RootFolder { get; protected set; } + /// + /// Show a button which let's the user create a new folder at the current directory + /// + public bool ShowNewFolderButton { get; set; } + + public string NewFolderButtonTxt = "\uf65e"; + + /// + /// Caption of the create new folder window + /// + public string CreateNewFolderTxt = "Create new folder"; + + /// + /// Caption of the create folder button + /// + public string CreateFolderTxt = "Create folder"; + + /// + /// Create new folder name hint txt + /// + public string CreateNewFolderHintTxt = "Insert folder name"; + + private bool _isNewFolderNameWindowOpen; + + // as we cannot use the property as ref, we need to check and set all variables every time + // so that, when we call if(IsNewFolderNameWindowOpen), the variables are being set properly, too even when the window itself was closed via 'x' + private bool IsNewFolderNameWindowOpen + { + get + { + if (_isNewFolderNameWindowOpen) + { + // push the folder window to the back + DoFocusPicker = false; + } + else + { + // reset text and reset windows + _createFolderException = null; + _newFolderName = ""; + DoFocusPicker = true; + } + return _isNewFolderNameWindowOpen; + } + set + { + _isNewFolderNameWindowOpen = value; + if (_isNewFolderNameWindowOpen) + { + // push the folder window to the back + DoFocusPicker = false; + } + else + { + // reset text and reset windows + _createFolderException = null; + _newFolderName = ""; + DoFocusPicker = true; + } + } + } + private string _newFolderName = ""; + private Exception? _createFolderException; + + public FileInfo? SelectedFile { get; protected set; } + public DirectoryInfo RootFolder { get; protected set; } public int FontSize; public ImFontPtr SymbolsFontPtr = null; - protected string CurrentOpenFolder; - protected readonly Stack LastOpenendFolders = new(); - protected string CurrentlySelectedFolder; - protected readonly string StartingFolder; + protected DirectoryInfo CurrentOpenFolder; + protected readonly Stack LastOpenendFolders = new(); + protected DirectoryInfo CurrentlySelectedFolder; + protected readonly DirectoryInfo StartingFolder; - protected const float FolderTextInputWidth = 350; - protected const float FileTextInputWidth = 300; - protected const float DriveSelectionWidth = 100; - protected const float BrowserHeight = 200; protected readonly Vector2 WindowPadding = new(15, 15); protected readonly Vector2 BottomButtonSize = new(55, 26); protected readonly Vector2 TopButtonSize = new(35, 30); - protected Vector2 WinSize; + protected bool DoFocusPicker = true; + private static int _filePickerCount = 0; public bool IsOpen @@ -113,6 +183,12 @@ public bool IsOpen // needed for width calculation protected Vector2 _sizeOfInputText; + /// + /// Checks input before returning if file exists. + /// Disable this check for file saving. + /// + public bool NonExistingFilesAllowed = false; + /// /// Text color of folder /// @@ -121,7 +197,18 @@ public bool IsOpen /// /// Background color of pop up window /// - public Vector4 WindowBackground = new(200, 200, 200, 255); + public Vector4 WindowBackground + { + get => _windowBackground; + set + { + _windowBackground = value; + _windowBackgroundUint = _windowBackground.ToUintColor(); + } + } + private Vector4 _windowBackground = new(200, 200, 200, 255); + + public uint _windowBackgroundUint = new Vector4(200, 200, 200, 255).ToUintColor(); /// /// Background of file selection menu @@ -141,28 +228,21 @@ public bool IsOpen /// /// Generate a new ImGuiFilePicker instance /// - /// Starting path, defaults to C:\ - /// Allow folder picking only + /// Starting path, defaults to /// search filter with dot. Example (".json") - public ImGuiFilePicker(string startingPath = "C:\\", bool onlyAllowFolders = true, string allowedExtensions = "") + public ImGuiFilePicker(DirectoryInfo? startingPath = null, string allowedExtensions = "") { _filePickerCount++; - if (File.Exists(startingPath)) + if (startingPath == null || !startingPath.Exists) { - startingPath = new FileInfo(startingPath)?.DirectoryName ?? "C:\\"; - } - else if (string.IsNullOrEmpty(startingPath) || !Directory.Exists(startingPath)) - { - startingPath = Environment.CurrentDirectory; - if (string.IsNullOrEmpty(startingPath)) - startingPath = AppContext.BaseDirectory; + startingPath = new DirectoryInfo(AppContext.BaseDirectory); } RootFolder = startingPath; CurrentOpenFolder = startingPath; StartingFolder = startingPath; - OnlyAllowFolders = onlyAllowFolders; + CurrentlySelectedFolder = startingPath; if (!string.IsNullOrEmpty(allowedExtensions)) { @@ -174,64 +254,130 @@ public ImGuiFilePicker(string startingPath = "C:\\", bool onlyAllowFolders = tru AllowedExtensions.AddRange(allowedExtensions.Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries)); } } - public virtual unsafe void Draw(ref bool filePickerOpen) { IsOpen = filePickerOpen; if (!filePickerOpen) return; - ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, WindowPadding); - ImGui.PushStyleVar(ImGuiStyleVar.ChildBorderSize, 0); - ImGui.PushStyleColor(ImGuiCol.WindowBg, WindowBackground.ToUintColor()); + // close on ESC + if (ImGui.IsKeyReleased(ImGuiKey.Escape)) + { + OnCancel?.Invoke(this, EventArgs.Empty); + filePickerOpen = false; + } if (DoFocusPicker) ImGui.SetNextWindowFocus(); - var headerHeight = FontSize + WindowPadding.Y * 2; - var itemSpacing = ImGui.GetStyle().ItemSpacing; - WinSize = new Vector2(FolderTextInputWidth + DriveSelectionWidth + (WindowPadding.X * 2) + itemSpacing.X, headerHeight + BrowserHeight + TopButtonSize.Y + BottomButtonSize.Y + 4 * WindowPadding.Y + 3 * itemSpacing.Y + 5); - ImGui.SetNextWindowSize(WinSize); - ImGui.Begin(Id, ref filePickerOpen, ImGuiWindowFlags.Modal | ImGuiWindowFlags.NoCollapse | ImGuiWindowFlags.AlwaysAutoResize | ImGuiWindowFlags.NoDocking); + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, WindowPadding); + ImGui.PushStyleVar(ImGuiStyleVar.ChildBorderSize, 0); + ImGui.PushStyleColor(ImGuiCol.WindowBg, _windowBackgroundUint); + + // Begin window + ImGui.SetNextWindowSizeConstraints(new Vector2(500, 300), ImGui.GetWindowViewport().Size * 0.75f); + var allowResizeFlag = AllowFilePickerResize ? ImGuiWindowFlags.None : ImGuiWindowFlags.NoResize; + ImGui.Begin(Id, ref filePickerOpen, ImGuiWindowFlags.Modal | ImGuiWindowFlags.NoCollapse | allowResizeFlag); + + // draw navigation buttons and folder selection on the same line + DrawNavButtons(); + DrawFolderSelectionTextInput(); + + // draw drive and file selector window + ImGui.NewLine(); + ImGui.PushStyleColor(ImGuiCol.ChildBg, FileSelectionMenuBackground.ToUintColor()); + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(10, 10)); + + DrawDriveSelector(); + DrawFolderSelector(ref filePickerOpen); + + ImGui.PopStyleColor(); + ImGui.PopStyleVar(); + + // draw okay, cancel button and file selector + ImGui.NewLine(); + DrawFileSelector(ref filePickerOpen); + + ImGui.End(); + + + if (ShowNewFolderButton && IsNewFolderNameWindowOpen) + { + DrawNewFolderOverlay(CurrentOpenFolder); + } + + ImGui.PopStyleVar(2); + ImGui.PopStyleColor(); + } + + private unsafe void DrawNavButtons() + { if ((IntPtr)SymbolsFontPtr.NativePtr != IntPtr.Zero) ImGui.PushFont(SymbolsFontPtr); ImGui.BeginGroup(); - var di = new DirectoryInfo(CurrentOpenFolder); - if (ImGui.Button($"{ParentFolderTxt}##{_filePickerCount}", TopButtonSize)) + var parentFolderButtonSize = ImGui.CalcTextSize(ParentFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var backButtonSize = ImGui.CalcTextSize(ParentFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var newFolderButtonSize = ImGui.CalcTextSize(NewFolderButtonTxt) + ImGui.GetStyle().FramePadding * 2; + + parentFolderButtonSize += new Vector2(5, 0); // add a little offset as the arrows aren't wide enough + backButtonSize += new Vector2(5, 0); // add a little offset as the arrows aren't wide enough + + if (ImGui.Button($"{ParentFolderTxt}##{_filePickerCount}", parentFolderButtonSize)) { - if (di.Exists && di.Parent != null) + if (CurrentOpenFolder.Exists && CurrentOpenFolder.Parent != null) { LastOpenendFolders.Push(CurrentOpenFolder); - CurrentOpenFolder = di.Parent.FullName; - SelectedFile = ""; + CurrentOpenFolder = CurrentOpenFolder.Parent; + SelectedFile = null; } } ImGui.SameLine(); - if (ImGui.Button($"{BackTxt}##{_filePickerCount}", TopButtonSize)) + if (LastOpenendFolders.Count != 0) { - if (LastOpenendFolders.Count != 0) + + if (ImGui.Button($"{BackTxt}##{_filePickerCount}", backButtonSize)) { + var lastFolder = LastOpenendFolders.Pop(); - var lastDi = new DirectoryInfo(lastFolder); + var lastDi = lastFolder; if (lastDi.Exists) { CurrentOpenFolder = lastFolder; - SelectedFile = ""; + SelectedFile = null; } } } + else + { + ImGui.BeginDisabled(); + ImGui.Button($"{BackTxt}##{_filePickerCount}", backButtonSize); + ImGui.EndDisabled(); + } + + if (ShowNewFolderButton) + { + ImGui.SameLine(); + if (ImGui.Button($"{NewFolderButtonTxt}##{_filePickerCount}", newFolderButtonSize)) + { + _isNewFolderNameWindowOpen = true; + } + } if ((IntPtr)SymbolsFontPtr.NativePtr != IntPtr.Zero) ImGui.PopFont(); ImGui.EndGroup(); + } - // Folder Selection - var currentFolder = CurrentOpenFolder; - ImGui.SameLine(DriveSelectionWidth + WindowPadding.X + ImGui.GetStyle().ItemSpacing.X); - ImGui.SetNextItemWidth(FolderTextInputWidth - ImGui.CalcTextSize(FolderLabelTxt).X - ImGui.GetStyle().ItemSpacing.X); - ImGui.InputTextWithHint($"{FolderLabelTxt}##{_filePickerCount}", PathToFolderTxt, ref currentFolder, 400, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => + private unsafe void DrawFolderSelectionTextInput() + { + var currentFolder = Environment.ExpandEnvironmentVariables(CurrentOpenFolder.FullName); + + ImGui.SameLine(); + // occupy the max available space, minus the label text length + ImGui.SetNextItemWidth(-ImGui.CalcTextSize(FolderLabelTxt).X); + ImGui.InputTextWithHint($"{FolderLabelTxt}##{_filePickerCount}", PathToFolderTxt, ref currentFolder, 4098, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => { var arr = currentFolder.ToCharArray(); @@ -244,75 +390,104 @@ public virtual unsafe void Draw(ref bool filePickerOpen) return 0; }); - if (Directory.Exists(currentFolder)) - { - CurrentOpenFolder = currentFolder; - } - else if (File.Exists(currentFolder)) + + var envCurrentFolder = Environment.ExpandEnvironmentVariables(currentFolder); + + if (!string.IsNullOrEmpty(envCurrentFolder)) { - var fi = new FileInfo(currentFolder); - if (fi.DirectoryName != null) + var currentFolderFi = new FileInfo(envCurrentFolder); // parse something like folder and file (e. g. C:\test\test.las) + + if (Directory.Exists(envCurrentFolder) || envCurrentFolder.Contains(Path.DirectorySeparatorChar) || envCurrentFolder.Contains(Path.AltDirectorySeparatorChar)) + { + if (Directory.Exists(envCurrentFolder)) + { + CurrentOpenFolder = new DirectoryInfo(envCurrentFolder); + CurrentlySelectedFolder = new DirectoryInfo(envCurrentFolder); + } + + + if (File.Exists(currentFolderFi.FullName) && !string.IsNullOrEmpty(currentFolderFi.DirectoryName)) + { + CurrentOpenFolder = new DirectoryInfo(currentFolderFi.DirectoryName); + CurrentlySelectedFolder = new DirectoryInfo(currentFolderFi.DirectoryName); + SelectedFile = currentFolderFi; + } + else if (currentFolderFi.Extension == ".las") // only check if extension is given, print error only when no las file is found + { + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); + ImGui.BeginTooltip(); + ImGui.TextColored(WarningTextColor, FileNotFoundTxt); + ImGui.EndTooltip(); + ImGui.PopStyleVar(); + } + + } + else { - CurrentOpenFolder = fi.DirectoryName; - SelectedFile = fi.Name; + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); + ImGui.BeginTooltip(); + ImGui.TextColored(WarningTextColor, FolderNotFoundTxt); + ImGui.EndTooltip(); + ImGui.PopStyleVar(); } } - else - { - ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); - ImGui.BeginTooltip(); - ImGui.TextColored(WarningTextColor, FolderNotFoundTxt); - ImGui.EndTooltip(); - ImGui.PopStyleVar(); - } + } - // Folder Browser - ImGui.NewLine(); - ImGui.PushStyleColor(ImGuiCol.ChildBg, FileSelectionMenuBackground.ToUintColor()); - ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(10, 10)); + private void DrawDriveSelector() + { + var driveSelectionWidth = ImGui.GetWindowSize().X * 0.25f; // 25% of windowSize.x + // take all space in y, however shrink in y in item height + standard padding + WindowPadding + var offsetFromBottom = ImGui.CalcTextSize(PickedFileTxt) + ImGui.GetStyle().FramePadding * 2 + ImGui.GetStyle().WindowPadding * 2; + ImGui.BeginChild($"DriveSelection##{_filePickerCount}", new Vector2(driveSelectionWidth, -offsetFromBottom.Y), false, ImGuiWindowFlags.AlwaysUseWindowPadding); - ImGui.BeginChild($"DriveSelection##{_filePickerCount}", new Vector2(DriveSelectionWidth, BrowserHeight), false, ImGuiWindowFlags.AlwaysUseWindowPadding | ImGuiWindowFlags.AlwaysAutoResize); - // Drive Selection var driveCount = 0; foreach (var drive in DriveInfo.GetDrives()) { - if (drive.IsReady) + if (drive.IsReady && (drive.DriveType == DriveType.Fixed || drive.DriveType == DriveType.Removable)) { if (ImGui.Selectable($"{drive.Name} {drive.DriveType}##{_filePickerCount}")) { - RootFolder = drive.Name; + RootFolder = new DirectoryInfo(drive.Name); LastOpenendFolders.Push(CurrentOpenFolder); - CurrentOpenFolder = drive.Name; - SelectedFile = ""; + CurrentOpenFolder = new DirectoryInfo(drive.Name); + SelectedFile = null; } driveCount++; } } ImGui.EndChild(); - ImGui.SameLine(); + } - if (ImGui.BeginChild($"#FolderBrowser##{_filePickerCount}", new Vector2(FolderTextInputWidth, BrowserHeight), false, ImGuiWindowFlags.AlwaysUseWindowPadding | ImGuiWindowFlags.AlwaysAutoResize | ImGuiWindowFlags.HorizontalScrollbar)) + private void DrawFolderSelector(ref bool filePickerOpen) + { + ImGui.SameLine(); + ImGui.GetWindowHeight(); + // take all space in y, however shrink in y in item height + standard padding + WindowPadding + var offsetFromBottom = ImGui.CalcTextSize(PickedFileTxt) + ImGui.GetStyle().FramePadding * 2 + ImGui.GetStyle().WindowPadding * 2; + if (ImGui.BeginChild($"#FolderBrowser##{_filePickerCount}", new Vector2(-1, -offsetFromBottom.Y), false, ImGuiWindowFlags.AlwaysUseWindowPadding | ImGuiWindowFlags.HorizontalScrollbar)) { - di = new DirectoryInfo(CurrentOpenFolder); - if (di.Exists) + if (CurrentOpenFolder != null && CurrentOpenFolder.Exists) { - var fileSystemEntries = GetFileSystemEntries(di.FullName); + var fileSystemEntries = GetFileSystemEntries(CurrentOpenFolder.FullName); foreach (var fse in fileSystemEntries) { - if (Directory.Exists(fse)) + if (fse.Attributes.HasFlag(FileAttributes.Directory)) { - var name = Path.GetFileName(fse); + var directory = new DirectoryInfo(fse.FullName); ImGui.PushStyleColor(ImGuiCol.Text, FolderColor.ToUintColor()); - if (ImGui.Selectable(name + "/", CurrentlySelectedFolder == fse, ImGuiSelectableFlags.DontClosePopups | ImGuiSelectableFlags.AllowDoubleClick)) + if (ImGui.Selectable(directory.Name + "/", CurrentlySelectedFolder == directory, ImGuiSelectableFlags.DontClosePopups | ImGuiSelectableFlags.AllowDoubleClick)) { - SelectedFile = ""; - CurrentlySelectedFolder = fse; - if (ImGui.IsMouseDoubleClicked(0) && (SelectedFile == null || SelectedFile == "") && ImGui.GetIO().WantCaptureMouse) + SelectedFile = null; + CurrentlySelectedFolder = directory; + if (ImGui.IsMouseDoubleClicked(0)) { - LastOpenendFolders.Push(CurrentOpenFolder); - CurrentOpenFolder = fse; + if (SelectedFile == null && ImGui.GetIO().WantCaptureMouse) + { + LastOpenendFolders.Push(CurrentOpenFolder); + CurrentOpenFolder = directory; + } } } @@ -320,43 +495,53 @@ public virtual unsafe void Draw(ref bool filePickerOpen) } else { - var name = Path.GetFileName(fse); + var name = fse.Name; ImGui.PushStyleColor(ImGuiCol.Header, SelectedColor.ToUintColor()); - if (ImGui.Selectable(name, SelectedFile == name, ImGuiSelectableFlags.DontClosePopups | ImGuiSelectableFlags.AllowDoubleClick)) + if (ImGui.Selectable(name, SelectedFile?.Name == name, ImGuiSelectableFlags.DontClosePopups | ImGuiSelectableFlags.AllowDoubleClick)) { - SelectedFile = name; - if (ImGui.IsMouseDoubleClicked(0) && (SelectedFile != null && SelectedFile != "") && ImGui.GetIO().WantCaptureMouse) + if (ImGui.IsMouseDoubleClicked(0)) { - if (HandlePickedFile(SelectedFile)) + if (SelectedFile != null && ImGui.GetIO().WantCaptureMouse) { - filePickerOpen = false; - OnPicked?.Invoke(this, Path.Combine(CurrentOpenFolder, SelectedFile)); + if (HandlePickedFile(SelectedFile)) + { + filePickerOpen = false; + OnPicked?.Invoke(this, SelectedFile); + } } } + else + { + if (SelectedFile == fse) + SelectedFile = null; + else + SelectedFile = new FileInfo(fse.FullName); + } } ImGui.PopStyleColor(); } } } - - ImGui.PopStyleColor(); - ImGui.PopStyleVar(); - ImGui.EndChild(); } + ImGui.EndChild(); + } - // File Selector - ImGui.NewLine(); - ImGui.BeginChild($"FileSelector##{_filePickerCount}", new Vector2(-1, -1), false, ImGuiWindowFlags.AlwaysAutoResize); + private unsafe void DrawFileSelector(ref bool filePickerOpen) + { + var pickedFileButtonSize = ImGui.CalcTextSize(PickedFileTxt) + ImGui.GetStyle().FramePadding * 2; + var cancelFileButtonSize = ImGui.CalcTextSize(CancelFileOpenTxt) + ImGui.GetStyle().FramePadding * 2; - var selectedFile = !string.IsNullOrWhiteSpace(SelectedFile) ? SelectedFile : ""; - ImGui.SetNextItemWidth(FileTextInputWidth - ImGui.CalcTextSize(FileLabelTxt).X - ImGui.GetStyle().ItemSpacing.X); - ImGui.InputTextWithHint(FileLabelTxt, FileInputHintTxt, ref selectedFile, 400, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => + var selectedFile = SelectedFile?.Name ?? ""; + // take all available space minus the label text and minus both buttons + var inputTextMaxLength = ImGui.CalcTextSize(FileLabelTxt).X + ImGui.GetStyle().ItemInnerSpacing.X * 4 + pickedFileButtonSize.X + cancelFileButtonSize.X; + ImGui.SetNextItemWidth(-inputTextMaxLength); + + if (ImGui.InputTextWithHint(FileLabelTxt, FileInputHintTxt, ref selectedFile, 4096, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => { var arr = selectedFile.ToCharArray(); - if (x->SelectionStart < x->SelectionEnd && x->SelectionStart >= 0 && x->SelectionEnd <= arr.Length) { var selectedText = arr[x->SelectionStart..x->SelectionEnd]; @@ -364,126 +549,211 @@ public virtual unsafe void Draw(ref bool filePickerOpen) ImGuiInputImp.CurrentlySelectedText = new string(selectedText); } return 0; - }); - if (_sizeOfInputText == Vector2.Zero) - _sizeOfInputText = ImGui.GetItemRectSize(); + })) + { + if (!string.IsNullOrEmpty(selectedFile) && !char.IsWhiteSpace(selectedFile[0]) && CurrentOpenFolder != null) + SelectedFile = new FileInfo(Path.Combine(CurrentOpenFolder.FullName, selectedFile)); + } - var sameLineOffset = WinSize.X - WindowPadding.X - (BottomButtonSize.X * 2 + ImGui.GetStyle().ItemSpacing.X * 4); - if (!string.IsNullOrWhiteSpace(SelectedFile)) + // increase spacing between okay button and input + var normalSpacing = ImGui.GetStyle().ItemSpacing; + ImGui.PushStyleVar(ImGuiStyleVar.ItemSpacing, new Vector2(15, normalSpacing.Y)); + + if (SelectedFile != null) { - var fi = new FileInfo(Path.Combine(CurrentOpenFolder, SelectedFile)); - if (fi.Exists && AllowedExtensions != null && AllowedExtensions.Contains(fi.Extension)) + var fi = SelectedFile; + if (AllowedExtensions != null && AllowedExtensions.Contains(fi.Extension)) { - ImGui.SameLine(sameLineOffset); - if (ImGui.Button($"{PickedFileTxt}##{_filePickerCount}", BottomButtonSize)) + ImGui.SameLine(); + + if (ImGui.Button($"{PickedFileTxt}##{_filePickerCount}", pickedFileButtonSize) || + (ImGui.IsKeyReleased(ImGuiKey.Enter) && !IsNewFolderNameWindowOpen)) { - if (HandlePickedFile(selectedFile)) + if (HandlePickedFile(fi)) { - OnPicked?.Invoke(this, Path.Combine(CurrentOpenFolder, selectedFile)); + OnPicked?.Invoke(this, fi); filePickerOpen = false; } } } + else + { + ImGui.SameLine(); + ImGui.BeginDisabled(); + ImGui.Button(PickedFileTxt, pickedFileButtonSize); + ImGui.EndDisabled(); + } } else { - ImGui.SameLine(sameLineOffset); + ImGui.SameLine(); ImGui.BeginDisabled(); - ImGui.Button(PickedFileTxt, BottomButtonSize); + ImGui.Button(PickedFileTxt, pickedFileButtonSize); ImGui.EndDisabled(); } + ImGui.PopStyleVar(); + ImGui.SameLine(); - if (ImGui.Button($"{CancelFileOpenTxt}##{_filePickerCount}", BottomButtonSize)) + if (ImGui.Button($"{CancelFileOpenTxt}##{_filePickerCount}", cancelFileButtonSize)) { OnCancel?.Invoke(this, EventArgs.Empty); filePickerOpen = false; } - - ImGui.EndChild(); - - ImGui.End(); - - ImGui.PopStyleVar(2); - ImGui.PopStyleColor(); - - return; } - private List GetFileSystemEntries(string fullName) + private unsafe void DrawNewFolderOverlay(DirectoryInfo currentFolder) { - try + ImGui.SetNextWindowFocus(); + // Calculate min height with button size + var createFolderButtonSize = ImGui.CalcTextSize(CreateFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var minWindowHeight = createFolderButtonSize.Y + ImGui.GetStyle().WindowPadding.Y * 4; + var minWindowLength = createFolderButtonSize.X + ImGui.CalcTextSize(CreateNewFolderHintTxt).X + ImGui.GetStyle().FramePadding.X * 4 + ImGui.GetStyle().ItemSpacing.X * 4; + ImGui.SetNextWindowSizeConstraints(new Vector2(minWindowLength, minWindowHeight), new Vector2(ImGui.GetWindowViewport().Size.X * 0.5f, minWindowHeight)); + ImGui.SetNextItemWidth(minWindowLength + ImGui.GetStyle().WindowPadding.X); + + var allowResizeFlag = AllowNewFolderResize ? ImGuiWindowFlags.None : ImGuiWindowFlags.NoResize; + ImGui.Begin($"{CreateNewFolderTxt}##{_filePickerCount}", ref _isNewFolderNameWindowOpen, ImGuiWindowFlags.Modal | ImGuiWindowFlags.NoCollapse | allowResizeFlag); + + // take the full width minus the button size + ImGui.SetNextItemWidth(-createFolderButtonSize.X); + ImGui.InputTextWithHint($"", $"{CreateNewFolderHintTxt}", ref _newFolderName, 4096, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => { - var files = new List(); - var dirs = new List(); + var arr = _newFolderName.ToCharArray(); - foreach (var fse in Directory.GetFileSystemEntries(fullName, "")) + if (x->SelectionStart < x->SelectionEnd && x->SelectionStart >= 0 && x->SelectionEnd <= arr.Length) { - if (Directory.Exists(fse)) - { - dirs.Add(fse); - } - else if (!OnlyAllowFolders) + var selectedText = arr[x->SelectionStart..x->SelectionEnd]; + if (selectedText != null) + ImGuiInputImp.CurrentlySelectedText = new string(selectedText); + } + + return 0; + }); + ImGui.SameLine(); + + if (ImGui.Button($"{CreateFolderTxt}", createFolderButtonSize) || + ImGui.IsKeyReleased(ImGuiKey.Enter)) + { + if (!string.IsNullOrEmpty(_newFolderName)) + { + var folderName = string.Empty; + try { - if (AllowedExtensions != null) + if (Path.IsPathRooted(_newFolderName)) { - var ext = Path.GetExtension(fse); - if (AllowedExtensions.Contains(ext)) - files.Add(fse); + folderName = _newFolderName; + Directory.CreateDirectory(_newFolderName); } else { - files.Add(fse); + folderName = Path.Combine(currentFolder.FullName, _newFolderName); + Directory.CreateDirectory(folderName); } } - } - - var ret = new List(dirs); - ret.AddRange(files); + catch (Exception ex) + { + _createFolderException = ex; + return; + } - return ret; + // open new folder + CurrentlySelectedFolder = new DirectoryInfo(folderName); + LastOpenendFolders.Push(CurrentOpenFolder); + CurrentOpenFolder = new DirectoryInfo(folderName); + } + IsNewFolderNameWindowOpen = false; } - catch (Exception) + + // display a possible exception during folder creation as a tooltip text + if (_createFolderException != null) { - return new List(); + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); + var size = ImGui.CalcTextSize(_createFolderException?.Message); + ImGui.SetNextWindowSize(new Vector2(size.X / 4, -1)); + ImGui.BeginTooltip(); + ImGui.PushStyleColor(ImGuiCol.Text, WarningTextColor); + ImGui.TextWrapped(_createFolderException?.Message); + ImGui.PopStyleColor(); + ImGui.EndTooltip(); + ImGui.PopStyleVar(); } + + ImGui.End(); } - protected virtual bool HandlePickedFile(string selectedFile) + + /// + /// We differentiate between files and folders, as we want to print the folders first + /// If we collect everything in one list all files and folders are being sorted alphabetically + /// + /// + /// + private List GetFileSystemEntries(string fullName) { - if (File.Exists(selectedFile)) + var folders = new List(); + var files = new List(); + + foreach (var f in Directory.GetFileSystemEntries(fullName, "")) { - var fi = new FileInfo(selectedFile); - if (fi.DirectoryName != null) + var attr = File.GetAttributes(f); + // skip unaccessible files and folders + if (attr.HasFlag(FileAttributes.Encrypted) + || attr.HasFlag(FileAttributes.Hidden) + || attr.HasFlag(FileAttributes.System) + || attr.HasFlag(FileAttributes.Temporary)) + continue; + + if (attr.HasFlag(FileAttributes.Directory)) { - CurrentOpenFolder = fi.DirectoryName; - SelectedFile = fi.Name; - return true; + folders.Add(new DirectoryInfo(f)); + } + else + { + var fse = new FileInfo(f); + if (AllowedExtensions != null) + { + var ext = fse.Extension; + if (AllowedExtensions.Contains(ext)) + files.Add(fse); + } + else + { + files.Add(fse); + } } } - else if (File.Exists(Path.Combine(CurrentOpenFolder, selectedFile))) - { - SelectedFile = selectedFile; - return true; - } - else if (Directory.Exists(selectedFile)) - { - SelectedFile = ""; - CurrentOpenFolder = selectedFile; - return false; - } - else if (!string.IsNullOrWhiteSpace(selectedFile)) + + var ret = new List(folders); + ret.AddRange(files); + return ret; + } + + protected virtual bool HandlePickedFile(FileInfo selectedFile) + { + + if (selectedFile.Directory != null) { - ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); - ImGui.BeginTooltip(); - ImGui.TextColored(WarningTextColor, FileNotFoundTxt); - ImGui.EndTooltip(); - ImGui.PopStyleVar(); + if (NonExistingFilesAllowed || selectedFile.Exists) + { + CurrentOpenFolder = selectedFile.Directory; + SelectedFile = selectedFile; + return true; + } + else if (selectedFile != null) + { + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); + ImGui.BeginTooltip(); + ImGui.TextColored(WarningTextColor, FileNotFoundTxt); + ImGui.EndTooltip(); + ImGui.PopStyleVar(); - return false; + return false; + } } + return false; } } diff --git a/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFolderPicker.cs b/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFolderPicker.cs new file mode 100644 index 000000000..c275639a2 --- /dev/null +++ b/src/ImGui/Desktop/Fusee.ImGui.Desktop/Templates/ImGuiFolderPicker.cs @@ -0,0 +1,621 @@ +using ImGuiNET; +using System; +using System.Collections.Generic; +using System.IO; +using System.Numerics; + +namespace Fusee.ImGuiImp.Desktop.Templates +{ + public class ImGuiFolderPicker + { + /// + /// Invoked on clicked "open". + /// + public EventHandler? OnPicked; + + /// + /// Invoked on cancel. + /// + public EventHandler? OnCancel; + + /// + /// Allow resizing of file picker window + /// + public bool AllowFolderPickerResize { get; set; } = true; + + /// + /// Allow resizing of new folder window + /// + public bool AllowNewFolderResize { get; set; } = true; + + /// + /// Title of window (visible in top bar). + /// + public string Id = "Open Folder"; + + /// + /// Caption of the "Open" button. + /// + public string PickedFolderTxt = "Open"; + + /// + /// Caption of the "Cancel" button. + /// + public string CancelFolderOpenTxt = "Cancel"; + + /// + /// Path to folder text. + /// + public string PathToFolderTxt = "Path to folder"; + + /// + /// Folder not found warning text. + /// + public string FolderNotFoundTxt = "Folder not found!"; + + /// + /// Caption of folder input text + /// + public string FolderLabelTxt = "Folder"; + + /// + /// Caption of folder input text (bottom) + /// + public string SelectedFolderLabelTxt = "Folder"; + + public string ParentFolderTxt = "Parent"; + public string BackTxt = "Back"; + + /// + /// Show a button which let's the user create a new folder at the current directory + /// + public bool ShowNewFolderButton { get; set; } + + public string NewFolderButtonTxt = "\uf65e"; + + /// + /// Caption of the create new folder window + /// + public string CreateNewFolderTxt = "Create new folder"; + + /// + /// Caption of the create folder button + /// + public string CreateFolderTxt = "Create folder"; + + /// + /// Create new folder name hint txt + /// + public string CreateNewFolderHintTxt = "Insert folder name"; + + private bool _isNewFolderNameWindowOpen; + + // as we cannot use the property as ref, we need to check and set all variables every time + // so that, when we call if(IsNewFolderNameWindowOpen), the variables are being set properly, too even when the window itself was closed via 'x' + private bool IsNewFolderNameWindowOpen + { + get + { + if (_isNewFolderNameWindowOpen) + { + // push the folder window to the back + DoFocusPicker = false; + } + else + { + // reset text and reset windows + _createFolderException = null; + _newFolderName = ""; + DoFocusPicker = true; + } + return _isNewFolderNameWindowOpen; + } + set + { + _isNewFolderNameWindowOpen = value; + if (_isNewFolderNameWindowOpen) + { + // push the folder window to the back + DoFocusPicker = false; + } + else + { + // reset text and reset windows + _createFolderException = null; + _newFolderName = ""; + DoFocusPicker = true; + } + } + } + private string _newFolderName = ""; + private Exception? _createFolderException; + + public DirectoryInfo? SelectedFolder { get; protected set; } + public DirectoryInfo RootFolder { get; protected set; } + + public int FontSize; + public ImFontPtr SymbolsFontPtr = null; + + protected DirectoryInfo CurrentOpenFolder; + protected readonly Stack LastOpenendFolders = new(); + protected DirectoryInfo? CurrentlySelectedFolder; + protected readonly DirectoryInfo StartingFolder; + + protected readonly Vector2 WindowPadding = new(15, 15); + protected readonly Vector2 BottomButtonSize = new(55, 26); + protected readonly Vector2 TopButtonSize = new(35, 30); + + protected bool DoFocusPicker = true; + + private static int _folderPickerCount = 0; + + public bool IsOpen + { + get + { + return _isOpen; + } + set + { + if (value != _isOpen) + { + _isOpen = value; + if (!_isOpen) + CurrentOpenFolder = StartingFolder; + } + } + } + protected bool _isOpen; + + // needed for width calculation + protected Vector2 _sizeOfInputText; + + /// + /// Text color of folder + /// + public Vector4 FolderColor = new(255, 0, 255, 255); + + /// + /// Background color of pop up window + /// + public Vector4 WindowBackground + { + get => _windowBackground; + set + { + _windowBackground = value; + _windowBackgroundUint = _windowBackground.ToUintColor(); + } + } + private Vector4 _windowBackground = new(200, 200, 200, 255); + + public uint _windowBackgroundUint = new Vector4(200, 200, 200, 255).ToUintColor(); + + /// + /// Background of file selection menu + /// + public Vector4 FolderSelectionMenuBackground = new(125, 125, 125, 255); + + /// + /// Color of when an error occurs + /// + public Vector4 WarningTextColor = new(200, 0, 0, 255); + + /// + /// Background color of one object + /// + public Vector4 SelectedColor = new(125, 75, 75, 255); + + /// + /// Color of one file object + /// This should be a lighter color, as these elements are being printed, but are not selectable + /// + public Vector4 LightFileColor = new(125, 125, 125, 255); + + /// + /// Generate a new ImGuiFolderPicker instance + /// + /// Starting path, defaults to + public ImGuiFolderPicker(DirectoryInfo? startingPath = null) + { + _folderPickerCount++; + + if (startingPath == null || !startingPath.Exists) + { + startingPath = new DirectoryInfo(AppContext.BaseDirectory); + } + + RootFolder = startingPath; + CurrentOpenFolder = startingPath; + StartingFolder = startingPath; + SelectedFolder = startingPath; + } + + + public virtual unsafe void Draw(ref bool folderPickerOpen) + { + IsOpen = folderPickerOpen; + if (!folderPickerOpen) return; + + // close on ESC + if (ImGui.IsKeyReleased(ImGuiKey.Escape)) + { + OnCancel?.Invoke(this, EventArgs.Empty); + folderPickerOpen = false; + } + + if (DoFocusPicker) + ImGui.SetNextWindowFocus(); + + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, WindowPadding); + ImGui.PushStyleVar(ImGuiStyleVar.ChildBorderSize, 0); + ImGui.PushStyleColor(ImGuiCol.WindowBg, _windowBackgroundUint); + + // Begin window + ImGui.SetNextWindowSizeConstraints(new Vector2(500, 300), ImGui.GetWindowViewport().Size * 0.75f); + var allowResizeFlag = AllowFolderPickerResize ? ImGuiWindowFlags.None : ImGuiWindowFlags.NoResize; + ImGui.Begin(Id, ref folderPickerOpen, ImGuiWindowFlags.Modal | ImGuiWindowFlags.NoCollapse | allowResizeFlag); + + // draw navigation buttons and folder selection on the same line + DrawNavButtons(); + DrawFolderSelectionTextInput(); + + // draw drive and file selector window + ImGui.NewLine(); + ImGui.PushStyleColor(ImGuiCol.ChildBg, FolderSelectionMenuBackground.ToUintColor()); + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(10, 10)); + + DrawDriveSelector(); + DrawFolderSelector(ref folderPickerOpen); + + ImGui.PopStyleColor(); + ImGui.PopStyleVar(); + + // draw okay, cancel button + ImGui.NewLine(); + DrawFolderSelectorButtons(ref folderPickerOpen); + + ImGui.End(); + + + if (ShowNewFolderButton && IsNewFolderNameWindowOpen) + { + DrawNewFolderOverlay(CurrentOpenFolder); + } + + ImGui.PopStyleVar(2); + ImGui.PopStyleColor(); + + return; + } + + private unsafe void DrawNavButtons() + { + if ((IntPtr)SymbolsFontPtr.NativePtr != IntPtr.Zero) + ImGui.PushFont(SymbolsFontPtr); + + ImGui.BeginGroup(); + + var parentFolderButtonSize = ImGui.CalcTextSize(ParentFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var backButtonSize = ImGui.CalcTextSize(ParentFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var newFolderButtonSize = ImGui.CalcTextSize(NewFolderButtonTxt) + ImGui.GetStyle().FramePadding * 2; + + parentFolderButtonSize += new Vector2(5, 0); // add a little offset as the arrows aren't wide enough + backButtonSize += new Vector2(5, 0); // add a little offset as the arrows aren't wide enough + + if (ImGui.Button($"{ParentFolderTxt}##{_folderPickerCount}", parentFolderButtonSize)) + { + if (CurrentOpenFolder.Exists && CurrentOpenFolder.Parent != null) + { + LastOpenendFolders.Push(CurrentOpenFolder); + CurrentOpenFolder = CurrentOpenFolder.Parent; + } + } + ImGui.SameLine(); + + if (LastOpenendFolders.Count != 0) + { + if (ImGui.Button($"{BackTxt}##{_folderPickerCount}", backButtonSize)) + { + + var lastFolder = LastOpenendFolders.Pop(); + if (lastFolder.Exists) + { + CurrentOpenFolder = lastFolder; + } + } + } + else + { + ImGui.BeginDisabled(); + ImGui.Button($"{BackTxt}##{_folderPickerCount}", backButtonSize); + ImGui.EndDisabled(); + } + + if (ShowNewFolderButton) + { + ImGui.SameLine(); + if (ImGui.Button($"{NewFolderButtonTxt}##{_folderPickerCount}", newFolderButtonSize)) + { + _isNewFolderNameWindowOpen = true; + } + } + + if ((IntPtr)SymbolsFontPtr.NativePtr != IntPtr.Zero) + ImGui.PopFont(); + + ImGui.EndGroup(); + } + + private unsafe void DrawFolderSelectionTextInput() + { + // Folder Selection + var currentFolder = Environment.ExpandEnvironmentVariables(CurrentOpenFolder.FullName); + + ImGui.SameLine(); + // occupy the max available space, minus the label text length + ImGui.SetNextItemWidth(-ImGui.CalcTextSize(FolderLabelTxt).X); + ImGui.InputTextWithHint($"{FolderLabelTxt}##{_folderPickerCount}", PathToFolderTxt, ref currentFolder, 4098, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => + { + var arr = currentFolder.ToCharArray(); + + if (x->SelectionStart < x->SelectionEnd && x->SelectionStart >= 0 && x->SelectionEnd <= arr.Length) + { + var selectedText = arr[x->SelectionStart..x->SelectionEnd]; + if (selectedText != null) + ImGuiInputImp.CurrentlySelectedText = new string(selectedText); + } + + return 0; + }); + + var envCurrentFolder = Environment.ExpandEnvironmentVariables(currentFolder); + + if (Directory.Exists(envCurrentFolder)) + { + CurrentOpenFolder = new DirectoryInfo(envCurrentFolder); + CurrentlySelectedFolder = new DirectoryInfo(envCurrentFolder); + } + else + { + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); + ImGui.BeginTooltip(); + ImGui.TextColored(WarningTextColor, FolderNotFoundTxt); + ImGui.EndTooltip(); + ImGui.PopStyleVar(); + } + } + + private void DrawDriveSelector() + { + // take all space in y, however shrink in y in item height + standard padding + WindowPadding + var offsetFromBottom = ImGui.CalcTextSize(PickedFolderTxt) + ImGui.GetStyle().FramePadding * 2 + ImGui.GetStyle().WindowPadding * 2; + var driveSelectionWidth = ImGui.GetWindowSize().X * 0.25f; // 25% of windowSize.x + + ImGui.BeginChild($"DriveSelection##{_folderPickerCount}", new Vector2(driveSelectionWidth, -offsetFromBottom.Y), false, ImGuiWindowFlags.AlwaysUseWindowPadding | ImGuiWindowFlags.AlwaysAutoResize); + // Drive Selection + var driveCount = 0; + foreach (var drive in DriveInfo.GetDrives()) + { + if (drive.IsReady && (drive.DriveType == DriveType.Fixed || drive.DriveType == DriveType.Removable)) + { + if (ImGui.Selectable($"{drive.Name} {drive.DriveType}##{_folderPickerCount}")) + { + RootFolder = new DirectoryInfo(drive.Name); + LastOpenendFolders.Push(CurrentOpenFolder); + CurrentOpenFolder = new DirectoryInfo(drive.Name); + } + driveCount++; + } + } + ImGui.EndChild(); + } + + private void DrawFolderSelector(ref bool filePickerOpen) + { + + ImGui.SameLine(); + // take all space in y, however shrink in y in item height + standard padding + WindowPadding + var offsetFromBottom = ImGui.CalcTextSize(PickedFolderTxt) + ImGui.GetStyle().FramePadding * 2 + ImGui.GetStyle().WindowPadding * 2; + if (ImGui.BeginChild($"#FolderBrowser##{_folderPickerCount}", new Vector2(-1, -offsetFromBottom.Y), false, ImGuiWindowFlags.AlwaysUseWindowPadding | ImGuiWindowFlags.AlwaysAutoResize | ImGuiWindowFlags.HorizontalScrollbar)) + { + var fileSystemEntries = GetFileSystemEntries(CurrentOpenFolder.FullName); + foreach (var fse in fileSystemEntries) + { + var name = fse.Name; + + if (fse.Attributes.HasFlag(FileAttributes.Directory)) + { + ImGui.PushStyleColor(ImGuiCol.Text, FolderColor.ToUintColor()); + ImGui.PushStyleColor(ImGuiCol.Header, SelectedColor.ToUintColor()); + if (ImGui.Selectable(name + "/", CurrentlySelectedFolder?.Name == name, ImGuiSelectableFlags.DontClosePopups | ImGuiSelectableFlags.AllowDoubleClick)) + { + if (ImGui.IsMouseDoubleClicked(0)) + { + if (ImGui.GetIO().WantCaptureMouse) + { + CurrentlySelectedFolder = new DirectoryInfo(fse.FullName); + LastOpenendFolders.Push(CurrentOpenFolder); + CurrentOpenFolder = new DirectoryInfo(fse.FullName); + } + } + } + ImGui.PopStyleColor(); + ImGui.PopStyleColor(); + } + else + { + // just print the files, but with lighter color + ImGui.PushStyleColor(ImGuiCol.Text, LightFileColor.ToUintColor()); + ImGui.Selectable(name, false, ImGuiSelectableFlags.DontClosePopups); + ImGui.PopStyleColor(); + } + } + + } + ImGui.EndChild(); + } + + private void DrawFolderSelectorButtons(ref bool filePickerOpen) + { + var pickedFileButtonSize = ImGui.CalcTextSize(PickedFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var cancelFileButtonSize = ImGui.CalcTextSize(CancelFolderOpenTxt) + ImGui.GetStyle().FramePadding * 2; + + ImGui.BeginChild($"FolderSelector##{_folderPickerCount}", new Vector2(-1, -1), false, ImGuiWindowFlags.AlwaysAutoResize); + + // take all available window space minus the minus both buttons + // push buttons therefore to the right + var dummyMaxLength = ImGui.GetWindowSize().X - (ImGui.GetStyle().ItemInnerSpacing.X * 4 + pickedFileButtonSize.X + cancelFileButtonSize.X); + ImGui.Dummy(new Vector2(dummyMaxLength, -1)); + if (CurrentlySelectedFolder != null && CurrentlySelectedFolder.Exists) + { + ImGui.SameLine(); + + if (ImGui.Button($"{PickedFolderTxt}##{_folderPickerCount}", pickedFileButtonSize) || + (ImGui.IsKeyReleased(ImGuiKey.Enter) && !IsNewFolderNameWindowOpen)) + { + if (CurrentlySelectedFolder != null) + OnPicked?.Invoke(this, CurrentlySelectedFolder); + else + OnPicked?.Invoke(this, CurrentOpenFolder); + filePickerOpen = false; + } + } + else + { + ImGui.SameLine(); + ImGui.BeginDisabled(); + ImGui.Button(PickedFolderTxt, pickedFileButtonSize); + ImGui.EndDisabled(); + } + + ImGui.SameLine(); + if (ImGui.Button($"{CancelFolderOpenTxt}##{_folderPickerCount}", cancelFileButtonSize)) + { + OnCancel?.Invoke(this, EventArgs.Empty); + filePickerOpen = false; + } + + ImGui.EndChild(); + } + + private unsafe void DrawNewFolderOverlay(DirectoryInfo currentFolder) + { + ImGui.SetNextWindowFocus(); + // Calculate min height with button size + var createFolderButtonSize = ImGui.CalcTextSize(CreateFolderTxt) + ImGui.GetStyle().FramePadding * 2; + var minWindowHeight = createFolderButtonSize.Y + ImGui.GetStyle().WindowPadding.Y * 4; + var minWindowLength = createFolderButtonSize.X + ImGui.CalcTextSize(CreateNewFolderHintTxt).X + ImGui.GetStyle().FramePadding.X * 4 + ImGui.GetStyle().ItemSpacing.X * 4; + ImGui.SetNextWindowSizeConstraints(new Vector2(minWindowLength, minWindowHeight), new Vector2(ImGui.GetWindowViewport().Size.X * 0.5f, minWindowHeight)); + ImGui.SetNextItemWidth(minWindowLength + ImGui.GetStyle().WindowPadding.X); + + var allowResizeFlag = AllowNewFolderResize ? ImGuiWindowFlags.None : ImGuiWindowFlags.NoResize; + ImGui.Begin($"{CreateNewFolderTxt}##{_folderPickerCount}", ref _isNewFolderNameWindowOpen, ImGuiWindowFlags.Modal | ImGuiWindowFlags.NoCollapse | allowResizeFlag); + + // take the full width minus the button size + ImGui.SetNextItemWidth(-createFolderButtonSize.X); + ImGui.InputTextWithHint($"", $"{CreateNewFolderHintTxt}", ref _newFolderName, 4096, ImGuiInputTextFlags.AutoSelectAll | ImGuiInputTextFlags.CallbackAlways, (x) => + { + var arr = _newFolderName.ToCharArray(); + + if (x->SelectionStart < x->SelectionEnd && x->SelectionStart >= 0 && x->SelectionEnd <= arr.Length) + { + var selectedText = arr[x->SelectionStart..x->SelectionEnd]; + if (selectedText != null) + ImGuiInputImp.CurrentlySelectedText = new string(selectedText); + } + + return 0; + }); + ImGui.SameLine(); + + if (ImGui.Button($"{CreateFolderTxt}", createFolderButtonSize) || + ImGui.IsKeyReleased(ImGuiKey.Enter)) + { + if (!string.IsNullOrEmpty(_newFolderName)) + { + var folderName = string.Empty; + try + { + if (Path.IsPathRooted(_newFolderName)) + { + folderName = _newFolderName; + Directory.CreateDirectory(_newFolderName); + } + else + { + folderName = Path.Combine(currentFolder.FullName, _newFolderName); + Directory.CreateDirectory(folderName); + } + } + catch (Exception ex) + { + _createFolderException = ex; + return; + + } + + // open new folder + CurrentlySelectedFolder = new DirectoryInfo(folderName); + LastOpenendFolders.Push(CurrentOpenFolder); + CurrentOpenFolder = new DirectoryInfo(folderName); + } + IsNewFolderNameWindowOpen = false; + } + + // display a possible exception during folder creation as a tooltip text + if (_createFolderException != null) + { + ImGui.PushStyleVar(ImGuiStyleVar.WindowPadding, new Vector2(5, 5)); + var size = ImGui.CalcTextSize(_createFolderException?.Message); + ImGui.SetNextWindowSize(new Vector2(size.X / 4, -1)); + ImGui.BeginTooltip(); + ImGui.PushStyleColor(ImGuiCol.Text, WarningTextColor); + ImGui.TextWrapped(_createFolderException?.Message); + ImGui.PopStyleColor(); + ImGui.EndTooltip(); + ImGui.PopStyleVar(); + } + + ImGui.End(); + } + + /// + /// We differentiate between files and folders, as we want to print the folders first + /// If we collect everything in one list all files and folders are being sorted alphabetically + /// + /// + /// + private static List GetFileSystemEntries(string fullName) + { + var folders = new List(); + var files = new List(); + + foreach (var f in Directory.GetFileSystemEntries(fullName, "")) + { + var attr = File.GetAttributes(f); + // skip unaccessible files and folders + if (attr.HasFlag(FileAttributes.Encrypted) + || attr.HasFlag(FileAttributes.Hidden) + || attr.HasFlag(FileAttributes.System) + || attr.HasFlag(FileAttributes.Temporary)) + continue; + + if (attr.HasFlag(FileAttributes.Directory)) + { + folders.Add(new DirectoryInfo(f)); + } + else + { + var fse = new FileInfo(f); + files.Add(fse); + } + } + + var ret = new List(folders); + ret.AddRange(files); + return ret; + + } + } +} \ No newline at end of file diff --git a/src/Math/Core/AABBd.cs b/src/Math/Core/AABBd.cs index bd351cdd8..74518c960 100644 --- a/src/Math/Core/AABBd.cs +++ b/src/Math/Core/AABBd.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Runtime.InteropServices; @@ -7,6 +8,7 @@ namespace Fusee.Math.Core /// /// Represents an axis aligned bounding box. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct AABBd @@ -14,11 +16,13 @@ public struct AABBd /// /// The minimum values of the axis aligned bounding box in x, y and z direction /// + [JsonProperty(PropertyName = "Min")] [ProtoMember(1)] public double3 min; /// /// The maximum values of the axis aligned bounding box in x, y and z direction /// + [JsonProperty(PropertyName = "Max")] [ProtoMember(2)] public double3 max; /// @@ -250,6 +254,31 @@ public bool IntersectRay(RayD ray) return tmax >= M.Max(tmin, 0.0); } + /// + /// Returns the closest point to a point p, that lies on the surface of the . + /// + /// The reference point. + /// + public double3 ClosestPoint(double3 point) + { + double3 d = point - Center; + double3 q = Center; + + for (int i = 0; i < 3; i++) + { + var axis = i == 0 ? double3.UnitX : i == 1 ? double3.UnitY : double3.UnitZ; + var halfLength = i == 0 ? Size.x / 2 : i == 1 ? Size.y / 2 : Size.z / 2; + double dist = double3.Dot(d, axis); + + if (dist > halfLength) dist = halfLength; + if (dist < -halfLength) dist = -halfLength; + + q += dist * axis; + } + + return q; + } + /// /// Check if two AABBs intersect each other /// diff --git a/src/Math/Core/AABBf.cs b/src/Math/Core/AABBf.cs index 764a1ac45..2322e81cd 100644 --- a/src/Math/Core/AABBf.cs +++ b/src/Math/Core/AABBf.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Runtime.InteropServices; @@ -7,6 +8,7 @@ namespace Fusee.Math.Core /// /// Represents an axis aligned bounding box. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct AABBf @@ -14,11 +16,13 @@ public struct AABBf /// /// The minimum values of the axis aligned bounding box in x, y and z direction /// + [JsonProperty(PropertyName = "Min")] [ProtoMember(1)] public float3 min; /// /// The maximum values of the axis aligned bounding box in x, y and z direction /// + [JsonProperty(PropertyName = "Max")] [ProtoMember(2)] public float3 max; /// @@ -229,8 +233,8 @@ public bool IntersectRay(RayF ray) if (this.Intersects(ray.Origin)) return true; - float t1 = (min[0] - ray.Origin[0]) * ray.Inverse[0]; - float t2 = (max[0] - ray.Origin[0]) * ray.Inverse[0]; + float t1 = (min.x - ray.Origin.x) * ray.Inverse.x; + float t2 = (max.x - ray.Origin.x) * ray.Inverse.x; float tmin = M.Min(t1, t2); float tmax = M.Max(t1, t2); @@ -250,6 +254,32 @@ public bool IntersectRay(RayF ray) return tmax >= M.Max(tmin, 0.0); } + /// + /// Returns the closest point to a point p, that lies on the surface of the . + /// + /// The reference point. + /// + public float3 ClosestPoint(float3 point) + { + float3 d = point - Center; + float3 q = Center; + + for (int i = 0; i < 3; i++) + { + var axis = i == 0 ? float3.UnitX : i == 1 ? float3.UnitY : float3.UnitZ; + var halfLength = i == 0 ? Size.x / 2 : i == 1 ? Size.y / 2 : Size.z / 2; + float dist = float3.Dot(d, axis); + + if (dist > halfLength) dist = halfLength; + if (dist < -halfLength) dist = -halfLength; + + q += dist * axis; + } + + return q; + } + + /// /// Check if two AABBs intersect each other /// @@ -299,7 +329,7 @@ public static bool Intersects(AABBf aabb, float3 point) /// public override bool Equals(object? obj) { - if (obj?.GetType() != typeof(AABBf)) throw new ArgumentException($"{obj} is not of Type 'AABBf'."); + if (obj?.GetType() != typeof(AABBf)) return false; var other = (AABBf)obj; return max.Equals(other.max) && min.Equals(other.min); diff --git a/src/Math/Core/Curve.cs b/src/Math/Core/Curve.cs index 3d251751c..5522f81fa 100644 --- a/src/Math/Core/Curve.cs +++ b/src/Math/Core/Curve.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Linq; @@ -7,11 +8,13 @@ namespace Fusee.Math.Core /// /// Represents a curve, using a list of CurveParts. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class Curve { /// /// The parts forming the curve. /// + [JsonProperty(PropertyName = "CurveParts")] public IList CurveParts = new List(); /// @@ -97,21 +100,25 @@ public IEnumerable CalcAdaptivePolyline(float acreage) /// /// Represents a open or closed part of a curve, using a list of CurveSegments and its starting point. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class CurvePart { /// /// A CurvePart can be closed or open. /// + [JsonProperty(PropertyName = "IsClosed")] public bool IsClosed; /// /// The starting point of the CurvePart. /// + [JsonProperty(PropertyName = "StartPoint")] public float3 StartPoint; /// /// The segments making up the CurvePart. /// + [JsonProperty(PropertyName = "CurveSegments")] public IList CurveSegments = new List(); /// @@ -225,11 +232,13 @@ public IEnumerable CalcAdaptivePolyline(float acreage) /// A CurveSgment does not know its own start point. For the first CurveSegment in a sequence the start point is saved in the CurvePart belonging to the segment. /// The start point for the CurveSegment with index i is the last vertex in the CurveSegent[i-1]'s list of vertices. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public abstract class CurveSegment { /// ///The vertices of a CurveSegment represented by float3s. /// + [JsonProperty(PropertyName = "Vertices")] public IList Vertices = new List(); /// diff --git a/src/Math/Core/Eigen.cs b/src/Math/Core/Eigen.cs index 7a8bc3fec..9ec252342 100644 --- a/src/Math/Core/Eigen.cs +++ b/src/Math/Core/Eigen.cs @@ -1,20 +1,26 @@ -using System.Linq; +using Newtonsoft.Json; +using System; +using System.Diagnostics.Tracing; +using System.Linq; namespace Fusee.Math.Core { /// /// Eigen data structure with values and vectors in double precision. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public struct Eigen { /// /// Eigen values. /// + [JsonProperty(PropertyName = "Values")] public double[] Values; /// /// Eigen vectors. /// + [JsonProperty(PropertyName = "Vectors")] public double3[] Vectors; /// @@ -69,6 +75,7 @@ public Eigen(double3[] vals) /// Creates a new instance. /// /// + /// Throws if calculation fails due to numeric instability public Eigen(float3[] vals) { var valsD = vals.Select(val => (double3)val).ToArray(); @@ -78,6 +85,11 @@ public Eigen(float3[] vals) Values = new double[3]; Vectors = new double3[3]; CalculateVectorsAndValues(covarianceMatrix); + + if(Values.Any(x => double.IsNaN(x)) || Vectors.Any(x => x.ToArray().Any(y => double.IsNaN(y)))) + { + throw new ArithmeticException($"Calculation of Eigen values failed. NaN as values. Values: [{string.Join(",", Values)}] and Vectors: [{string.Join(",", Vectors.ToArray())}]"); + } } /// diff --git a/src/Math/Core/FrustumD.cs b/src/Math/Core/FrustumD.cs index 4608e93d4..7f180af55 100644 --- a/src/Math/Core/FrustumD.cs +++ b/src/Math/Core/FrustumD.cs @@ -1,4 +1,5 @@  +using Newtonsoft.Json; using System.Collections.Generic; namespace Fusee.Math.Core @@ -6,36 +7,43 @@ namespace Fusee.Math.Core /// /// Describes a frustum by using six s. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class FrustumD { /// /// The near plane of the frustum. /// + [JsonProperty(PropertyName = "Near")] public PlaneD Near { get; private set; } /// /// The far plane of the frustum. /// + [JsonProperty(PropertyName = "Far")] public PlaneD Far { get; private set; } /// /// The left plane of the frustum. /// + [JsonProperty(PropertyName = "Left")] public PlaneD Left { get; private set; } /// /// The right plane of the frustum. /// + [JsonProperty(PropertyName = "Right")] public PlaneD Right { get; private set; } /// /// The top plane of the frustum. /// + [JsonProperty(PropertyName = "Top")] public PlaneD Top { get; private set; } /// /// The bottom plane of the frustum. /// + [JsonProperty(PropertyName = "Bottom")] public PlaneD Bottom { get; private set; } /// diff --git a/src/Math/Core/FrustumF.cs b/src/Math/Core/FrustumF.cs index c1c500f4d..f0b1f289e 100644 --- a/src/Math/Core/FrustumF.cs +++ b/src/Math/Core/FrustumF.cs @@ -1,4 +1,5 @@  +using Newtonsoft.Json; using System.Collections.Generic; namespace Fusee.Math.Core @@ -6,36 +7,43 @@ namespace Fusee.Math.Core /// /// Describes a frustum by using six s. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class FrustumF { /// /// The near plane of the frustum. /// + [JsonProperty(PropertyName = "Near")] public PlaneF Near { get; private set; } /// /// The far plane of the frustum. /// + [JsonProperty(PropertyName = "Far")] public PlaneF Far { get; private set; } /// /// The left plane of the frustum. /// + [JsonProperty(PropertyName = "Left")] public PlaneF Left { get; private set; } /// /// The right plane of the frustum. /// + [JsonProperty(PropertyName = "Right")] public PlaneF Right { get; private set; } /// /// The top plane of the frustum. /// + [JsonProperty(PropertyName = "Top")] public PlaneF Top { get; private set; } /// /// The bottom plane of the frustum. /// + [JsonProperty(PropertyName = "Bottom")] public PlaneF Bottom { get; private set; } /// diff --git a/src/Math/Core/Fusee.Math.Core.csproj b/src/Math/Core/Fusee.Math.Core.csproj index 7dac979e0..620390a2f 100644 --- a/src/Math/Core/Fusee.Math.Core.csproj +++ b/src/Math/Core/Fusee.Math.Core.csproj @@ -1,18 +1,23 @@  - + netstandard2.1;net7.0 $(OutputPath)\$(RootNamespace).xml true Core Math implementation for the Fusee Project - + enable true - + + - + + + + + \ No newline at end of file diff --git a/src/Math/Core/MathNetExtensions.cs b/src/Math/Core/MathNetExtensions.cs new file mode 100644 index 000000000..b86862e84 --- /dev/null +++ b/src/Math/Core/MathNetExtensions.cs @@ -0,0 +1,240 @@ +#if MathNet + +using System; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Fusee.Math.Core +{ + /// + /// Vector extensions for MathNet compatibility + /// + public static class MathNetVectorExtension + { + #region FuseeToMathNet + + public static MathNet.Numerics.LinearAlgebra.Single.DenseVector ToMathNetSingleVector(this float3 f3) + + { + float[] f = new float[] { f3.x, f3.y, f3.z }; + return new MathNet.Numerics.LinearAlgebra.Single.DenseVector(f); + } + + public static MathNet.Numerics.LinearAlgebra.Single.DenseVector ToMathNetSingleVector(this double3 d3) + { + return ToMathNetSingleVector((float3)d3); + } + + public static MathNet.Numerics.LinearAlgebra.Double.DenseVector ToMathNetDoubleVector(this double3 d3) + { + double[] d = new double[] { d3.x, d3.y, d3.z }; + + return new MathNet.Numerics.LinearAlgebra.Double.DenseVector(d); + } + + public static MathNet.Numerics.LinearAlgebra.Double.DenseVector ToMathNetDoubleVector(this float3 f3) + { + return ToMathNetDoubleVector(f3); + } + + #endregion FuseeToMathNet + + #region MathNetToFusee + + public static float3 ToFuseeSingleVector(this MathNet.Numerics.LinearAlgebra.Single.DenseVector sdv) + { + if (sdv.Values.Length != 3) + throw new ArgumentException(); + + var f = sdv.Storage.AsArray(); + + return new float3(f[0], f[1], f[2]); + } + + public static float3 ToFuseeSingleVector(this MathNet.Numerics.LinearAlgebra.Double.DenseVector ddv) + { + if (ddv.Values.Length != 3) + throw new ArgumentException(); + + var d = ddv.Storage.AsArray(); + + return new float3((float)d[0], (float)d[1], (float)d[2]); + } + + public static double3 ToFuseeDoubleVector(this MathNet.Numerics.LinearAlgebra.Single.DenseVector sdv) + { + if (sdv.Values.Length != 3) + throw new ArgumentException(); + + var f = sdv.Storage.AsArray(); + + return new double3(f[0], f[1], f[2]); + } + + public static double3 ToFuseeDoubleVector(this MathNet.Numerics.LinearAlgebra.Double.DenseVector ddv) + { + if (ddv.Values.Length != 3) + throw new ArgumentException(); + + var d = ddv.Storage.AsArray(); + + return new double3(d[0], d[1], d[2]); + } + + #endregion MathNetToFusee + } + + /// + /// Matrix extensions for MathNet compatibility + /// + public static class MathNetMatrixExtension + { + #region FuseeToMathNet + + public static MathNet.Numerics.LinearAlgebra.Single.DenseMatrix ToMathNetSingleMatrix(this float4x4 f4x4) + { + float[] f = new float[] { f4x4.M11, f4x4.M21, f4x4.M31, f4x4.M41, + f4x4.M12, f4x4.M22, f4x4.M32, f4x4.M42, + f4x4.M13, f4x4.M23, f4x4.M33, f4x4.M43, + f4x4.M14, f4x4.M24, f4x4.M34, f4x4.M44 + }; + + return new MathNet.Numerics.LinearAlgebra.Single.DenseMatrix(4, 4, f); + } + + public static MathNet.Numerics.LinearAlgebra.Single.DenseMatrix ToMathNetSingleMatrix(this double4x4 d4x4) + { + return ToMathNetSingleMatrix((float4x4)d4x4); + } + + public static MathNet.Numerics.LinearAlgebra.Double.DenseMatrix ToMathNetDoubleMatrix(this double4x4 d4x4) + { + double[] f = new double[] { d4x4.M11, d4x4.M21, d4x4.M31, d4x4.M41, + d4x4.M12, d4x4.M22, d4x4.M32, d4x4.M42, + d4x4.M13, d4x4.M23, d4x4.M33, d4x4.M43, + d4x4.M14, d4x4.M24, d4x4.M34, d4x4.M44 + }; + + return new MathNet.Numerics.LinearAlgebra.Double.DenseMatrix(4, 4, f); + } + + public static MathNet.Numerics.LinearAlgebra.Double.DenseMatrix ToMathNetDoubleMatrix(this float4x4 f4x4) + { + return ToMathNetDoubleMatrix((double4x4)f4x4); + } + + #endregion FuseeToMathNet + + #region MathNetToFusee + + public static float4x4 ToFuseeSingleMatrix(this MathNet.Numerics.LinearAlgebra.Single.DenseMatrix sdm) + { + if (sdm.Values.Length != 16 || sdm.ColumnCount != 4 || sdm.RowCount != 4) + throw new ArgumentException(); + + return new float4x4 + { + M11 = sdm.Values[0], + M21 = sdm.Values[1], + M31 = sdm.Values[2], + M41 = sdm.Values[3], + M12 = sdm.Values[4], + M22 = sdm.Values[5], + M32 = sdm.Values[6], + M42 = sdm.Values[7], + M13 = sdm.Values[8], + M23 = sdm.Values[9], + M33 = sdm.Values[10], + M43 = sdm.Values[11], + M14 = sdm.Values[12], + M24 = sdm.Values[13], + M34 = sdm.Values[14], + M44 = sdm.Values[15] + }; + } + + public static float4x4 ToFuseeSingleMatrix(this MathNet.Numerics.LinearAlgebra.Double.DenseMatrix ddm) + { + if (ddm.Values.Length != 16 || ddm.ColumnCount != 4 || ddm.RowCount != 4) + throw new ArgumentException(); + + return new float4x4 + { + M11 = (float)ddm.Values[0], + M21 = (float)ddm.Values[1], + M31 = (float)ddm.Values[2], + M41 = (float)ddm.Values[3], + M12 = (float)ddm.Values[4], + M22 = (float)ddm.Values[5], + M32 = (float)ddm.Values[6], + M42 = (float)ddm.Values[7], + M13 = (float)ddm.Values[8], + M23 = (float)ddm.Values[9], + M33 = (float)ddm.Values[10], + M43 = (float)ddm.Values[11], + M14 = (float)ddm.Values[12], + M24 = (float)ddm.Values[13], + M34 = (float)ddm.Values[14], + M44 = (float)ddm.Values[15] + }; + } + + public static double4x4 ToFuseeDoubleMatrix(this MathNet.Numerics.LinearAlgebra.Single.DenseMatrix sdm) + { + if (sdm.Values.Length != 16 || sdm.ColumnCount != 4 || sdm.RowCount != 4) + throw new ArgumentException(); + + return new double4x4 + { + M11 = sdm.Values[0], + M21 = sdm.Values[1], + M31 = sdm.Values[2], + M41 = sdm.Values[3], + M12 = sdm.Values[4], + M22 = sdm.Values[5], + M32 = sdm.Values[6], + M42 = sdm.Values[7], + M13 = sdm.Values[8], + M23 = sdm.Values[9], + M33 = sdm.Values[10], + M43 = sdm.Values[11], + M14 = sdm.Values[12], + M24 = sdm.Values[13], + M34 = sdm.Values[14], + M44 = sdm.Values[15] + }; + } + + public static double4x4 ToFuseeDoubleMatrix(this MathNet.Numerics.LinearAlgebra.Double.DenseMatrix ddm) + { + if (ddm.Values.Length != 16 || ddm.ColumnCount != 4 || ddm.RowCount != 4) + throw new ArgumentException(); + + return new double4x4 + { + M11 = ddm.Values[0], + M21 = ddm.Values[1], + M31 = ddm.Values[2], + M41 = ddm.Values[3], + M12 = ddm.Values[4], + M22 = ddm.Values[5], + M32 = ddm.Values[6], + M42 = ddm.Values[7], + M13 = ddm.Values[8], + M23 = ddm.Values[9], + M33 = ddm.Values[10], + M43 = ddm.Values[11], + M14 = ddm.Values[12], + M24 = ddm.Values[13], + M34 = ddm.Values[14], + M44 = ddm.Values[15] + }; + } + + #endregion MathNetToFusee + } +} + +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + +#endif \ No newline at end of file diff --git a/src/Math/Core/MinMaxRect.cs b/src/Math/Core/MinMaxRect.cs index ee1888955..b3e0ef3ba 100644 --- a/src/Math/Core/MinMaxRect.cs +++ b/src/Math/Core/MinMaxRect.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Globalization; namespace Fusee.Math.Core @@ -7,16 +8,19 @@ namespace Fusee.Math.Core /// Class containing an axis aligned (two-dimensional) rectangle specified by its minimum (lower-left) and maximum (upper-right) /// points in 2d space. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public struct MinMaxRect { /// /// Returns the minimum (lower-left corner) as a float2 vector. - /// + /// + [JsonProperty(PropertyName = "Min")] public float2 Min; /// /// Returns the maximum (upper-right corner) as a float2 vector. /// + [JsonProperty(PropertyName = "Max")] public float2 Max; /// diff --git a/src/Math/Core/OBBd.cs b/src/Math/Core/OBBd.cs index 453615149..d491fab30 100644 --- a/src/Math/Core/OBBd.cs +++ b/src/Math/Core/OBBd.cs @@ -1,4 +1,5 @@ -using ProtoBuf; +using Newtonsoft.Json; +using ProtoBuf; using System.Linq; using System.Runtime.InteropServices; @@ -7,6 +8,7 @@ namespace Fusee.Math.Core /// /// Represents an oriented bounding box. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct OBBd @@ -14,27 +16,37 @@ public struct OBBd /// /// The minimum values of the oriented bounding box in x, y and z direction /// - [ProtoMember(1)] public double3 Min; + [JsonProperty(PropertyName = "Min")] + [ProtoMember(1)] + public double3 Min; /// /// The maximum values of the oriented bounding box in x, y and z direction /// - [ProtoMember(2)] public double3 Max; + [JsonProperty(PropertyName = "Max")] + [ProtoMember(2)] + public double3 Max; /// /// The rotation of the oriented bounding box /// - [ProtoMember(3)] public double4x4 Rotation; + [JsonProperty(PropertyName = "Rotation")] + [ProtoMember(3)] + public double4x4 Rotation; /// /// The translation of the oriented bounding box /// - [ProtoMember(4)] public double3 Translation; + [JsonProperty(PropertyName = "Translation")] + [ProtoMember(4)] + public double3 Translation; /// /// Returns the with, height and depth of the box in x, y and z /// - [ProtoMember(5)] public double3 Size; + [JsonProperty(PropertyName = "Size")] + [ProtoMember(5)] + public double3 Size; /// /// Returns the center of the bounding box diff --git a/src/Math/Core/OBBf.cs b/src/Math/Core/OBBf.cs index af4ec4c16..bfa4b8aa1 100644 --- a/src/Math/Core/OBBf.cs +++ b/src/Math/Core/OBBf.cs @@ -1,4 +1,5 @@ -using ProtoBuf; +using Newtonsoft.Json; +using ProtoBuf; using System.Linq; using System.Runtime.InteropServices; @@ -7,6 +8,7 @@ namespace Fusee.Math.Core /// /// Represents an oriented bounding box. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct OBBf @@ -14,27 +16,37 @@ public struct OBBf /// /// The minimum values of the oriented bounding box in x, y and z direction /// - [ProtoMember(1)] public float3 Min; + [JsonProperty(PropertyName = "Min")] + [ProtoMember(1)] + public float3 Min; /// /// The maximum values of the oriented bounding box in x, y and z direction /// - [ProtoMember(2)] public float3 Max; + [JsonProperty(PropertyName = "Max")] + [ProtoMember(2)] + public float3 Max; /// /// The rotation of the oriented bounding box /// - [ProtoMember(3)] public float4x4 Rotation; + [JsonProperty(PropertyName = "Rotation")] + [ProtoMember(3)] + public float4x4 Rotation; /// /// The translation of the oriented bounding box /// - [ProtoMember(4)] public float3 Translation; + [JsonProperty(PropertyName = "Translation")] + [ProtoMember(4)] + public float3 Translation; /// /// Returns the with, height and depth of the box in x, y and z /// - [ProtoMember(5)] public float3 Size; + [JsonProperty(PropertyName = "Size")] + [ProtoMember(5)] + public float3 Size; /// /// Returns the center of the bounding box diff --git a/src/Math/Core/PlaneD.cs b/src/Math/Core/PlaneD.cs index 816ad89d3..12541b4aa 100644 --- a/src/Math/Core/PlaneD.cs +++ b/src/Math/Core/PlaneD.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; namespace Fusee.Math.Core { @@ -8,11 +9,13 @@ namespace Fusee.Math.Core /// The plane divides a space into two half-spaces.The direction plane's normal vector defines the "outer" or negative half-space. /// Points that lie in the positive half space of the plane do have a negative signed distance to the plane. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public struct PlaneD { /// /// The A plane coefficient. /// + [JsonProperty(PropertyName = "A")] public double A { get => _a; @@ -22,11 +25,12 @@ public double A _normal.x = _a; } } - private double _a; + private double _a = 0; /// /// The B plane coefficient. /// + [JsonProperty(PropertyName = "B")] public double B { get => _b; @@ -36,11 +40,12 @@ public double B _normal.y = _b; } } - private double _b; + private double _b = 0; /// /// The C plane coefficient. /// + [JsonProperty(PropertyName = "C")] public double C { get => _c; @@ -50,12 +55,13 @@ public double C _normal.z = _c; } } - private double _c; + private double _c = 0; /// /// The D plane coefficient. /// - public double D; + [JsonProperty(PropertyName = "D")] + public double D = 1; /// /// The plane's normal vector. May NOT be of unit length if the plane isn't normalized. @@ -70,7 +76,21 @@ public double3 Normal return _normal; } } - private double3 _normal; + private double3 _normal = double3.UnitZ; + + /// + /// Creates a plane from a normal and a point. + /// + /// The plane's normal vector. + /// A point on the plane. + public PlaneD(double3 normal, double3 point) + { + A = normal.x; + B = normal.y; + C = normal.z; + + D = A * point.x + B * point.y + C * point.z; + } /// /// Normalizes this plane. @@ -120,7 +140,7 @@ public double AngleBetween(PlaneD other) /// Test whether a intersects this plane. /// See: Ericson 2005, Real Time Collision Detection, p. 161 - 164 /// - /// The axis aligned bounding box. + /// The axis aligned bounding box. public bool Intersects(AABBd aabb) { var r = BoxExtendInNormalDirection(aabb); @@ -147,7 +167,7 @@ public bool Intersects(double3 center, double3 size) /// Test whether a intersects this plane. /// See: Ericson 2005, Real Time Collision Detection, p. 161 - 164 /// - /// The axis aligned bounding box. + /// The axis aligned bounding box. public bool Intersects(OBBd obb) { var r = BoxExtendInNormalDirection(obb); @@ -159,7 +179,7 @@ public bool Intersects(OBBd obb) /// /// Test whether a cuboid intersects this plane. /// See: Ericson 2005, Real Time Collision Detection, p. 161 - 164 - /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, + /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, /// because FUSEE defines a point with a negative signed distance to be inside. /// /// The center of the cuboid. @@ -184,7 +204,7 @@ public bool InsideOrIntersecting(double3 center, double3 size) /// /// Test whether a cuboid intersects this plane. /// See: Ericson 2005, Real Time Collision Detection, p. 161 - 164 - /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, + /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, /// because FUSEE defines a point with a negative signed distance to be inside. /// /// The center of the cuboid. @@ -209,10 +229,10 @@ public bool InsideOrIntersecting(double3 center, double size) /// /// Test whether a intersects this plane. /// See: Ericson 2005, Real Time Collision Detection, p. 161 - 164 - /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, + /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, /// because FUSEE defines a point with a negative signed distance to be inside. /// - /// The axis aligned bounding box. + /// The axis aligned bounding box. public bool InsideOrIntersecting(AABBd aabb) { var r = BoxExtendInNormalDirection(aabb); @@ -233,10 +253,10 @@ public bool InsideOrIntersecting(AABBd aabb) /// /// Test whether a intersects this plane. /// See: Ericson 2005, Real Time Collision Detection, p. 161 - 164 - /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, + /// CAREFUL: the definition whats completely inside and outside is flipped in comparison to Ericson, /// because FUSEE defines a point with a negative signed distance to be inside. /// - /// The object oriented bounding box. + /// The object oriented bounding box. public bool InsideOrIntersecting(OBBd obb) { var r = BoxExtendInNormalDirection(obb); @@ -254,7 +274,7 @@ public bool InsideOrIntersecting(OBBd obb) } /// - /// Calculates the projection interval radius of an cuboid onto line L(t) = cuboid.Center + t * plane.Normal (extend (radius) in direction of the plane normal). + /// Calculates the projection interval radius of an cuboid onto line L(t) = cuboid.Center + t * plane.Normal (extend (radius) in direction of the plane normal). /// The width, height and length of a cuboid. /// private double BoxExtendInNormalDirection(double3 size) @@ -264,7 +284,7 @@ private double BoxExtendInNormalDirection(double3 size) } /// - /// Calculates the projection interval radius of an cuboid onto line L(t) = cuboid.Center + t * plane.Normal (extend (radius) in direction of the plane normal). + /// Calculates the projection interval radius of an cuboid onto line L(t) = cuboid.Center + t * plane.Normal (extend (radius) in direction of the plane normal). /// The width, height and length of a cuboid. /// private double BoxExtendInNormalDirection(double size) @@ -274,7 +294,7 @@ private double BoxExtendInNormalDirection(double size) } /// - /// Calculates the projection interval radius of aabb onto line L(t) = aabb.Center + t * plane.Normal (extend (radius) in direction of the plane normal). + /// Calculates the projection interval radius of aabb onto line L(t) = aabb.Center + t * plane.Normal (extend (radius) in direction of the plane normal). /// The axis aligned bounding box. /// private double BoxExtendInNormalDirection(AABBd aabb) @@ -284,7 +304,7 @@ private double BoxExtendInNormalDirection(AABBd aabb) } /// - /// Calculates the projection interval radius of obb onto line L(t) = aabb.Center + t * plane.Normal (extend (radius) in direction of the plane normal). + /// Calculates the projection interval radius of obb onto line L(t) = aabb.Center + t * plane.Normal (extend (radius) in direction of the plane normal). /// The object oriented bounding box. /// private double BoxExtendInNormalDirection(OBBd obb) @@ -327,7 +347,7 @@ private double BoxExtendInNormalDirection(OBBd obb) /// Operator override for equality. /// /// The plane. - /// The scalar value. + /// The scalar value. public static bool operator ==(PlaneD left, PlaneD right) { return left.Equals(right); @@ -337,7 +357,7 @@ private double BoxExtendInNormalDirection(OBBd obb) /// Operator override for inequality. /// /// The plane. - /// The scalar value. + /// The scalar value. public static bool operator !=(PlaneD left, PlaneD right) { return !(left == right); @@ -362,7 +382,7 @@ public override bool Equals(object? obj) /// /// Generates a hash code for this plane. - /// + /// public override int GetHashCode() { return HashCode.Combine(A, B, C, D); diff --git a/src/Math/Core/PlaneF.cs b/src/Math/Core/PlaneF.cs index c69683499..bcb21cd0d 100644 --- a/src/Math/Core/PlaneF.cs +++ b/src/Math/Core/PlaneF.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; namespace Fusee.Math.Core { @@ -8,11 +9,13 @@ namespace Fusee.Math.Core /// The plane divides a space into two half-spaces.The direction plane's normal vector defines the "outer" or negative half-space. /// Points that lie in the positive half space of the plane do have a negative signed distance to the plane. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public struct PlaneF { /// /// The A plane coefficient. /// + [JsonProperty(PropertyName = "A")] public float A { get => _a; @@ -22,11 +25,12 @@ public float A _normal.x = _a; } } - private float _a; + private float _a = 0; /// /// The B plane coefficient. /// + [JsonProperty(PropertyName = "B")] public float B { get => _b; @@ -36,11 +40,12 @@ public float B _normal.y = _b; } } - private float _b; + private float _b = 0; /// /// The C plane coefficient. /// + [JsonProperty(PropertyName = "C")] public float C { get => _c; @@ -50,12 +55,13 @@ public float C _normal.z = _c; } } - private float _c; + private float _c = 1; /// /// The D plane coefficient. /// - public float D; + [JsonProperty(PropertyName = "D")] + public float D = 1; /// /// The plane's normal vector. May NOT be of unit length if the plane isn't normalized. @@ -70,7 +76,21 @@ public float3 Normal return _normal; } } - private float3 _normal; + private float3 _normal = float3.UnitZ; + + /// + /// Creates a plane from a normal and a point. + /// + /// The plane's normal vector. + /// A point on the plane. + public PlaneF(float3 normal, float3 point) + { + A = normal.x; + B = normal.y; + C = normal.z; + + D = A * point.x + B * point.y + C * point.z; + } /// /// Normalizes this plane. diff --git a/src/Math/Core/QuaternionD.cs b/src/Math/Core/QuaternionD.cs index 7feaeb8d2..93bdf46bf 100644 --- a/src/Math/Core/QuaternionD.cs +++ b/src/Math/Core/QuaternionD.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using System; using System.Globalization; using System.Runtime.InteropServices; @@ -7,12 +8,16 @@ namespace Fusee.Math.Core /// /// Represents a QuaternionD (single precision). /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] public struct QuaternionD : IEquatable { #region Fields + [JsonProperty(PropertyName = "XYZ")] private double3 _xyz; + + [JsonProperty(PropertyName = "W")] private double _w; #endregion Fields diff --git a/src/Math/Core/QuaternionF.cs b/src/Math/Core/QuaternionF.cs index 0d7965b36..df7f5b18f 100644 --- a/src/Math/Core/QuaternionF.cs +++ b/src/Math/Core/QuaternionF.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using System; using System.Globalization; using System.Runtime.InteropServices; @@ -7,12 +8,16 @@ namespace Fusee.Math.Core /// /// Represents a Quaternion (single precision). /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] public struct QuaternionF : IEquatable { #region Fields + [JsonProperty(PropertyName = "XYZ")] private float3 _xyz; + + [JsonProperty(PropertyName = "W")] private float _w; #endregion Fields diff --git a/src/Math/Core/Rayd.cs b/src/Math/Core/Rayd.cs index 886edc8b2..7c4eb2503 100644 --- a/src/Math/Core/Rayd.cs +++ b/src/Math/Core/Rayd.cs @@ -1,24 +1,56 @@ -namespace Fusee.Math.Core +using Newtonsoft.Json; + +namespace Fusee.Math.Core { /// /// Represents a ray with a given origin and direction. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public struct RayD { /// /// The point in world coordinates from which the ray originates. /// + [JsonProperty(PropertyName = "Origin")] public double3 Origin; + private double3 _direction; + /// /// The direction of the ray. /// - public double3 Direction { get; private set; } + [JsonProperty(PropertyName = "Direction")] + public double3 Direction + { + get { return _direction; } + set + { + _direction = double3.Normalize(value); + + _inverseDirty = true; + } + } + + private double3 _inverse; + private bool _inverseDirty; /// /// The inverse of the direction vector of the ray (1 / direction). /// - public double3 Inverse { get; private set; } + public double3 Inverse + { + get + { + if (_inverseDirty) + { + _inverse = new double3(1 / Direction.x, 1 / Direction.y, 1 / Direction.z); + + _inverseDirty = false; + } + + return _inverse; + } + } /// /// Create a new ray. @@ -28,8 +60,11 @@ public struct RayD public RayD(double3 origin_, double3 direction_) { Origin = origin_; - Direction = double3.Normalize(direction_); - Inverse = new double3(1 / Direction.x, 1 / Direction.y, 1 / Direction.z); + + _direction = double3.Normalize(direction_); + + _inverse = default; + _inverseDirty = true; } /// @@ -40,16 +75,17 @@ public RayD(double3 origin_, double3 direction_) /// The Projection Matrix of the rendered scene. public RayD(double2 pickPosClip, double4x4 view, double4x4 projection) { - Origin = double4x4.Invert(view).Translation(); - double4x4 invViewProjection = double4x4.Invert(projection * view); - var pickPosWorld4 = double4x4.Transform(invViewProjection, new double4(pickPosClip.x, pickPosClip.y, 1, 1)); - var pickPosWorld = (pickPosWorld4 / pickPosWorld4.w).xyz; + var pickPosFarWorld = double4x4.TransformPerspective(invViewProjection, new double3(pickPosClip.x, pickPosClip.y, 1)); + var pickPosNearWorld = double4x4.TransformPerspective(invViewProjection, new double3(pickPosClip.x, pickPosClip.y, -1)); + + Origin = pickPosNearWorld; - Direction = (pickPosWorld - Origin).Normalize(); + _direction = (pickPosFarWorld - pickPosNearWorld).Normalize(); - Inverse = new double3(1 / Direction.x, 1 / Direction.y, 1 / Direction.z); + _inverse = new double3(1 / _direction.x, 1 / _direction.y, 1 / _direction.z); + _inverseDirty = false; } } } \ No newline at end of file diff --git a/src/Math/Core/Rayf.cs b/src/Math/Core/Rayf.cs index c1de24b42..b3e335c9c 100644 --- a/src/Math/Core/Rayf.cs +++ b/src/Math/Core/Rayf.cs @@ -1,24 +1,56 @@ -namespace Fusee.Math.Core +using Newtonsoft.Json; + +namespace Fusee.Math.Core { /// /// Represents a ray with a given origin and direction. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public struct RayF { /// /// The point in world coordinates from which the ray originates. /// + [JsonProperty(PropertyName = "Origin")] public float3 Origin; + private float3 _direction; + /// /// The direction of the ray. /// - public float3 Direction { get; private set; } + [JsonProperty(PropertyName = "Direction")] + public float3 Direction + { + get { return _direction; } + set + { + _direction = float3.Normalize(value); + + _inverseDirty = true; + } + } + + private float3 _inverse; + private bool _inverseDirty; /// /// The inverse of the direction vector of the ray (1 / direction). /// - public float3 Inverse { get; private set; } + public float3 Inverse + { + get + { + if (_inverseDirty) + { + _inverse = new float3(1 / Direction.x, 1 / Direction.y, 1 / Direction.z); + + _inverseDirty = false; + } + + return _inverse; + } + } /// /// Create a new ray. @@ -28,8 +60,11 @@ public struct RayF public RayF(float3 origin_, float3 direction_) { Origin = origin_; - Direction = float3.Normalize(direction_); - Inverse = new float3(1 / Direction.x, 1 / Direction.y, 1 / Direction.z); + + _direction = float3.Normalize(direction_); + + _inverse = default; + _inverseDirty = true; } /// @@ -40,16 +75,17 @@ public RayF(float3 origin_, float3 direction_) /// The Projection Matrix of the rendered scene. public RayF(float2 pickPosClip, float4x4 view, float4x4 projection) { - Origin = float4x4.Invert(view).Translation(); - float4x4 invViewProjection = float4x4.Invert(projection * view); - var pickPosWorld4 = float4x4.Transform(invViewProjection, new float4(pickPosClip.x, pickPosClip.y, 1, 1)); - var pickPosWorld = (pickPosWorld4 / pickPosWorld4.w).xyz; + var pickPosFarWorld = float4x4.TransformPerspective(invViewProjection, new float3(pickPosClip.x, pickPosClip.y, 1)); + var pickPosNearWorld = float4x4.TransformPerspective(invViewProjection, new float3(pickPosClip.x, pickPosClip.y, -1)); + + Origin = pickPosNearWorld; - Direction = (pickPosWorld - Origin).Normalize(); + _direction = (pickPosFarWorld - pickPosNearWorld).Normalize(); - Inverse = new float3(1 / Direction.x, 1 / Direction.y, 1 / Direction.z); + _inverse = new float3(1 / _direction.x, 1 / _direction.y, 1 / _direction.z); + _inverseDirty = false; } } } \ No newline at end of file diff --git a/src/Math/Core/double2.cs b/src/Math/Core/double2.cs index d9c41162a..334bfef7c 100644 --- a/src/Math/Core/double2.cs +++ b/src/Math/Core/double2.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -11,6 +12,7 @@ namespace Fusee.Math.Core /// /// The double2 structure is suitable for inter-operation with unmanaged code requiring two consecutive doubles. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] [ProtoContract] public struct double2 : IEquatable @@ -20,12 +22,14 @@ public struct double2 : IEquatable /// /// The x component of the double2. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public double x; /// /// The y component of the double2. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public double y; diff --git a/src/Math/Core/double3.cs b/src/Math/Core/double3.cs index c3431903d..c20988bf7 100644 --- a/src/Math/Core/double3.cs +++ b/src/Math/Core/double3.cs @@ -1,4 +1,5 @@ -using ProtoBuf; +using Newtonsoft.Json; +using ProtoBuf; using System; using System.Globalization; using System.Runtime.InteropServices; @@ -11,6 +12,7 @@ namespace Fusee.Math.Core /// /// The double3 structure is suitable for inter-operation with unmanaged code requiring three consecutive doubles. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct double3 : IEquatable @@ -20,18 +22,21 @@ public struct double3 : IEquatable /// /// The x component of the double3. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public double x; /// /// The y component of the double3. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public double y; /// /// The z component of the double3. /// + [JsonProperty(PropertyName = "Z")] [ProtoMember(3)] public double z; @@ -1155,6 +1160,46 @@ public static explicit operator double3(float3 d3) return new double3(d3); } +#if MathNet + + /// + /// Explicit cast operator to cast a MathNet Single DenseVector into a double3 value. + /// + /// + public static explicit operator double3(MathNet.Numerics.LinearAlgebra.Single.DenseVector sdv) + { + return sdv.ToFuseeDoubleVector(); + } + + /// + /// Explicit cast operator to cast a MathNet Double DenseVector into a double3 value. + /// + /// + public static explicit operator double3(MathNet.Numerics.LinearAlgebra.Double.DenseVector ddv) + { + return ddv.ToFuseeDoubleVector(); + } + + /// + /// Explicit cast operator to cast a double3 into a MathNet Single DenseVector value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Single.DenseVector(double3 d3) + { + return d3.ToMathNetSingleVector(); + } + + /// + /// Explicit cast operator to cast a double3 into a MathNet Double DenseVector value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Double.DenseVector(double3 d3) + { + return d3.ToMathNetDoubleVector(); + } + +#endif + #endregion Operators #region Overrides diff --git a/src/Math/Core/double3x3.cs b/src/Math/Core/double3x3.cs index aca0bd3a8..2cdf4ae2f 100644 --- a/src/Math/Core/double3x3.cs +++ b/src/Math/Core/double3x3.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Globalization; using System.Runtime.InteropServices; @@ -7,6 +8,7 @@ namespace Fusee.Math.Core /// /// Represents a 3x3 Matrix /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] public struct double3x3 : IEquatable { @@ -15,16 +17,19 @@ public struct double3x3 : IEquatable /// /// Top row of the matrix /// + [JsonProperty(PropertyName = "Row1")] public double3 Row1; /// /// 2nd row of the matrix /// + [JsonProperty(PropertyName = "Row2")] public double3 Row2; /// /// 3rd row of the matrix /// + [JsonProperty(PropertyName = "Row3")] public double3 Row3; /// diff --git a/src/Math/Core/double4.cs b/src/Math/Core/double4.cs index 2bf0ac80d..9d7b8364b 100644 --- a/src/Math/Core/double4.cs +++ b/src/Math/Core/double4.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -9,6 +10,7 @@ namespace Fusee.Math.Core /// /// The double4 structure is suitable for interoperation with unmanaged code requiring four consecutive doubles. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] [ProtoContract] public struct double4 : IEquatable @@ -18,24 +20,28 @@ public struct double4 : IEquatable /// /// The x component of the double4. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public double x; /// /// The y component of the double4. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public double y; /// /// The z component of the double4. /// + [JsonProperty(PropertyName = "Z")] [ProtoMember(3)] public double z; /// /// The w component of the double4. /// + [JsonProperty(PropertyName = "W")] [ProtoMember(4)] public double w; diff --git a/src/Math/Core/double4x4.cs b/src/Math/Core/double4x4.cs index 0baa1c569..2c60356a3 100644 --- a/src/Math/Core/double4x4.cs +++ b/src/Math/Core/double4x4.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -44,6 +45,7 @@ namespace Fusee.Math.Core /// of methods are postfixed with "RH". /// /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct double4x4 : IEquatable @@ -53,24 +55,28 @@ public struct double4x4 : IEquatable /// /// Top row of the matrix /// + [JsonProperty(PropertyName = "Row1")] [ProtoMember(1)] public double4 Row1; /// /// 2nd row of the matrix /// + [JsonProperty(PropertyName = "Row2")] [ProtoMember(2)] public double4 Row2; /// /// 3rd row of the matrix /// + [JsonProperty(PropertyName = "Row3")] [ProtoMember(3)] public double4 Row3; /// /// Bottom row of the matrix /// + [JsonProperty(PropertyName = "Row4")] [ProtoMember(4)] public double4 Row4; @@ -2330,6 +2336,46 @@ public static explicit operator double4x4(float4x4 d4x4) return new double4x4(d4x4); } +#if MathNet + + /// + /// Explicit cast operator to cast a MathNet Single DenseMatrix into a double4x4 value. + /// + /// + public static explicit operator double4x4(MathNet.Numerics.LinearAlgebra.Single.DenseMatrix sdm) + { + return sdm.ToFuseeDoubleMatrix(); + } + + /// + /// Explicit cast operator to cast a MathNet Double DenseMatrix into a double4x4 value. + /// + /// + public static explicit operator double4x4(MathNet.Numerics.LinearAlgebra.Double.DenseMatrix ddm) + { + return ddm.ToFuseeDoubleMatrix(); + } + + /// + /// Explicit cast operator to cast a double4x4 into a MathNet Single DenseMatrix value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Single.DenseMatrix(double4x4 d4x4) + { + return d4x4.ToMathNetSingleMatrix(); + } + + /// + /// Explicit cast operator to cast a double4x4 into a MathNet Double DenseMatrix value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Double.DenseMatrix(double4x4 d4x4) + { + return d4x4.ToMathNetDoubleMatrix(); + } + +#endif + #endregion Operators #region Overrides diff --git a/src/Math/Core/float2.cs b/src/Math/Core/float2.cs index 0df58b5db..63d7ced78 100644 --- a/src/Math/Core/float2.cs +++ b/src/Math/Core/float2.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -11,6 +12,7 @@ namespace Fusee.Math.Core /// /// The float2 structure is suitable for interoperation with unmanaged code requiring two consecutive floats. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] [ProtoContract] public struct float2 : IEquatable @@ -20,12 +22,14 @@ public struct float2 : IEquatable /// /// The x component of the float2. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public float x; /// /// The y component of the float2. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public float y; diff --git a/src/Math/Core/float3.cs b/src/Math/Core/float3.cs index 57caa77ec..8f242b4eb 100644 --- a/src/Math/Core/float3.cs +++ b/src/Math/Core/float3.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -11,6 +12,7 @@ namespace Fusee.Math.Core /// /// The float3 structure is suitable for inter-operation with unmanaged code requiring three consecutive floats. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct float3 : IEquatable @@ -20,18 +22,21 @@ public struct float3 : IEquatable /// /// The x component of the float3. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public float x; /// /// The y component of the float3. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public float y; /// /// The z component of the float3. /// + [JsonProperty(PropertyName = "Z")] [ProtoMember(3)] public float z; @@ -1155,6 +1160,46 @@ public static explicit operator float3(double3 d3) return new float3(d3); } +#if MathNet + + /// + /// Explicit cast operator to cast a MathNet Single DenseVector into a float3 value. + /// + /// + public static explicit operator float3(MathNet.Numerics.LinearAlgebra.Single.DenseVector sdv) + { + return sdv.ToFuseeSingleVector(); + } + + /// + /// Explicit cast operator to cast a MathNet Double DenseVector into a float3 value. + /// + /// + public static explicit operator float3(MathNet.Numerics.LinearAlgebra.Double.DenseVector ddv) + { + return ddv.ToFuseeSingleVector(); + } + + /// + /// Explicit cast operator to cast a float3 into a MathNet Single DenseVector value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Single.DenseVector(float3 f3) + { + return f3.ToMathNetSingleVector(); + } + + /// + /// Explicit cast operator to cast a float3 into a MathNet Double DenseVector value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Double.DenseVector(float3 f3) + { + return f3.ToMathNetDoubleVector(); + } + +#endif + #endregion Operators #region Overrides diff --git a/src/Math/Core/float3x3.cs b/src/Math/Core/float3x3.cs index cd759640a..b4121efd9 100644 --- a/src/Math/Core/float3x3.cs +++ b/src/Math/Core/float3x3.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Globalization; using System.Runtime.InteropServices; @@ -7,6 +8,7 @@ namespace Fusee.Math.Core /// /// Represents a 3x3 Matrix /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] public struct float3x3 : IEquatable { @@ -15,16 +17,19 @@ public struct float3x3 : IEquatable /// /// Top row of the matrix /// + [JsonProperty(PropertyName = "Row1")] public float3 Row1; /// /// 2nd row of the matrix /// + [JsonProperty(PropertyName = "Row2")] public float3 Row2; /// /// 3rd row of the matrix /// + [JsonProperty(PropertyName = "Row3")] public float3 Row3; /// diff --git a/src/Math/Core/float4.cs b/src/Math/Core/float4.cs index 03bbf1123..0c713addb 100644 --- a/src/Math/Core/float4.cs +++ b/src/Math/Core/float4.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -9,6 +10,7 @@ namespace Fusee.Math.Core /// /// The float4 structure is suitable for interoperation with unmanaged code requiring four consecutive floats. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] [ProtoContract] public struct float4 : IEquatable @@ -18,24 +20,28 @@ public struct float4 : IEquatable /// /// The x component of the float4. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public float x; /// /// The y component of the float4. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public float y; /// /// The z component of the float4. /// + [JsonProperty(PropertyName = "Z")] [ProtoMember(3)] public float z; /// /// The w component of the float4. /// + [JsonProperty(PropertyName = "W")] [ProtoMember(4)] public float w; diff --git a/src/Math/Core/float4x4.cs b/src/Math/Core/float4x4.cs index 236b752a8..3ac39bd9f 100644 --- a/src/Math/Core/float4x4.cs +++ b/src/Math/Core/float4x4.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -44,6 +45,7 @@ namespace Fusee.Math.Core /// of methods are postfixed with "RH". /// /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct float4x4 : IEquatable @@ -53,24 +55,28 @@ public struct float4x4 : IEquatable /// /// Top row of the matrix /// + [JsonProperty(PropertyName = "Row1")] [ProtoMember(1)] public float4 Row1; /// /// 2nd row of the matrix /// + [JsonProperty(PropertyName = "Row2")] [ProtoMember(2)] public float4 Row2; /// /// 3rd row of the matrix /// + [JsonProperty(PropertyName = "Row3")] [ProtoMember(3)] public float4 Row3; /// /// Bottom row of the matrix /// + [JsonProperty(PropertyName = "Row4")] [ProtoMember(4)] public float4 Row4; @@ -2140,6 +2146,11 @@ public static float4x4 RotationDecomposition(float4x4 mat) var scalevector = GetScale(mat); var rotationMtx = float4x4.Identity; + if (scalevector.x <= 0 || scalevector.y <= 0 || scalevector.z <= 0) + { + throw new ArgumentException("Scale vector <= 0!"); + } + rotationMtx.M11 = mat.M11 / scalevector.x; rotationMtx.M21 = mat.M21 / scalevector.x; rotationMtx.M31 = mat.M31 / scalevector.x; @@ -2330,6 +2341,46 @@ public static explicit operator float4x4(double4x4 d4x4) return new float4x4(d4x4); } +#if MathNet + + /// + /// Explicit cast operator to cast a MathNet Single DenseMatrix into a float4x4 value. + /// + /// + public static explicit operator float4x4(MathNet.Numerics.LinearAlgebra.Single.DenseMatrix sdv) + { + return sdv.ToFuseeSingleMatrix(); + } + + /// + /// Explicit cast operator to cast a MathNet Double DenseMatrix into a float4x4 value. + /// + /// + public static explicit operator float4x4(MathNet.Numerics.LinearAlgebra.Double.DenseMatrix ddv) + { + return ddv.ToFuseeSingleMatrix(); + } + + /// + /// Explicit cast operator to cast a float4x4 into a MathNet Single DenseMatrix value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Single.DenseMatrix(float4x4 f3) + { + return f3.ToMathNetSingleMatrix(); + } + + /// + /// Explicit cast operator to cast a float4x4 into a MathNet Double DenseMatrix value. + /// + /// + public static explicit operator MathNet.Numerics.LinearAlgebra.Double.DenseMatrix(float4x4 f3) + { + return f3.ToMathNetDoubleMatrix(); + } + +#endif + #endregion Operators #region Overrides diff --git a/src/Math/Core/int2.cs b/src/Math/Core/int2.cs index f1d2b1122..31c937d20 100644 --- a/src/Math/Core/int2.cs +++ b/src/Math/Core/int2.cs @@ -1,4 +1,5 @@ -using ProtoBuf; +using Newtonsoft.Json; +using ProtoBuf; using System; using System.Globalization; using System.Runtime.InteropServices; @@ -11,6 +12,7 @@ namespace Fusee.Math.Core /// /// The int2 structure is suitable for interoperation with unmanaged code requiring two consecutive ints. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] [ProtoContract] public struct int2 : IEquatable @@ -20,12 +22,14 @@ public struct int2 : IEquatable /// /// The x component of the int2. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public int x; /// /// The y component of the int2. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public int y; diff --git a/src/Math/Core/int3.cs b/src/Math/Core/int3.cs index 54033be92..80d4ee64b 100644 --- a/src/Math/Core/int3.cs +++ b/src/Math/Core/int3.cs @@ -1,4 +1,5 @@ -using ProtoBuf; +using Newtonsoft.Json; +using ProtoBuf; using System; using System.Globalization; using System.Runtime.InteropServices; @@ -11,6 +12,7 @@ namespace Fusee.Math.Core /// /// The int3 structure is suitable for inter-operation with unmanaged code requiring three consecutive ints. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [ProtoContract] [StructLayout(LayoutKind.Sequential)] public struct int3 : IEquatable @@ -20,18 +22,21 @@ public struct int3 : IEquatable /// /// The x component of the int3. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public int x; /// /// The y component of the int3. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public int y; /// /// The z component of the int3. /// + [JsonProperty(PropertyName = "Z")] [ProtoMember(3)] public int z; diff --git a/src/Math/Core/int4.cs b/src/Math/Core/int4.cs index a3900f4a3..fe00755ac 100644 --- a/src/Math/Core/int4.cs +++ b/src/Math/Core/int4.cs @@ -1,3 +1,4 @@ +using Newtonsoft.Json; using ProtoBuf; using System; using System.Globalization; @@ -9,6 +10,7 @@ namespace Fusee.Math.Core /// /// The int4 structure is suitable for interoperation with unmanaged code requiring four consecutive ints. /// + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] [StructLayout(LayoutKind.Sequential)] [ProtoContract] public struct int4 : IEquatable @@ -18,24 +20,28 @@ public struct int4 : IEquatable /// /// The x component of the int4. /// + [JsonProperty(PropertyName = "X")] [ProtoMember(1)] public int x; /// /// The y component of the int4. /// + [JsonProperty(PropertyName = "Y")] [ProtoMember(2)] public int y; /// /// The z component of the int4. /// + [JsonProperty(PropertyName = "Z")] [ProtoMember(3)] public int z; /// /// The w component of the int4. /// + [JsonProperty(PropertyName = "W")] [ProtoMember(4)] public int w; diff --git a/src/PointCloud/Common/Accessors/IPointAccessor.cs b/src/PointCloud/Common/Accessors/IPointAccessor.cs deleted file mode 100644 index 8a6ca1479..000000000 --- a/src/PointCloud/Common/Accessors/IPointAccessor.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Every point cloud needs a point accessor. Provides access to the point parameters like position or color. - /// - public interface IPointAccessor - { - /// - /// Returns the point type this accessor can use. - /// - PointType PointType { get; } - - /// - /// Data type of the position values. - /// - PointPositionType PositionType { get; } - - /// - /// Data type of the intensity values. - /// - PointIntensityType IntensityType { get; } - - /// - /// Data type of the normal vectors. - /// - PointNormalType NormalType { get; } - - /// - /// Data type of the color values. - /// - PointColorType ColorType { get; } - - /// - /// Data type of the label values. - /// - PointLabelType LabelType { get; } - - /// - /// Data type of the curvature values. - /// - PointCurvatureType CurvatureType { get; } - - /// - /// Data type of the hit count values. - /// - PointHitCountType HitCountType { get; } - - /// - /// Data type of the gps time values. - /// - PointGpsTimeType GpsTimeType { get; } - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointColorType.cs b/src/PointCloud/Common/Accessors/PointColorType.cs deleted file mode 100644 index 8555e1d82..000000000 --- a/src/PointCloud/Common/Accessors/PointColorType.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Fusee.Math.Core; - -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's per point color data. - /// - public enum PointColorType - { - /// - /// A point cloud point without a color value. - /// - None, - /// - /// A point cloud point has a color value of type . - /// - SByte, - /// - /// A point cloud point has a color value of type . - /// - Short, - /// - /// A point cloud point has a color value of type . - /// - Int, - /// - /// A point cloud point has a color value of type . - /// - Long, - /// - /// A point cloud point has a color value of type . - /// - Byte, - /// - /// A point cloud point has a color value of type . - /// - Ushort, - /// - /// A point cloud point has a color value of type . - /// - Uint, - /// - /// A point cloud point has a color value of type . - /// - Ulong, - /// - /// A point cloud point has a color value of type . - /// - Float, - /// - /// A point cloud point has a color value of type . - /// - Double, - /// - /// A point cloud point has a color value of type . - /// - Float3, - /// - /// A point cloud point has a color value of type . - /// - Double3 - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointCurvatureType.cs b/src/PointCloud/Common/Accessors/PointCurvatureType.cs deleted file mode 100644 index 340ccc9f8..000000000 --- a/src/PointCloud/Common/Accessors/PointCurvatureType.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's curvature data. - /// - public enum PointCurvatureType - { - /// - /// A point cloud without a curvature. - /// - None, - /// - /// A point cloud point has a curvature value of type . - /// - SByte, - /// - /// A point cloud point has a curvature value of type . - /// - Short, - /// - /// A point cloud point has a curvature value of type . - /// - Int, - /// - /// A point cloud point has a curvature value of type . - /// - Long, - /// - /// A point cloud point has a curvature value of type . - /// - Byte, - /// - /// A point cloud point has a curvature value of type . - /// - UShort, - /// - /// A point cloud point has a curvature value of type . - /// - Uint, - /// - /// A point cloud point has a curvature value of type . - /// - ULong, - /// - /// A point cloud point has a curvature value of type . - /// - Float, - /// - /// A point cloud point has a curvature value of type . - /// - Double, - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointGpsTimeType.cs b/src/PointCloud/Common/Accessors/PointGpsTimeType.cs deleted file mode 100644 index 3e1428a40..000000000 --- a/src/PointCloud/Common/Accessors/PointGpsTimeType.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's gps time data. - /// - public enum PointGpsTimeType - { - /// - /// A point cloud point without gps time. - /// - None, - /// - /// A point cloud point has a gps time value of type . - /// - SByte, - /// - /// A point cloud point has a gps time value of type . - /// - Short, - /// - /// A point cloud point has a gps time value of type . - /// - Int, - /// - /// A point cloud point has a gps time value of type . - /// - Long, - /// - /// A point cloud point has a gps time value of type . - /// - Byte, - /// - /// A point cloud point has a gps time value of type . - /// - UShort, - /// - /// A point cloud point has a gps time value of type . - /// - Uint, - /// - /// A point cloud point has a gps time value of type . - /// - ULong, - /// - /// A point cloud point has a gps time value of type . - /// - Float, - /// - /// A point cloud point has a gps time value of type . - /// - Double - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointHitCountType.cs b/src/PointCloud/Common/Accessors/PointHitCountType.cs deleted file mode 100644 index c3013c8fd..000000000 --- a/src/PointCloud/Common/Accessors/PointHitCountType.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's hit count data. - /// - public enum PointHitCountType - { - /// - /// A point cloud point has a label without a hit count. - /// - None, - /// - /// A point cloud point has a hit count value of type . - /// - SByte, - /// - /// A point cloud point has a hit count value of type . - /// - Short, - /// - /// A point cloud point has a hit count value of type . - /// - Int, - /// - /// A point cloud point has a hit count value of type . - /// - Long, - /// - /// A point cloud point has a hit count value of type . - /// - Byte, - /// - /// A point cloud point has a hit count value of type . - /// - UShort, - /// - /// A point cloud point has a hit count value of type . - /// - Uint, - /// - /// A point cloud point has a hit count value of type . - /// - ULong, - /// - /// A point cloud point has a hit count value of type . - /// - Float, - /// - /// A point cloud point has a hit count value of type . - /// - Double, - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointIntensityType.cs b/src/PointCloud/Common/Accessors/PointIntensityType.cs deleted file mode 100644 index 9e065119e..000000000 --- a/src/PointCloud/Common/Accessors/PointIntensityType.cs +++ /dev/null @@ -1,63 +0,0 @@ -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's intensity data. - /// - public enum PointIntensityType - { - /// - /// A point cloud point without an intensity value. - /// - None, - - /// - /// A point cloud point has a intensity value of type . - /// - SByte, - - /// - ///A point cloud point has a intensity value of type . - /// - Short, - - /// - /// A point cloud point has a intensity value of type . - /// - Int, - - /// - /// A point cloud point has a intensity value of type . - /// - Long, - - /// - /// A point cloud point has a intensity value of type . - /// - Byte, - - /// - /// A point cloud point has a intensity value of type . - /// - UShort, - - /// - /// A point cloud point has a intensity value of type . - /// - UInt, - - /// - /// Returns a bool that tells if a point cloud point has a intensity value of type . - /// - ULong, - - /// - /// Returns a bool that tells if a point cloud point has a intensity value of type . - /// - Float, - - /// - /// Returns a bool that tells if a point cloud point has a intensity value of type . - /// - Double - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointLabelType.cs b/src/PointCloud/Common/Accessors/PointLabelType.cs deleted file mode 100644 index 4cf9d8de8..000000000 --- a/src/PointCloud/Common/Accessors/PointLabelType.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's label data. - /// - public enum PointLabelType - { - /// - /// A point cloud without a label. - /// - None, - /// - /// A point cloud point has a label with a value of type . - /// - SByte, - /// - /// A point cloud point has a label with a value of type . - /// - Short, - /// - /// A point cloud point has a label with a value of type . - /// - Int, - /// - /// A point cloud point has a label with a value of type . - /// - Long, - /// - /// A point cloud point has a label with a value of type . - /// - Byte, - /// - /// A point cloud point has a label with a value of type . - /// - UShort, - /// - /// A point cloud point has a label with a value of type . - /// - UInt, - /// - /// A point cloud point has a label with a value of type . - /// - ULong, - /// - /// A point cloud point has a label with a value of type . - /// - Float, - /// - /// A point cloud point has a label with a value of type . - /// - Double, - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointNormalType.cs b/src/PointCloud/Common/Accessors/PointNormalType.cs deleted file mode 100644 index 5489b8401..000000000 --- a/src/PointCloud/Common/Accessors/PointNormalType.cs +++ /dev/null @@ -1,24 +0,0 @@ -using Fusee.Math.Core; - -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's normal vectors. - /// - public enum PointNormalType - { - /// - /// A point cloud point without an normal. - /// - None, - - /// - /// A point cloud point has a position value of type . - /// - Float3, - /// - /// A point cloud point has a position value of type . - /// - Double3 - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointPositionType.cs b/src/PointCloud/Common/Accessors/PointPositionType.cs deleted file mode 100644 index 2089072ab..000000000 --- a/src/PointCloud/Common/Accessors/PointPositionType.cs +++ /dev/null @@ -1,24 +0,0 @@ -using Fusee.Math.Core; - -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Declares valid data types for a point cloud's position data. - /// - public enum PointPositionType - { - /// - /// The position of this point is undefined - renders the point unusable. - /// - Undefined, - - /// - /// A point cloud point has a position value of type . - /// - Float3, - /// - /// A point cloud point has a position value of type . - /// - Double3 - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Accessors/PointTypes.cs b/src/PointCloud/Common/Accessors/PointTypes.cs deleted file mode 100644 index 92d8c83f1..000000000 --- a/src/PointCloud/Common/Accessors/PointTypes.cs +++ /dev/null @@ -1,248 +0,0 @@ - -using Fusee.Math.Core; - -namespace Fusee.PointCloud.Common.Accessors -{ - /// - /// Enum that contains all available point types. - /// Abbreviations: - /// F3: float - /// D3: double - /// Us: ushort - /// B: byte - /// Pos: position - /// Col: Color - /// In: Intensity - /// Nor: Normal - /// Lbl: Label / Classification - /// - public enum PointType - { - /// - /// Point type is not set yet. - /// - Undefined, - - /// - /// Position only (double) - /// - PosD3, - /// - /// Position (double), Color (float), Intensity (short) - /// - PosD3ColF3InUs, - /// - /// Position (double), Intensity (ushort) - /// - PosD3InUs, - /// - /// Position (double), Color (float) - /// - PosD3ColF3, - /// - /// Position (double), Label (byte) - /// - PosD3LblB, - /// - /// Position (double), Normal (float), Color (float), Intensity (ushort) - /// - PosD3NorF3ColF3InUs, - /// - /// Position (double), Normal (float), Intensity (short) - /// - PosD3NorF3InUs, - /// - /// Position (double), Normal (float), Color (float) - /// - PosD3NorF3ColF3, - /// - /// Position (double), Color (float), Classification (byte) - /// - PosD3ColF3LblB, - /// - /// Position (double), Color (float), Classification (byte), Intensity (ushort) - /// - PosD3ColF3InUsLblB - } - - /// - /// Point type: Position double3. - /// - public struct PosD3 - { - /// - /// The points coordinate in 3D space. - /// - public double3 Position; - } - - /// - /// Point type: Position, color, intensity. - /// - public struct PosD3ColF3InUs - { - /// - /// The points coordinate in 3D space. - /// - public double3 Position; - /// - /// The points rgb color. - /// - public float3 Color; - /// - /// The points intensity (gray scale). - /// - public ushort Intensity; - } - - /// - /// Point type: Position, intensity. - /// - public struct PosD3InUs - { - /// - /// The points coordinate in 3D space. - /// - public double3 Position; - /// - /// The points intensity (gray scale). - /// - public ushort Intensity; - } - - /// - /// Point type: Position, color. - /// - public struct PosD3ColF3 - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's rgb color. - /// - public float3 Color; - } - - /// - /// Point type: Position and label color. - /// - public struct PosD3LblB - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's struct label. - /// - public byte Label; - } - - /// - /// Point type: Position, normal, color, intensity. - /// - public struct PosD3NorF3ColF3InUs - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's normal vector. - /// - public float3 Normal; - /// - /// The point's rgb color. - /// - public float3 Color; - /// - /// The point's intensity (gray scale). - /// - public ushort Intensity; - } - - // - - /// - /// Point type: Position, normal, intensity. - /// - public struct PosD3NorF3InUs - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's normal vector. - /// - public float3 Normal; - /// - /// The point's intensity (gray scale). - /// - public ushort Intensity; - } - - /// - /// Point type: , , . - /// - public struct PosD3NorF3ColF3 - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's normal vector. - /// - public float3 Normal; - /// - /// The point's rgb color. - /// - public float3 Color; - } - - - /// - /// Point type: , , . - /// - public struct PosD3ColF3LblB - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's rgb color. - /// - public float3 Color; - /// - /// The point's classification. - /// - public byte Label; - } - - /// - /// Point type: , , and . - /// - public struct PosD3ColF3InUsLblB - { - /// - /// The point's coordinate in 3D space. - /// - public double3 Position; - /// - /// The point's rgb color. - /// - public float3 Color; - /// - /// The point's classification. - /// - public byte Label; - /// - /// The point's intensity (gray scale). - /// - public ushort Intensity; - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/Assets/PointCloudInstanced.vert b/src/PointCloud/Common/Assets/PointCloudInstanced.vert index d03cccbde..7c2ce14d3 100644 --- a/src/PointCloud/Common/Assets/PointCloudInstanced.vert +++ b/src/PointCloud/Common/Assets/PointCloudInstanced.vert @@ -1,4 +1,4 @@ -#version 300 es +#version 460 core precision highp float; @@ -16,7 +16,6 @@ out float vWorldSpacePointRad; out vec2 vPointCoord; //equivalent to gl_pointCoord in vec3 fuVertex; -in vec3 fuColor; in mat4 fuInstanceModelMat; //in vec3 fuInstanceColor; diff --git a/src/PointCloud/Common/Assets/PointDepthInstanced.frag b/src/PointCloud/Common/Assets/PointDepthInstanced.frag index 454300221..1a39b15d0 100644 --- a/src/PointCloud/Common/Assets/PointDepthInstanced.frag +++ b/src/PointCloud/Common/Assets/PointDepthInstanced.frag @@ -1,4 +1,4 @@ -#version 330 core +#version 460 core uniform mat4 FUSEE_P; uniform int PointShape; diff --git a/src/PointCloud/Common/Fusee.PointCloud.Common.csproj b/src/PointCloud/Common/Fusee.PointCloud.Common.csproj index ef134cf84..2855d6f75 100644 --- a/src/PointCloud/Common/Fusee.PointCloud.Common.csproj +++ b/src/PointCloud/Common/Fusee.PointCloud.Common.csproj @@ -1,8 +1,9 @@ - - + + netstandard2.1;net7.0 $(OutputPath)\$(RootNamespace).xml + enable @@ -10,11 +11,13 @@ - - + + - + + + diff --git a/src/PointCloud/Common/IPointCloudImp.cs b/src/PointCloud/Common/IPointCloudImp.cs index 7dc571e89..08b25d50c 100644 --- a/src/PointCloud/Common/IPointCloudImp.cs +++ b/src/PointCloud/Common/IPointCloudImp.cs @@ -1,4 +1,5 @@ -using Fusee.Engine.Core; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Engine.Core; using Fusee.Math.Core; using System.Collections.Generic; @@ -9,6 +10,8 @@ namespace Fusee.PointCloud.Common /// public interface IPointCloudImpBase { + public InvalidateGpuDataCache InvalidateGpuDataCache { get; } + /// /// Center of the PointCloud's AABB /// @@ -63,15 +66,31 @@ public float UpdateRate public void Update(float fov, int viewportHeight, FrustumF renderFrustum, float3 camPos, float4x4 modelMat); } + /// + /// Delegate that allows to inject a method that knows how to update gpu/mesh data with data from points. + /// + /// + /// + /// The gpu/mesh data. + /// The points with the desired values. + public delegate void UpdateGpuData(ref IEnumerable gpuData, MemoryOwner points); + /// /// Smallest common set of properties that are needed to render point clouds out of core. /// - public interface IPointCloudImp : IPointCloudImpBase + public interface IPointCloudImp : IPointCloudImpBase { /// /// The , created from visible octants/point chunks, that are ready to be rendered. /// public List GpuDataToRender { get; set; } + /// + /// Allows to update meshes with data from the points. + /// + /// The meshes that have to be updated. + /// The points with the desired values. + public void UpdateGpuDataCache(ref IEnumerable meshes, MemoryOwner points); + } } \ No newline at end of file diff --git a/src/PointCloud/Common/IPointReader.cs b/src/PointCloud/Common/IPointReader.cs index 8a63fceea..3693d48f8 100644 --- a/src/PointCloud/Common/IPointReader.cs +++ b/src/PointCloud/Common/IPointReader.cs @@ -1,40 +1,19 @@ -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Common +namespace Fusee.PointCloud.Common { /// /// Implement this into any Point Cloud Reader. /// public interface IPointReader { - /// - /// A PointAccessor allows access to the point information (position, color, ect.) without casting it to a specific . - /// - public IPointAccessor PointAccessor { get; } - /// /// Returns a renderable point cloud component. /// - /// Path to the file. /// Determines which is used to display the returned point cloud."/> public IPointCloud GetPointCloudComponent(RenderMode renderMode); - /// - /// Returns the point type. - /// - public PointType PointType { get; } - /// /// Reads the Potree file and returns an octree. /// public IPointCloudOctree GetOctree(); - - /// - /// Returns the points for one octant as generic array. - /// - /// The generic point type. - /// The unique id of the octant. - public TPoint[] LoadNodeData(OctantId id) where TPoint : new(); - } } \ No newline at end of file diff --git a/src/PointCloud/Common/IPointWriter.cs b/src/PointCloud/Common/IPointWriter.cs new file mode 100644 index 000000000..0898f7c06 --- /dev/null +++ b/src/PointCloud/Common/IPointWriter.cs @@ -0,0 +1,112 @@ +using Fusee.Math.Core; +using System.IO; +using System.Text; + +namespace Fusee.PointCloud.Common +{ + /// + /// Information about the hierarchy data of the given point cloud + /// + public interface IPointWriterHierarchy + { + /// + /// Size of the first chunk + /// + public int FirstChunkSize { get; set; } + + /// + /// Hierarchy step size + /// + public int StepSize { get; set; } + + /// + /// Depth of the Hierarchy + /// + public int Depth { get; set; } + } + + /// + /// Metadata about the point cloud the "attributes" which are needed for e. g. the LAS export. + /// + public interface IPointWriterMetadata + { + /// + /// Version flag + /// + public string Version { get; } + + /// + /// Point cloud name + /// + public string Name { get; } + + /// + /// Description of point cloud + /// + public string Description { get; } + + /// + /// How many points does this point cloud contain + /// + public int PointCount { get; } + + /// + /// A possible projection method + /// + public string Projection { get; } + + /// + /// The point cloud hierarchy information + /// + public IPointWriterHierarchy Hierarchy { get; } + + /// + /// Global offset of each point + /// + public double3 Offset { get; } + + /// + /// Global scale value. Points are being converted to int + /// During load this scale factor is being applied to convert int to double + /// + public double3 Scale { get; } + + /// + /// The spacing between points (set during the sampling process) + /// + public double Spacing { get; } + + /// + /// Global of the point cloud + /// These values are not yet y/z flipped + /// + public AABBd AABB { get; } + + /// + /// The encoding of every point, as we save the point cloud as elements + /// Default is + /// + public string Encoding { get; } + + /// + /// The size of one point in bytes (for sanity checks later on, compare with e. g. LASPoint type data) + /// + public int PointSize { get; } + } + + /// + /// Every point writer (e. g. Potree, LAS, etc.) implements this interface + /// + public interface IPointWriter + { + /// + /// The necessary metadata + /// + IPointWriterMetadata Metadata { get; } + + /// + /// The file to write to + /// + FileInfo SavePath { get; } + } +} \ No newline at end of file diff --git a/src/PointCloud/Common/InvalidateGpuDataCache.cs b/src/PointCloud/Common/InvalidateGpuDataCache.cs new file mode 100644 index 000000000..66316fdd5 --- /dev/null +++ b/src/PointCloud/Common/InvalidateGpuDataCache.cs @@ -0,0 +1,31 @@ +using System; + +namespace Fusee.PointCloud.Common +{ + /// + /// Token for invalidating the cached gpu data . + /// + public class InvalidateGpuDataCache + { + /// + /// Called when the value of changes. + /// + public Action? IsDirtyPropertyChanged; + + /// + /// Set this to true if the Data Handler should invalidate the gpu data cache. + /// Is set to false internally. + /// + public bool IsDirty + { + get => _isDirty; + set + { + _isDirty = value; + IsDirtyPropertyChanged?.Invoke(_isDirty); + } + + } + private bool _isDirty; + } +} \ No newline at end of file diff --git a/src/PointCloud/Common/OctantID.cs b/src/PointCloud/Common/OctantID.cs index 1e3400722..05f3c38b2 100644 --- a/src/PointCloud/Common/OctantID.cs +++ b/src/PointCloud/Common/OctantID.cs @@ -64,7 +64,7 @@ enum OctantHelper : long // - public struct OctantId : IEnumerable<(int, OctantOrientation)> + public struct OctantId : IEnumerable<(int, OctantOrientation)>, IEquatable { private long _id = -1; @@ -227,9 +227,28 @@ IEnumerator IEnumerable.GetEnumerator() public static bool IsDown(OctantOrientation oo) => (oo & OctantOrientation.DownUpMask) == 0; public static bool IsUp(OctantOrientation oo) => (oo & OctantOrientation.DownUpMask) != 0; + /// public override string ToString() { return Convert.ToString(this, 2); } + + /// + public override int GetHashCode() + { + return _id.GetHashCode(); + } + + /// + public override bool Equals(object obj) + { + return Equals((OctantId)obj); + } + + /// + public bool Equals(OctantId other) + { + return _id == other._id; + } } } \ No newline at end of file diff --git a/src/PointCloud/Common/PointCloudDataHandlerBase.cs b/src/PointCloud/Common/PointCloudDataHandlerBase.cs deleted file mode 100644 index 908c69f80..000000000 --- a/src/PointCloud/Common/PointCloudDataHandlerBase.cs +++ /dev/null @@ -1,59 +0,0 @@ -using System.Collections.Generic; - -namespace Fusee.PointCloud.Common -{ - /// - /// Manages the caching and loading of point and mesh data. - /// - public abstract class PointCloudDataHandlerBase - { - /// - /// Used to manage gpu pressure when disposing of a large quantity of meshes. - /// - public float DisposeRate = 1 / 3f; - - /// - /// Number of nodes that will be loaded, starting with the one with the biggest screen projected size to ensure no octant is loaded that will be invisible in a few frames. - /// Load the five biggest nodes (screen projected size) as proposed in Schütz' thesis. - /// - protected int MaxNumberOfNodesToLoad = 5; - - /// - /// Contains nodes that are queued for loading in the background. - /// - protected List LoadingQueue; - - /// - /// Contains meshes that are marked for disposal. - /// - protected Dictionary> DisposeQueue; - - /// - /// Locking object for the loading queue. - /// - protected static object LockLoadingQueue = new(); - /// - /// Locking object for the dispose queue. - /// - protected static object LockDisposeQueue = new(); - - /// - /// First looks in the mesh cache, if there are meshes return, - /// else look in the DisposeQueue, if there are meshes return, - /// else look in the point cache, if there are points create a mesh and add to the MeshCache. - /// - /// The unique id of an octant. - public abstract IEnumerable GetGpuData(OctantId guid); - - /// - /// Loads points from the hard drive if they are neither in the loading queue nor in the PointCahce. - /// - /// The octant for which the points should be loaded. - public abstract void TriggerPointLoading(OctantId guid); - - /// - /// Disposes of unused meshes, if needed. Depends on the dispose rate and the expiration frequency of the MeshCache. - /// - public abstract void ProcessDisposeQueue(); - } -} \ No newline at end of file diff --git a/src/PointCloud/Common/PointRenderEnums.cs b/src/PointCloud/Common/PointRenderEnums.cs index 33f0954e1..17e36013c 100644 --- a/src/PointCloud/Common/PointRenderEnums.cs +++ b/src/PointCloud/Common/PointRenderEnums.cs @@ -1,5 +1,32 @@ namespace Fusee.PointCloud.Common { + /// + /// The gpu data can take different states in its life cycle. + /// Gpu data may need to be handled differently according to its current state. + /// + public enum GpuDataState + { + /// + /// Default, gpu data wasn't created yet and or points havent been loaded yet. + /// + None = -1, + + /// + /// Gpu data was newly created. + /// + New = 0, + + /// + /// Gpu data accessed but hasn't changed. + /// + Unchanged = 1, + + /// + /// Gpu data accessed and has changed. For example if a property of the data was updated. + /// + Changed = 2, + } + /// /// Available render modes. /// diff --git a/src/PointCloud/Core/Accessors/PointAccessor.cs b/src/PointCloud/Core/Accessors/PointAccessor.cs deleted file mode 100644 index 0ee9a1f4c..000000000 --- a/src/PointCloud/Core/Accessors/PointAccessor.cs +++ /dev/null @@ -1,2170 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// Every point cloud needs a point accessor. Provides access to the point parameters like position or color. - /// - public abstract class PointAccessor : IPointAccessor - { - /// - /// Returns the type of the point as list of the HasXY methods. - /// - /// - public List GetSetPropertyNames() - { - // Point Type (enum) - return GetType().GetProperties().Where(p => p.PropertyType == typeof(bool) && (bool)p.GetValue(this, null)).Select(p => p.Name).ToList(); - } - - /// - /// Returns the point type of this accessor. If it is "undefined" it will translate the type from its type properties. - /// - public PointType PointType - { - get - { - if (_pointType == PointType.Undefined) - GetPointType(); - - return _pointType; - } - - } - private PointType _pointType = PointType.Undefined; - - private void GetPointType() - { - // Pos64 - _pointType = PositionType switch - { - PointPositionType.Double3 when IntensityType == PointIntensityType.None && NormalType == PointNormalType.None && ColorType == PointColorType.None && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3, - PointPositionType.Double3 when IntensityType == PointIntensityType.UShort && NormalType == PointNormalType.None && ColorType == PointColorType.Float && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3ColF3InUs, - PointPositionType.Double3 when IntensityType == PointIntensityType.UShort && NormalType == PointNormalType.None && ColorType == PointColorType.None && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3InUs, - PointPositionType.Double3 when IntensityType == PointIntensityType.None && NormalType == PointNormalType.None && ColorType == PointColorType.Float && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3ColF3, - PointPositionType.Double3 when IntensityType == PointIntensityType.None && NormalType == PointNormalType.None && ColorType == PointColorType.None && LabelType == PointLabelType.Byte && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3LblB, - PointPositionType.Double3 when IntensityType == PointIntensityType.UShort && NormalType == PointNormalType.Float3 && ColorType == PointColorType.Float && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3NorF3ColF3InUs, - PointPositionType.Double3 when IntensityType == PointIntensityType.UShort && NormalType == PointNormalType.Float3 && ColorType == PointColorType.None && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3NorF3InUs, - PointPositionType.Double3 when IntensityType == PointIntensityType.None && NormalType == PointNormalType.Float3 && ColorType == PointColorType.Float && LabelType == PointLabelType.None && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3NorF3ColF3, - PointPositionType.Double3 when IntensityType == PointIntensityType.None && NormalType == PointNormalType.None && ColorType == PointColorType.Float && LabelType == PointLabelType.Byte && CurvatureType == PointCurvatureType.None && HitCountType == PointHitCountType.None && GpsTimeType == PointGpsTimeType.None => PointType.PosD3ColF3LblB, - _ => throw new Exception("Undefined Point Type!"), - }; - } - - /// - /// Data type of the position values. - /// - public PointPositionType PositionType { get; set; } = PointPositionType.Undefined; - /// - /// Data type of the intensity values. - /// - public PointIntensityType IntensityType { get; set; } = PointIntensityType.None; - /// - /// Data type of the normal vectors. - /// - public PointNormalType NormalType { get; set; } = PointNormalType.None; - /// - /// Data type of the color values. - /// - public PointColorType ColorType { get; set; } = PointColorType.None; - /// - /// Data type of the label values. - /// - public PointLabelType LabelType { get; set; } = PointLabelType.None; - /// - /// Data type of the curvature values. - /// - public PointCurvatureType CurvatureType { get; set; } = PointCurvatureType.None; - /// - /// Data type of the hit count values. - /// - public PointHitCountType HitCountType { get; set; } = PointHitCountType.None; - /// - /// Data type of the gps time values. - /// - public PointGpsTimeType GpsTimeType { get; set; } = PointGpsTimeType.None; - - /// - /// Returns the generic raw point. - /// - /// The point cloud point. - public byte[] GetRawPoint(ref TPoint point) - { - if (point == null) - throw new NullReferenceException("Given point is null!"); - - var position = GetRawPosition(ref point); - var intensity = GetRawIntensity(ref point); - var normals = GetRawNormals(ref point); - var rgb = GetRawColor(ref point); - var label = GetRawLabel(ref point); - var curvature = GetRawCurvature(ref point); - var hitCount = GetRawHitCount(ref point); - var GPSTime = GetRawGPSTime(ref point); - - return position.Concat(intensity).Concat(normals).Concat(rgb).Concat(label).Concat(curvature).Concat(hitCount).Concat(GPSTime).ToArray(); - } - - /// - /// Sets the values of a point cloud point. - /// - /// The generic point. - /// The values as byte array. - public void SetRawPoint(ref TPoint pointIn, byte[] byteIn) - { - if (pointIn == null || byteIn == null || byteIn.Length < 8) - throw new NullReferenceException("Invalid data given"); - - // Call all methods to recreate the point - SetRawPosition(ref pointIn, byteIn); - SetRawIntensity(ref pointIn, byteIn); - SetRawNormals(ref pointIn, byteIn); - SetRawColor(ref pointIn, byteIn); - SetRawLabel(ref pointIn, byteIn); - SetRawCurvature(ref pointIn, byteIn); - SetRawHitCount(ref pointIn, byteIn); - SetRawGPSTime(ref pointIn, byteIn); - } - - #region PointT_Methods - - #region Get/Set Position - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float3 GetPositionFloat3_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetPositionFloat32"); - } - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public virtual void SetPositionFloat3_32(ref TPoint point, float3 val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetPositionFloat32"); - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double3 GetPositionFloat3_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetPositionFloat64"); - } - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public virtual void SetPositionFloat3_64(ref TPoint point, double3 val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetPositionFloat64"); - } - #endregion - - #region Get/Set Intensity - - #region Getter - - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref sbyte GetIntensityInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityInt_8"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref short GetIntensityInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityInt_16"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref int GetIntensityInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityInt_32"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref long GetIntensityInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityInt_64"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref byte GetIntensityUInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityUInt_8"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ushort GetIntensityUInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityUInt_16"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref uint GetIntensityUInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityUInt_32"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ulong GetIntensityUInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityUInt_64"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float GetIntensityFloat32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityFloat32"); - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double GetIntensityFloat64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetIntensityFloat64"); - } - #endregion - - #region Setter - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityInt_8(ref TPoint point, sbyte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityInt_8"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityInt_16(ref TPoint point, short val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityInt_16"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityInt_32(ref TPoint point, int val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityInt_32"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityInt_64(ref TPoint point, long val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityInt_64"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityUInt_8(ref TPoint point, byte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityUInt_8"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityUInt_16(ref TPoint point, ushort val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityUInt_16"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityUInt_32(ref TPoint point, uint val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityUInt_32"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityUInt_64(ref TPoint point, ulong val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityUInt_64"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityFloat32(ref TPoint point, float val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityFloat32"); - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public virtual void SetIntensityFloat64(ref TPoint point, double val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetIntensityFloat64"); - } - #endregion - - #endregion - - #region Get/Set Normal - - #region Getter - - /// - /// Returns the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float3 GetNormalFloat3_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetNormalFloat3_32"); - } - - /// - /// Returns the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double3 GetNormalFloat3_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetNormalFloat3_64"); - } - #endregion - - #region Setter - /// - /// Sets the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - /// The new normal vector. - public virtual void SetNormalFloat3_32(ref TPoint point, float3 val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetNormalFloat3_32"); - } - /// - /// Sets the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - /// The new normal vector. - public virtual void SetNormalFloat3_64(ref TPoint point, double3 val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetNormalFloat3_64"); - } - #endregion - - #endregion - - #region Get/Set Color - - #region Getter - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref sbyte GetColorInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorInt_8"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref short GetColorInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorInt_16"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref int GetColorInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorInt_32"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref long GetColorInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorInt_64"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref byte GetColorUInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorUInt_8"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ushort GetColorUInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorUInt_16"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref uint GetColorUInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorUInt_32"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ulong GetColorUInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorUInt_64"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float GetColorFloat32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorFloat32"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double GetColorFloat64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorFloat64"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float3 GetColorFloat3_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorFloat3_32"); - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double3 GetColorFloat3_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetColorFloat3_64"); - } - #endregion - - #region Setter - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorInt_8(ref TPoint point, sbyte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorInt_8"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorInt_16(ref TPoint point, short val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorInt_16"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorInt_32(ref TPoint point, int val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorInt_32"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorInt_64(ref TPoint point, long val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorInt_64"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorUInt_8(ref TPoint point, byte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorUInt_8"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorUInt_16(ref TPoint point, ushort val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorUInt_16"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorUInt_32(ref TPoint point, uint val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorUInt_32"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorUInt_64(ref TPoint point, ulong val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorUInt_64"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorFloat32(ref TPoint point, float val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorFloat32"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorFloat64(ref TPoint point, double val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorFloat64"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorFloat3_32(ref TPoint point, float3 val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorFloat3_32"); - } - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public virtual void SetColorFloat3_64(ref TPoint point, double3 val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetColorFloat3_64"); - } - #endregion - - #endregion - - #region Get/Set Label - - #region Getter - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref sbyte GetLabelInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelInt_8"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref short GetLabelInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelInt_16"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref int GetLabelInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelInt_32"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref long GetLabelInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelInt_64"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref byte GetLabelUInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelUInt_8"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ushort GetLabelUInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelUInt_16"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref uint GetLabelUInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelUInt_32"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ulong GetLabelUInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelUInt_64"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float GetLabelFloat32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelFloat32"); - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double GetLabelFloat64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetLabelFloat64"); - } - #endregion - - #region Setter - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelInt_8(ref TPoint point, sbyte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelInt_8"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelInt_16(ref TPoint point, short val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelInt_16"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelInt_32(ref TPoint point, int val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelInt_32"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelInt_64(ref TPoint point, long val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelInt_64"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelUInt_8(ref TPoint point, byte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelUInt_8"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelUInt_16(ref TPoint point, ushort val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelUInt_16"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelUInt_32(ref TPoint point, uint val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelUInt_32"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelUInt_64(ref TPoint point, ulong val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelUInt_64"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelFloat32(ref TPoint point, float val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelFloat32"); - } - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public virtual void SetLabelFloat64(ref TPoint point, double val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetLabelFloat64"); - } - #endregion - - #endregion - - #region Get/Set Curvature - - #region Getter - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref sbyte GetCurvatureInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureInt_8"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref short GetCurvatureInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureInt_16"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref int GetCurvatureInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureInt_32"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref long GetCurvatureInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureInt_64"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref byte GetCurvatureUInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureUInt_8"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ushort GetCurvatureUInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureUInt_16"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref uint GetCurvatureUInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureUInt_32"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ulong GetCurvatureUInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureUInt_64"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float GetCurvatureFloat32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureFloat32"); - } - /// - /// Returns the curvature of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double GetCurvatureFloat64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetCurvatureFloat64"); - } - #endregion - - #region Setter - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureInt_8(ref TPoint point, sbyte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureInt_8"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureInt_16(ref TPoint point, short val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureInt_16"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureInt_32(ref TPoint point, int val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureInt_32"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureInt_64(ref TPoint point, long val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureInt_64"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureUInt_8(ref TPoint point, byte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureUInt_8"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureUInt_16(ref TPoint point, ushort val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureUInt_16"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureUInt_32(ref TPoint point, uint val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureUInt_32"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureUInt_64(ref TPoint point, ulong val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureUInt_64"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureFloat32(ref TPoint point, float val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureFloat32"); - } - /// - /// Sets the curvature of a point cloud point if is true. - /// - /// The point cloud point. - /// The new curvature. - public virtual void SetCurvatureFloat64(ref TPoint point, double val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetCurvatureFloat64"); - } - #endregion - - #endregion - - #region Get/Set Hit Count - - #region Getter - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref sbyte GetHitCountInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountInt_8"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref short GetHitCountInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountInt_16"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref int GetHitCountInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountInt_32"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref long GetHitCountInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountInt_64"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref byte GetHitCountUInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountUInt_8"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ushort GetHitCountUInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountUInt_16"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref uint GetHitCountUInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountUInt_32"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ulong GetHitCountUInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountUInt_64"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float GetHitCountFloat32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountFloat32"); - } - /// - /// Returns the hit count of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double GetHitCountFloat64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetHitCountFloat64"); - } - #endregion - - #region Setter - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountInt_8(ref TPoint point, sbyte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountInt_8"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountInt_16(ref TPoint point, short val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountInt_16"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountInt_32(ref TPoint point, int val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountInt_32"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountInt_64(ref TPoint point, long val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountInt_64"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountUInt_8(ref TPoint point, byte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountUInt_8"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountUInt_16(ref TPoint point, ushort val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountUInt_16"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountUInt_32(ref TPoint point, uint val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountUInt_32"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountUInt_64(ref TPoint point, ulong val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountUInt_64"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountFloat32(ref TPoint point, float val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountFloat32"); - } - /// - /// Sets the hit count of a point cloud point if is true. - /// - /// The point cloud point. - /// The new hit count. - public virtual void SetHitCountFloat64(ref TPoint point, double val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetHitCountFloat64"); - } - #endregion - - #endregion - - #region Get/Set GPS Time - - #region Getter - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref sbyte GetGPSTimeInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeInt_8"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref short GetGPSTimeInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeInt_16"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref int GetGPSTimeInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeInt_32"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref long GetGPSTimeInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeInt_64"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref byte GetGPSTimeUInt_8(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeUInt_8"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ushort GetGPSTimeUInt_16(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeUInt_16"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref uint GetGPSTimeUInt_32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeUInt_32"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref ulong GetGPSTimeUInt_64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeUInt_64"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref float GetGPSTimeFloat32(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeFloat32"); - } - /// - /// Returns the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - public virtual ref double GetGPSTimeFloat64(ref TPoint point) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support GetGPSTimeFloat64"); - } - #endregion - - #region Setter - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeInt_8(ref TPoint point, sbyte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeInt_8"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeInt_16(ref TPoint point, short val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeInt_16"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeInt_32(ref TPoint point, int val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeInt_32"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeInt_64(ref TPoint point, long val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeInt_64"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeUInt_8(ref TPoint point, byte val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeUInt_8"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeUInt_16(ref TPoint point, ushort val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeUInt_16"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeUInt_32(ref TPoint point, uint val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeUInt_32"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeUInt_64(ref TPoint point, ulong val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeUInt_64"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeFloat32(ref TPoint point, float val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeFloat32"); - } - /// - /// Sets the GPS time of a point cloud point if is true. - /// - /// The point cloud point. - /// The new GPS time. - public virtual void SetGPSTimeFloat64(ref TPoint point, double val) - { - throw new NotSupportedException($"Point {typeof(TPoint).Name} does not support SetGPSTimeFloat64"); - } - #endregion - - #endregion - - #endregion - - #region RawDataEncode - private byte[] GetRawPosition(ref TPoint point) - { - if (PositionType == PointPositionType.Float3) - { - - var x = BitConverter.GetBytes(GetPositionFloat3_32(ref point).x); - var y = BitConverter.GetBytes(GetPositionFloat3_32(ref point).y); - var z = BitConverter.GetBytes(GetPositionFloat3_32(ref point).z); - - return x.Concat(y).Concat(z).ToArray(); - - } - else if (PositionType == PointPositionType.Double3) - { - - var x = BitConverter.GetBytes(GetPositionFloat3_64(ref point).x); - var y = BitConverter.GetBytes(GetPositionFloat3_64(ref point).y); - var z = BitConverter.GetBytes(GetPositionFloat3_64(ref point).z); - - return x.Concat(y).Concat(z).ToArray(); - - } - - byte[] vs = Array.Empty(); - return vs; - - } - - private byte[] GetRawIntensity(ref TPoint point) - { - switch (IntensityType) - { - case PointIntensityType.SByte: - return BitConverter.GetBytes(((short)GetIntensityInt_8(ref point))); - case PointIntensityType.Short: - return BitConverter.GetBytes(GetIntensityInt_16(ref point)); - case PointIntensityType.Int: - return BitConverter.GetBytes(GetIntensityInt_32(ref point)); - case PointIntensityType.Long: - return BitConverter.GetBytes(GetIntensityInt_64(ref point)); - case PointIntensityType.Byte: - return BitConverter.GetBytes(((short)GetIntensityUInt_8(ref point))); - case PointIntensityType.UShort: - return BitConverter.GetBytes(GetIntensityUInt_16(ref point)); - case PointIntensityType.UInt: - return BitConverter.GetBytes(GetIntensityUInt_32(ref point)); - case PointIntensityType.ULong: - return BitConverter.GetBytes(GetIntensityUInt_64(ref point)); - case PointIntensityType.Float: - return BitConverter.GetBytes(GetIntensityFloat32(ref point)); - case PointIntensityType.Double: - return BitConverter.GetBytes(GetIntensityFloat64(ref point)); - } - - return Array.Empty(); - } - - private byte[] GetRawNormals(ref TPoint point) - { - - if (NormalType == PointNormalType.Float3) - { - var x = BitConverter.GetBytes(GetNormalFloat3_32(ref point).x); - var y = BitConverter.GetBytes(GetNormalFloat3_32(ref point).y); - var z = BitConverter.GetBytes(GetNormalFloat3_32(ref point).z); - - return x.Concat(y).Concat(z).ToArray(); - - } - else if (NormalType == PointNormalType.Double3) - { - var x = BitConverter.GetBytes(GetNormalFloat3_64(ref point).x); - var y = BitConverter.GetBytes(GetNormalFloat3_64(ref point).y); - var z = BitConverter.GetBytes(GetNormalFloat3_64(ref point).z); - - return x.Concat(y).Concat(z).ToArray(); - - } - return Array.Empty(); - - } - - private byte[] GetRawColor(ref TPoint point) - { - switch (ColorType) - { - case PointColorType.SByte: - return BitConverter.GetBytes(((short)GetColorInt_8(ref point))); - case PointColorType.Short: - return BitConverter.GetBytes(GetColorInt_16(ref point)); - case PointColorType.Int: - return BitConverter.GetBytes(GetColorInt_32(ref point)); - case PointColorType.Long: - return BitConverter.GetBytes(GetColorInt_64(ref point)); - case PointColorType.Byte: - return BitConverter.GetBytes(((short)GetColorUInt_8(ref point))); - case PointColorType.Ushort: - return BitConverter.GetBytes(GetColorUInt_16(ref point)); - case PointColorType.Uint: - return BitConverter.GetBytes(GetColorUInt_32(ref point)); - case PointColorType.Ulong: - return BitConverter.GetBytes(GetColorUInt_64(ref point)); - case PointColorType.Float: - return BitConverter.GetBytes(GetColorFloat32(ref point)); - case PointColorType.Double: - return BitConverter.GetBytes(GetColorFloat64(ref point)); - } - - if (ColorType == PointColorType.Float3) - { - - var x = BitConverter.GetBytes(GetColorFloat3_32(ref point).x); - var y = BitConverter.GetBytes(GetColorFloat3_32(ref point).y); - var z = BitConverter.GetBytes(GetColorFloat3_32(ref point).z); - - return x.Concat(y).Concat(z).ToArray(); - - } - if (ColorType == PointColorType.Double3) - { - - var x = BitConverter.GetBytes(GetColorFloat3_64(ref point).x); - var y = BitConverter.GetBytes(GetColorFloat3_64(ref point).y); - var z = BitConverter.GetBytes(GetColorFloat3_64(ref point).z); - - return x.Concat(y).Concat(z).ToArray(); - - } - return Array.Empty(); - } - - private byte[] GetRawLabel(ref TPoint point) - { - switch (LabelType) - { - case PointLabelType.SByte: - return BitConverter.GetBytes(((short)GetLabelInt_8(ref point))); - case PointLabelType.Short: - return BitConverter.GetBytes(GetLabelInt_16(ref point)); - case PointLabelType.Int: - return BitConverter.GetBytes(GetLabelInt_32(ref point)); - case PointLabelType.Long: - return BitConverter.GetBytes(GetLabelInt_64(ref point)); - case PointLabelType.Byte: - return BitConverter.GetBytes(((short)GetLabelUInt_8(ref point))); - case PointLabelType.UShort: - return BitConverter.GetBytes(GetLabelUInt_16(ref point)); - case PointLabelType.UInt: - return BitConverter.GetBytes(GetLabelUInt_32(ref point)); - case PointLabelType.ULong: - return BitConverter.GetBytes(GetLabelUInt_64(ref point)); - case PointLabelType.Float: - return BitConverter.GetBytes(GetLabelFloat32(ref point)); - case PointLabelType.Double: - return BitConverter.GetBytes(GetLabelFloat64(ref point)); - } - - return Array.Empty(); - } - - private byte[] GetRawCurvature(ref TPoint point) - { - switch (CurvatureType) - { - case PointCurvatureType.SByte: - return BitConverter.GetBytes(((short)GetCurvatureInt_8(ref point))); - case PointCurvatureType.Short: - return BitConverter.GetBytes(GetCurvatureInt_16(ref point)); - case PointCurvatureType.Int: - return BitConverter.GetBytes(GetCurvatureInt_32(ref point)); - case PointCurvatureType.Long: - return BitConverter.GetBytes(GetCurvatureInt_64(ref point)); - case PointCurvatureType.Byte: - return BitConverter.GetBytes(((short)GetCurvatureUInt_8(ref point))); - case PointCurvatureType.UShort: - return BitConverter.GetBytes(GetCurvatureUInt_16(ref point)); - case PointCurvatureType.Uint: - return BitConverter.GetBytes(GetCurvatureUInt_32(ref point)); - case PointCurvatureType.ULong: - return BitConverter.GetBytes(GetCurvatureUInt_64(ref point)); - case PointCurvatureType.Float: - return BitConverter.GetBytes(GetCurvatureFloat32(ref point)); - case PointCurvatureType.Double: - return BitConverter.GetBytes(GetCurvatureFloat64(ref point)); - } - - return Array.Empty(); - } - - private byte[] GetRawHitCount(ref TPoint point) - { - switch (HitCountType) - { - case PointHitCountType.SByte: - return BitConverter.GetBytes(((short)GetHitCountInt_8(ref point))); - case PointHitCountType.Short: - return BitConverter.GetBytes(GetHitCountInt_16(ref point)); - case PointHitCountType.Int: - return BitConverter.GetBytes(GetHitCountInt_32(ref point)); - case PointHitCountType.Long: - return BitConverter.GetBytes(GetHitCountInt_64(ref point)); - case PointHitCountType.Byte: - return BitConverter.GetBytes(((short)GetHitCountUInt_8(ref point))); - case PointHitCountType.UShort: - return BitConverter.GetBytes(GetHitCountUInt_16(ref point)); - case PointHitCountType.Uint: - return BitConverter.GetBytes(GetHitCountUInt_32(ref point)); - case PointHitCountType.ULong: - return BitConverter.GetBytes(GetHitCountUInt_64(ref point)); - case PointHitCountType.Float: - return BitConverter.GetBytes(GetHitCountFloat32(ref point)); - case PointHitCountType.Double: - return BitConverter.GetBytes(GetHitCountFloat64(ref point)); - } - - return Array.Empty(); - } - - private byte[] GetRawGPSTime(ref TPoint point) - { - switch (GpsTimeType) - { - case PointGpsTimeType.SByte: - return BitConverter.GetBytes(((short)GetGPSTimeInt_8(ref point))); - case PointGpsTimeType.Short: - return BitConverter.GetBytes(GetGPSTimeInt_16(ref point)); - case PointGpsTimeType.Int: - return BitConverter.GetBytes(GetGPSTimeInt_32(ref point)); - case PointGpsTimeType.Long: - return BitConverter.GetBytes(GetGPSTimeInt_64(ref point)); - case PointGpsTimeType.Byte: - return BitConverter.GetBytes(((short)GetGPSTimeUInt_8(ref point))); - case PointGpsTimeType.UShort: - return BitConverter.GetBytes(GetGPSTimeUInt_16(ref point)); - case PointGpsTimeType.Uint: - return BitConverter.GetBytes(GetGPSTimeUInt_32(ref point)); - case PointGpsTimeType.ULong: - return BitConverter.GetBytes(GetGPSTimeUInt_64(ref point)); - case PointGpsTimeType.Float: - return BitConverter.GetBytes(GetGPSTimeFloat32(ref point)); - case PointGpsTimeType.Double: - return BitConverter.GetBytes(GetGPSTimeFloat64(ref point)); - } - - return Array.Empty(); - } - #endregion - - #region RawDataDecode - - /// - /// Needed for correct array offsets during read - /// - private struct ByteArrayOffsets - { - internal int PositionOffset; - internal int IntensityOffset; - internal int NormalsOffset; - internal int RGBOffset; - internal int LabelOffset; - internal int CurvatureOffset; - internal int HitCountOffset; - internal int GPSTimeOffset; - } - - private bool _offsetsCalculated = false; - private ByteArrayOffsets _offsets; - - private ByteArrayOffsets Offsets - { - get - { - if (_offsetsCalculated) - return _offsets; - - _offsets = new ByteArrayOffsets(); - - // position - switch (PositionType) - { - case PointPositionType.Float3: - _offsets.PositionOffset = 3 * Marshal.SizeOf(); - break; - case PointPositionType.Double3: - _offsets.PositionOffset = 3 * Marshal.SizeOf(); - break; - } - - // Intensity - switch (IntensityType) - { - case PointIntensityType.SByte: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.Short: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.Int: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.Long: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.Byte: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.UShort: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.UInt: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.ULong: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.Float: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - case PointIntensityType.Double: - _offsets.IntensityOffset = Marshal.SizeOf(); - break; - } - - // Normal - switch (NormalType) - { - case PointNormalType.Float3: - _offsets.NormalsOffset = 3 * Marshal.SizeOf(); - break; - case PointNormalType.Double3: - _offsets.NormalsOffset = 3 * Marshal.SizeOf(); - break; - } - - // Color - switch (ColorType) - { - case PointColorType.SByte: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Short: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Int: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Long: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Byte: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Ushort: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Uint: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Ulong: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Float: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Double: - _offsets.RGBOffset = Marshal.SizeOf(); - break; - case PointColorType.Float3: - _offsets.RGBOffset = 3 * Marshal.SizeOf(); - break; - case PointColorType.Double3: - _offsets.RGBOffset = 3 * Marshal.SizeOf(); - break; - } - - // Label - switch (LabelType) - { - case PointLabelType.SByte: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.Short: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.Int: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.Long: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.Byte: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.UShort: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.UInt: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.ULong: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.Float: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - case PointLabelType.Double: - _offsets.LabelOffset = Marshal.SizeOf(); - break; - } - - // Curvature - switch (CurvatureType) - { - case PointCurvatureType.SByte: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Short: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Int: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Long: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Byte: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.UShort: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Uint: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.ULong: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Float: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - case PointCurvatureType.Double: - _offsets.CurvatureOffset = Marshal.SizeOf(); - break; - } - - // Hit count - switch (HitCountType) - { - case PointHitCountType.SByte: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Short: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Int: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Long: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Byte: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.UShort: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Uint: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.ULong: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Float: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - case PointHitCountType.Double: - _offsets.HitCountOffset = Marshal.SizeOf(); - break; - } - - // GPSTime - switch (GpsTimeType) - { - case PointGpsTimeType.SByte: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Short: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Int: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Long: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Byte: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.UShort: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Uint: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.ULong: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Float: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - case PointGpsTimeType.Double: - _offsets.GPSTimeOffset = Marshal.SizeOf(); - break; - } - - _offsetsCalculated = true; - - return _offsets; - } - } - - private void SetRawPosition(ref TPoint pointIn, byte[] byteIn) - { - if (PositionType == PointPositionType.Float3) - { - var offset = Marshal.SizeOf(); - var x = BitConverter.ToSingle(byteIn, 0); - var y = BitConverter.ToSingle(byteIn, offset); - var z = BitConverter.ToSingle(byteIn, offset * 2); - - SetPositionFloat3_32(ref pointIn, new float3(x, y, z)); - } - else if (PositionType == PointPositionType.Double3) - { - var offset = Marshal.SizeOf(); - var x = BitConverter.ToDouble(byteIn, 0); - var y = BitConverter.ToDouble(byteIn, offset); - var z = BitConverter.ToDouble(byteIn, offset * 2); - - SetPositionFloat3_64(ref pointIn, new double3(x, y, z)); - } - } - - private void SetRawIntensity(ref TPoint pointIn, byte[] byteIn) - { - switch (IntensityType) - { - case PointIntensityType.SByte: - SetIntensityInt_8(ref pointIn, (sbyte)byteIn[Offsets.PositionOffset]); - break; - case PointIntensityType.Short: - SetIntensityInt_16(ref pointIn, byteIn[Offsets.PositionOffset]); - break; - case PointIntensityType.Int: - SetIntensityInt_32(ref pointIn, BitConverter.ToInt32(byteIn, Offsets.PositionOffset)); - break; - case PointIntensityType.Long: - SetIntensityInt_64(ref pointIn, BitConverter.ToInt64(byteIn, Offsets.PositionOffset)); - break; - case PointIntensityType.Byte: - SetIntensityUInt_8(ref pointIn, byteIn[Offsets.PositionOffset]); - break; - case PointIntensityType.UShort: - SetIntensityUInt_16(ref pointIn, BitConverter.ToUInt16(byteIn, Offsets.PositionOffset)); - break; - case PointIntensityType.UInt: - SetIntensityUInt_32(ref pointIn, BitConverter.ToUInt32(byteIn, Offsets.PositionOffset)); - break; - case PointIntensityType.ULong: - SetIntensityUInt_64(ref pointIn, BitConverter.ToUInt64(byteIn, Offsets.PositionOffset)); - break; - case PointIntensityType.Float: - SetIntensityFloat32(ref pointIn, BitConverter.ToSingle(byteIn, Offsets.PositionOffset)); - break; - case PointIntensityType.Double: - SetIntensityFloat64(ref pointIn, BitConverter.ToDouble(byteIn, Offsets.PositionOffset)); - break; - } - } - - private void SetRawNormals(ref TPoint pointIn, byte[] byteIn) - { - var offset = Offsets.PositionOffset + Offsets.IntensityOffset; - - if (NormalType == PointNormalType.Float3) - { - - var dataOffset = Marshal.SizeOf(); - var x = BitConverter.ToSingle(byteIn, offset); - var y = BitConverter.ToSingle(byteIn, offset + dataOffset); - var z = BitConverter.ToSingle(byteIn, offset + dataOffset * 2); - - SetNormalFloat3_32(ref pointIn, new float3(x, y, z)); - - } - if (NormalType == PointNormalType.Double3) - { - - var dataOffset = Marshal.SizeOf(); - - var x = BitConverter.ToDouble(byteIn, offset); - var y = BitConverter.ToDouble(byteIn, offset + dataOffset); - var z = BitConverter.ToDouble(byteIn, offset + dataOffset * 2); - - SetNormalFloat3_64(ref pointIn, new double3(x, y, z)); - - } - } - - private void SetRawColor(ref TPoint pointIn, byte[] byteIn) - { - var offset = Offsets.PositionOffset + Offsets.IntensityOffset + Offsets.NormalsOffset; - - switch (ColorType) - { - case PointColorType.SByte: - SetColorInt_8(ref pointIn, (sbyte)byteIn[offset]); - break; - case PointColorType.Short: - SetColorInt_16(ref pointIn, byteIn[offset]); - break; - case PointColorType.Int: - SetColorInt_32(ref pointIn, BitConverter.ToInt32(byteIn, offset)); - break; - case PointColorType.Long: - SetColorInt_64(ref pointIn, BitConverter.ToInt64(byteIn, offset)); - break; - case PointColorType.Byte: - SetColorUInt_8(ref pointIn, byteIn[offset + 1]); - break; - case PointColorType.Ushort: - SetColorUInt_16(ref pointIn, BitConverter.ToUInt16(byteIn, offset)); - break; - case PointColorType.Uint: - SetColorUInt_32(ref pointIn, BitConverter.ToUInt32(byteIn, offset)); - break; - case PointColorType.Ulong: - SetColorUInt_64(ref pointIn, BitConverter.ToUInt64(byteIn, offset)); - break; - case PointColorType.Float: - SetColorFloat32(ref pointIn, BitConverter.ToSingle(byteIn, offset)); - break; - case PointColorType.Double: - SetColorFloat64(ref pointIn, BitConverter.ToDouble(byteIn, offset)); - break; - case PointColorType.Float3: - { - - var dataOffset = Marshal.SizeOf(); - - var x = BitConverter.ToSingle(byteIn, offset); - var y = BitConverter.ToSingle(byteIn, offset + dataOffset); - var z = BitConverter.ToSingle(byteIn, offset + dataOffset * 2); - - SetColorFloat3_32(ref pointIn, new float3(x, y, z)); - - } - break; - - case PointColorType.Double3: - { - - var dataOffset = Marshal.SizeOf(); - - var x = BitConverter.ToDouble(byteIn, offset); - var y = BitConverter.ToDouble(byteIn, offset + dataOffset); - var z = BitConverter.ToDouble(byteIn, offset + dataOffset * 2); - - SetColorFloat3_64(ref pointIn, new double3(x, y, z)); - - } - break; - } - } - - private void SetRawLabel(ref TPoint pointIn, byte[] byteIn) - { - var offset = Offsets.PositionOffset + Offsets.IntensityOffset + Offsets.NormalsOffset + Offsets.RGBOffset; - - switch (LabelType) - { - case PointLabelType.SByte: - SetLabelInt_8(ref pointIn, (sbyte)byteIn[offset]); - break; - case PointLabelType.Short: - SetLabelInt_16(ref pointIn, byteIn[offset]); - break; - case PointLabelType.Int: - SetLabelInt_32(ref pointIn, BitConverter.ToInt32(byteIn, offset)); - break; - case PointLabelType.Long: - SetLabelInt_64(ref pointIn, BitConverter.ToInt64(byteIn, offset)); - break; - case PointLabelType.Byte: - SetLabelUInt_8(ref pointIn, byteIn[offset]); - break; - case PointLabelType.UShort: - SetLabelUInt_16(ref pointIn, BitConverter.ToUInt16(byteIn, offset)); - break; - case PointLabelType.UInt: - SetLabelUInt_32(ref pointIn, BitConverter.ToUInt32(byteIn, offset)); - break; - case PointLabelType.ULong: - SetLabelUInt_64(ref pointIn, BitConverter.ToUInt64(byteIn, offset)); - break; - case PointLabelType.Float: - SetLabelFloat32(ref pointIn, BitConverter.ToSingle(byteIn, offset)); - break; - case PointLabelType.Double: - SetLabelFloat64(ref pointIn, BitConverter.ToDouble(byteIn, offset)); - break; - } - } - - private void SetRawCurvature(ref TPoint pointIn, byte[] byteIn) - { - var offset = Offsets.PositionOffset + Offsets.IntensityOffset + Offsets.NormalsOffset + Offsets.RGBOffset + Offsets.LabelOffset; - - switch (CurvatureType) - { - case PointCurvatureType.SByte: - SetCurvatureInt_8(ref pointIn, (sbyte)byteIn[offset]); - break; - case PointCurvatureType.Short: - SetCurvatureInt_16(ref pointIn, byteIn[offset]); - break; - case PointCurvatureType.Int: - SetCurvatureInt_32(ref pointIn, BitConverter.ToInt32(byteIn, offset)); - break; - case PointCurvatureType.Long: - SetCurvatureInt_64(ref pointIn, BitConverter.ToInt64(byteIn, offset)); - break; - case PointCurvatureType.Byte: - SetCurvatureUInt_8(ref pointIn, byteIn[offset + 1]); - break; - case PointCurvatureType.UShort: - SetCurvatureUInt_16(ref pointIn, BitConverter.ToUInt16(byteIn, offset)); - break; - case PointCurvatureType.Uint: - SetCurvatureUInt_32(ref pointIn, BitConverter.ToUInt32(byteIn, offset)); - break; - case PointCurvatureType.ULong: - SetCurvatureUInt_64(ref pointIn, BitConverter.ToUInt64(byteIn, offset)); - break; - case PointCurvatureType.Float: - SetCurvatureFloat32(ref pointIn, BitConverter.ToSingle(byteIn, offset)); - break; - case PointCurvatureType.Double: - SetCurvatureFloat64(ref pointIn, BitConverter.ToDouble(byteIn, offset)); - break; - } - } - - private void SetRawHitCount(ref TPoint pointIn, byte[] byteIn) - { - var offset = Offsets.PositionOffset + Offsets.IntensityOffset + Offsets.NormalsOffset + Offsets.RGBOffset + Offsets.LabelOffset; - - switch (HitCountType) - { - case PointHitCountType.SByte: - SetHitCountInt_8(ref pointIn, (sbyte)byteIn[offset]); - break; - case PointHitCountType.Short: - SetHitCountInt_16(ref pointIn, byteIn[offset]); - break; - case PointHitCountType.Int: - SetHitCountInt_32(ref pointIn, BitConverter.ToInt32(byteIn, offset)); - break; - case PointHitCountType.Long: - SetHitCountInt_64(ref pointIn, BitConverter.ToInt64(byteIn, offset)); - break; - case PointHitCountType.Byte: - SetHitCountUInt_8(ref pointIn, byteIn[offset + 1]); - break; - case PointHitCountType.UShort: - SetHitCountUInt_16(ref pointIn, BitConverter.ToUInt16(byteIn, offset)); - break; - case PointHitCountType.Uint: - SetHitCountUInt_32(ref pointIn, BitConverter.ToUInt32(byteIn, offset)); - break; - case PointHitCountType.ULong: - SetHitCountUInt_64(ref pointIn, BitConverter.ToUInt64(byteIn, offset)); - break; - case PointHitCountType.Float: - SetHitCountFloat32(ref pointIn, BitConverter.ToSingle(byteIn, offset)); - break; - case PointHitCountType.Double: - SetHitCountFloat64(ref pointIn, BitConverter.ToDouble(byteIn, offset)); - break; - } - } - - private void SetRawGPSTime(ref TPoint pointIn, byte[] byteIn) - { - var offset = Offsets.PositionOffset + Offsets.IntensityOffset + Offsets.NormalsOffset + Offsets.RGBOffset + Offsets.LabelOffset + Offsets.HitCountOffset; - - switch (GpsTimeType) - { - case PointGpsTimeType.SByte: - SetGPSTimeInt_8(ref pointIn, (sbyte)byteIn[offset]); - break; - case PointGpsTimeType.Short: - SetGPSTimeInt_16(ref pointIn, byteIn[offset]); - break; - case PointGpsTimeType.Int: - SetGPSTimeInt_32(ref pointIn, BitConverter.ToInt32(byteIn, offset)); - break; - case PointGpsTimeType.Long: - SetGPSTimeInt_64(ref pointIn, BitConverter.ToInt64(byteIn, offset)); - break; - case PointGpsTimeType.Byte: - SetGPSTimeUInt_8(ref pointIn, byteIn[offset + 1]); - break; - case PointGpsTimeType.UShort: - SetGPSTimeUInt_16(ref pointIn, BitConverter.ToUInt16(byteIn, offset)); - break; - case PointGpsTimeType.Uint: - SetGPSTimeUInt_32(ref pointIn, BitConverter.ToUInt32(byteIn, offset)); - break; - case PointGpsTimeType.ULong: - SetGPSTimeUInt_64(ref pointIn, BitConverter.ToUInt64(byteIn, offset)); - break; - case PointGpsTimeType.Float: - SetGPSTimeFloat32(ref pointIn, BitConverter.ToSingle(byteIn, offset)); - break; - case PointGpsTimeType.Double: - SetGPSTimeFloat64(ref pointIn, BitConverter.ToDouble(byteIn, offset)); - break; - } - } - - #endregion - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3Accessor.cs b/src/PointCloud/Core/Accessors/PosD3Accessor.cs deleted file mode 100644 index c7a8811dd..000000000 --- a/src/PointCloud/Core/Accessors/PosD3Accessor.cs +++ /dev/null @@ -1,40 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - //Collection of PointAccessor classes. There has to be one for each PointType. - - /// - /// for Point Clouds which position information only. - /// - public class PosD3Accessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3Accessor() - { - PositionType = PointPositionType.Double3; - } - - /// - /// Sets the position of a point cloud point. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3 point, double3 val) - { - point.Position = val; - } - - /// - /// Returns the position of a point cloud point. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3 point) - { - return ref point.Position; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3ColF3Accessor.cs b/src/PointCloud/Core/Accessors/PosD3ColF3Accessor.cs deleted file mode 100644 index 7140a877d..000000000 --- a/src/PointCloud/Core/Accessors/PosD3ColF3Accessor.cs +++ /dev/null @@ -1,56 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position and color values. - /// - public class PosD3ColF3Accessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3ColF3Accessor() - { - PositionType = PointPositionType.Double3; - ColorType = PointColorType.Float3; - } - - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public override void SetColorFloat3_32(ref PosD3ColF3 point, float3 val) - { - point.Color = val; - } - - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetColorFloat3_32(ref PosD3ColF3 point) - { - return ref point.Color; - } - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3ColF3 point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3ColF3 point) - { - return ref point.Position; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3ColF3InUsAccessor.cs b/src/PointCloud/Core/Accessors/PosD3ColF3InUsAccessor.cs deleted file mode 100644 index bd8a206c9..000000000 --- a/src/PointCloud/Core/Accessors/PosD3ColF3InUsAccessor.cs +++ /dev/null @@ -1,73 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position, color and intensity values. - /// - public class PosD3ColF3InUsAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3ColF3InUsAccessor() - { - PositionType = PointPositionType.Double3; - ColorType = PointColorType.Float3; - IntensityType = PointIntensityType.UShort; - } - - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public override void SetColorFloat3_32(ref PosD3ColF3InUs point, float3 val) - { - point.Color = val; - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetColorFloat3_32(ref PosD3ColF3InUs point) - { - return ref point.Color; - } - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3ColF3InUs point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3ColF3InUs point) - { - return ref point.Position; - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public override ref ushort GetIntensityUInt_16(ref PosD3ColF3InUs point) - { - return ref point.Intensity; - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public override void SetIntensityUInt_16(ref PosD3ColF3InUs point, ushort val) - { - point.Intensity = val; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3ColF3InUsLblBAccessor.cs b/src/PointCloud/Core/Accessors/PosD3ColF3InUsLblBAccessor.cs deleted file mode 100644 index 32ea4c162..000000000 --- a/src/PointCloud/Core/Accessors/PosD3ColF3InUsLblBAccessor.cs +++ /dev/null @@ -1,93 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position, color and classification values. - /// - public class PosD3ColF3InUsLblBAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3ColF3InUsLblBAccessor() - { - PositionType = PointPositionType.Double3; - ColorType = PointColorType.Float3; - LabelType = PointLabelType.Byte; - IntensityType = PointIntensityType.UShort; - } - - /// - /// Sets the color of a point cloud point. - /// - /// The point cloud point. - /// The new color. - public override void SetColorFloat3_32(ref PosD3ColF3InUsLblB point, float3 val) - { - point.Color = val; - } - /// - /// Returns the normal color of a point cloud point. - /// - /// The point cloud point. - public override ref float3 GetColorFloat3_32(ref PosD3ColF3InUsLblB point) - { - return ref point.Color; - } - - /// - /// Sets the position of a point cloud point. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3ColF3InUsLblB point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3ColF3InUsLblB point) - { - return ref point.Position; - } - /// - /// Sets the label of a point cloud point. - /// - /// The point cloud point. - /// The new label. - public override void SetLabelUInt_8(ref PosD3ColF3InUsLblB point, byte val) - { - point.Label = val; - } - /// - /// Returns the label of a point cloud point. - /// - /// The point cloud point. - public override ref byte GetLabelUInt_8(ref PosD3ColF3InUsLblB point) - { - return ref point.Label; - } - - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public override ref ushort GetIntensityUInt_16(ref PosD3ColF3InUsLblB point) - { - return ref point.Intensity; - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public override void SetIntensityUInt_16(ref PosD3ColF3InUsLblB point, ushort val) - { - point.Intensity = val; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3ColF3LblBAccessor.cs b/src/PointCloud/Core/Accessors/PosD3ColF3LblBAccessor.cs deleted file mode 100644 index 1c386f2ff..000000000 --- a/src/PointCloud/Core/Accessors/PosD3ColF3LblBAccessor.cs +++ /dev/null @@ -1,74 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position, color and classification values. - /// - public class PosD3ColF3LblBAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3ColF3LblBAccessor() - { - PositionType = PointPositionType.Double3; - ColorType = PointColorType.Float; - LabelType = PointLabelType.Byte; - } - - /// - /// Sets the color of a point cloud point. - /// - /// The point cloud point. - /// The new color. - public override void SetColorFloat3_32(ref PosD3ColF3LblB point, float3 val) - { - point.Color = val; - } - /// - /// Returns the normal color of a point cloud point. - /// - /// The point cloud point. - public override ref float3 GetColorFloat3_32(ref PosD3ColF3LblB point) - { - return ref point.Color; - } - - /// - /// Sets the position of a point cloud point. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3ColF3LblB point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3ColF3LblB point) - { - return ref point.Position; - } - /// - /// Sets the label of a point cloud point. - /// - /// The point cloud point. - /// The new label. - public override void SetLabelUInt_8(ref PosD3ColF3LblB point, byte val) - { - point.Label = val; - } - /// - /// Returns the label of a point cloud point. - /// - /// The point cloud point. - public override ref byte GetLabelUInt_8(ref PosD3ColF3LblB point) - { - return ref point.Label; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3InUsAccessor.cs b/src/PointCloud/Core/Accessors/PosD3InUsAccessor.cs deleted file mode 100644 index 8039ae9e7..000000000 --- a/src/PointCloud/Core/Accessors/PosD3InUsAccessor.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position and intensity values. - /// - public class PosD3InUsAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3InUsAccessor() - { - PositionType = PointPositionType.Double3; - IntensityType = PointIntensityType.UShort; - } - - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3InUs point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3InUs point) - { - return ref point.Position; - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public override ref ushort GetIntensityUInt_16(ref PosD3InUs point) - { - return ref point.Intensity; - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public override void SetIntensityUInt_16(ref PosD3InUs point, ushort val) - { - point.Intensity = val; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3LblBAccessor.cs b/src/PointCloud/Core/Accessors/PosD3LblBAccessor.cs deleted file mode 100644 index f22a3dec2..000000000 --- a/src/PointCloud/Core/Accessors/PosD3LblBAccessor.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position and label values. - /// - public class PosD3LblBAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3LblBAccessor() - { - PositionType = PointPositionType.Double3; - LabelType = PointLabelType.Byte; - } - - /// - /// Sets the label of a point cloud point if is true. - /// - /// The point cloud point. - /// The new label. - public override void SetLabelUInt_8(ref PosD3LblB point, byte val) - { - point.Label = val; - } - /// - /// Returns the label of a point cloud point if is true. - /// - /// The point cloud point. - public override ref byte GetLabelUInt_8(ref PosD3LblB point) - { - return ref point.Label; - } - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3LblB point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3LblB point) - { - return ref point.Position; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3NorF3ColF3Accessor.cs b/src/PointCloud/Core/Accessors/PosD3NorF3ColF3Accessor.cs deleted file mode 100644 index 8338a5e26..000000000 --- a/src/PointCloud/Core/Accessors/PosD3NorF3ColF3Accessor.cs +++ /dev/null @@ -1,74 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position, color and intensity values. - /// - public class PosD3NorF3ColF3Accessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3NorF3ColF3Accessor() - { - PositionType = PointPositionType.Double3; - ColorType = PointColorType.Float3; - NormalType = PointNormalType.Float3; - } - - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public override void SetColorFloat3_32(ref PosD3NorF3ColF3 point, float3 val) - { - point.Color = val; - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetColorFloat3_32(ref PosD3NorF3ColF3 point) - { - return ref point.Color; - } - - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3NorF3ColF3 point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3NorF3ColF3 point) - { - return ref point.Position; - } - /// - /// Returns the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetNormalFloat3_32(ref PosD3NorF3ColF3 point) - { - return ref point.Normal; - } - /// - /// Sets the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - /// The new normal vector. - public override void SetNormalFloat3_32(ref PosD3NorF3ColF3 point, float3 val) - { - point.Normal = val; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3NorF3ColF3InUsAccessor.cs b/src/PointCloud/Core/Accessors/PosD3NorF3ColF3InUsAccessor.cs deleted file mode 100644 index f71f66993..000000000 --- a/src/PointCloud/Core/Accessors/PosD3NorF3ColF3InUsAccessor.cs +++ /dev/null @@ -1,91 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position, normal vectors, color and intensity values. - /// - public class PosD3NorF3ColF3InUsAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3NorF3ColF3InUsAccessor() - { - PositionType = PointPositionType.Double3; - ColorType = PointColorType.Float3; - IntensityType = PointIntensityType.UShort; - NormalType = PointNormalType.Float3; - } - - /// - /// Sets the color of a point cloud point if is true. - /// - /// The point cloud point. - /// The new color. - public override void SetColorFloat3_32(ref PosD3NorF3ColF3InUs point, float3 val) - { - point.Color = val; - } - /// - /// Returns the normal color of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetColorFloat3_32(ref PosD3NorF3ColF3InUs point) - { - return ref point.Color; - } - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3NorF3ColF3InUs point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3NorF3ColF3InUs point) - { - return ref point.Position; - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public override ref ushort GetIntensityUInt_16(ref PosD3NorF3ColF3InUs point) - { - return ref point.Intensity; - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public override void SetIntensityUInt_16(ref PosD3NorF3ColF3InUs point, ushort val) - { - point.Intensity = val; - } - /// - /// Returns the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetNormalFloat3_32(ref PosD3NorF3ColF3InUs point) - { - return ref point.Normal; - } - /// - /// Sets the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - /// The new normal vector. - public override void SetNormalFloat3_32(ref PosD3NorF3ColF3InUs point, float3 val) - { - point.Normal = val; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Accessors/PosD3NorF3InUsAccessor.cs b/src/PointCloud/Core/Accessors/PosD3NorF3InUsAccessor.cs deleted file mode 100644 index e294c5cb8..000000000 --- a/src/PointCloud/Core/Accessors/PosD3NorF3InUsAccessor.cs +++ /dev/null @@ -1,73 +0,0 @@ -using Fusee.Math.Core; -using Fusee.PointCloud.Common.Accessors; - -namespace Fusee.PointCloud.Core.Accessors -{ - /// - /// for Point Clouds which position, normal vectors and intensity values. - /// - public class PosD3NorF3InUsAccessor : PointAccessor - { - /// - /// Creates a new instance. - /// - public PosD3NorF3InUsAccessor() - { - PositionType = PointPositionType.Double3; - IntensityType = PointIntensityType.UShort; - NormalType = PointNormalType.Double3; - } - - /// - /// Sets the position of a point cloud point if is true. - /// - /// The point cloud point. - /// The new position value. - public override void SetPositionFloat3_64(ref PosD3NorF3InUs point, double3 val) - { - point.Position = val; - } - /// - /// Returns the position of a point cloud point if is true. - /// - /// The point cloud point. - public override ref double3 GetPositionFloat3_64(ref PosD3NorF3InUs point) - { - return ref point.Position; - } - /// - /// Returns the intensity of a point cloud point if is true. - /// - /// The point cloud point. - public override ref ushort GetIntensityUInt_16(ref PosD3NorF3InUs point) - { - return ref point.Intensity; - } - /// - /// Sets the intensity of a point cloud point if is true. - /// - /// The point cloud point. - /// The new intensity value. - public override void SetIntensityUInt_16(ref PosD3NorF3InUs point, ushort val) - { - point.Intensity = val; - } - /// - /// Returns the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - public override ref float3 GetNormalFloat3_32(ref PosD3NorF3InUs point) - { - return ref point.Normal; - } - /// - /// Sets the normal vector of a point cloud point if is true. - /// - /// The point cloud point. - /// The new normal vector. - public override void SetNormalFloat3_32(ref PosD3NorF3InUs point, float3 val) - { - point.Normal = val; - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Core/Fusee.PointCloud.Core.csproj b/src/PointCloud/Core/Fusee.PointCloud.Core.csproj index 7a280dd13..6c840e884 100644 --- a/src/PointCloud/Core/Fusee.PointCloud.Core.csproj +++ b/src/PointCloud/Core/Fusee.PointCloud.Core.csproj @@ -1,16 +1,19 @@ - - - - netstandard2.1;net7.0 - $(OutputPath)\$(RootNamespace).xml - - - - - - - - - - + + + + netstandard2.1;net7.0 + $(OutputPath)\$(RootNamespace).xml + enable + + + + + + + + + + + + diff --git a/src/PointCloud/Core/MeshMaker.cs b/src/PointCloud/Core/MeshMaker.cs index 6381debad..4324cba61 100644 --- a/src/PointCloud/Core/MeshMaker.cs +++ b/src/PointCloud/Core/MeshMaker.cs @@ -1,11 +1,9 @@ -using Fusee.Engine.Common; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Engine.Common; using Fusee.Engine.Core; using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using Fusee.PointCloud.Common; -using Fusee.PointCloud.Common.Accessors; -using Fusee.PointCloud.Core.Accessors; -using System; using System.Collections.Generic; namespace Fusee.PointCloud.Core @@ -18,11 +16,11 @@ public static class MeshMaker /// /// Generic method that creates meshes with 65k points maximum. /// - /// The point accessor allows access to the point data without casting to a explicit point type."/> /// The generic point cloud points. /// The method that defines how to create a GpuMesh from the point cloud points. + /// The octant identifier. /// - public static IEnumerable CreateMeshes(PointAccessor pointAccessor, TPoint[] points, CreateGpuData createGpuDataHandler, OctantId octantId) + public static IEnumerable CreateMeshes(MemoryOwner points, CreateGpuData createGpuDataHandler, OctantId octantId) { List meshes; @@ -41,18 +39,18 @@ public static IEnumerable CreateMeshes(PointAccessor else numberOfPointsInMesh = maxVertCount; - TPoint[] pointsPerMesh; + MemoryOwner pointsPerMesh; if (ptCnt > maxVertCount) { - pointsPerMesh = new TPoint[numberOfPointsInMesh]; - Array.Copy(points, i, pointsPerMesh, 0, numberOfPointsInMesh); + pointsPerMesh = MemoryOwner.Allocate(numberOfPointsInMesh); + points.Span.Slice(i, numberOfPointsInMesh).CopyTo(pointsPerMesh.Span[..]); } else { pointsPerMesh = points; } - meshes.Add(createGpuDataHandler(pointAccessor, pointsPerMesh, octantId)); + meshes.Add(createGpuDataHandler(pointsPerMesh, octantId)); meshCnt++; } return meshes; @@ -62,59 +60,29 @@ public static IEnumerable CreateMeshes(PointAccessor /// Returns the instance Data for a given point type by using the provided delegate. /// /// Can be of type or . The latter is used when rendering instanced. - /// The generic point type. - /// The point accessor allows access to the point data without casting to a explicit point type."/> /// The generic point cloud points. /// The method that defines how to create a InstanceData from the point cloud points. + /// The octant identifier. /// - public static IEnumerable CreateInstanceData(PointAccessor pointAccessor, TPoint[] points, CreateGpuData createGpuDataHandler, OctantId octantId) + public static IEnumerable CreateInstanceData(MemoryOwner points, CreateGpuData createGpuDataHandler, OctantId octantId) { return new List { - createGpuDataHandler(pointAccessor, points, octantId) + createGpuDataHandler(points, octantId) }; } /// - /// Returns meshes for point clouds of type . + /// Returns meshes for point clouds of type . /// - /// The point accessor allows access to the point data without casting to explicit a explicit point type."/> /// The lists of "raw" points. /// The id of the octant. - public static GpuMesh CreateMeshPosD3(PointAccessor pointAccessor, TPoint[] points) + public static GpuMesh CreateStaticMesh(MemoryOwner points, OctantId octantId) { int numberOfPointsInMesh; numberOfPointsInMesh = points.Length; - var firstPos = (float3)pointAccessor.GetPositionFloat3_64(ref points[0]); - var vertices = new float3[numberOfPointsInMesh]; - var triangles = new uint[numberOfPointsInMesh]; - var boundingBox = new AABBf(firstPos, firstPos); - - for (int i = 0; i < points.Length; i++) - { - var pos = (float3)pointAccessor.GetPositionFloat3_64(ref points[i]); - - vertices[i] = pos; - boundingBox |= pos; - triangles[i] = (uint)i; - } - var mesh = ModuleExtensionPoint.CreateGpuMesh(PrimitiveType.Points, vertices, triangles); - mesh.BoundingBox = boundingBox; - return mesh; - } - /// - /// Returns meshes for point clouds of type . - /// - /// The point accessor allows access to the point data without casting to explicit a explicit point type."/> - /// The lists of "raw" points. - /// The id of the octant. - public static GpuMesh CreateMeshPosD3ColF3LblB(PointAccessor pointAccessor, TPoint[] points, OctantId octantId) - { - int numberOfPointsInMesh; - numberOfPointsInMesh = points.Length; - - var firstPos = (float3)pointAccessor.GetPositionFloat3_64(ref points[0]); + var firstPos = points.Span[0].Position; var vertices = new float3[numberOfPointsInMesh]; var triangles = new uint[numberOfPointsInMesh]; var colors = new uint[numberOfPointsInMesh]; @@ -123,18 +91,15 @@ public static GpuMesh CreateMeshPosD3ColF3LblB(PointAccessor poi for (int i = 0; i < points.Length; i++) { - var pos = (float3)pointAccessor.GetPositionFloat3_64(ref points[i]); + var pos = points.Span[i].Position; vertices[i] = pos; boundingBox |= vertices[i]; triangles[i] = (uint)i; - var col = pointAccessor.GetColorFloat3_32(ref points[i]);//points[i].Color; + var col = points.Span[i].Color; colors[i] = ColorToUInt((int)col.r, (int)col.g, (int)col.b, 255); - flags[i] = 1 << 30; - - //TODO: add labels correctly - var label = pointAccessor.GetLabelUInt_8(ref points[i]);//points[i].Label; + flags[i] = points.Span[i].Flags; } var mesh = ModuleExtensionPoint.CreateGpuMesh(PrimitiveType.Points, vertices, triangles, null, colors, null, null, null, null, null, null, null, flags); mesh.BoundingBox = boundingBox; @@ -142,17 +107,16 @@ public static GpuMesh CreateMeshPosD3ColF3LblB(PointAccessor poi } /// - /// Returns meshes for point clouds of type . + /// Returns meshes for point clouds of type . /// - /// The point accessor allows access to the point data without casting to explicit a explicit point type."/> /// The lists of "raw" points. /// The id of the octant. - public static Mesh CreateDynamicMeshPosD3ColF3LblB(PointAccessor pointAccessor, TPoint[] points, OctantId octantId) + public static Mesh CreateDynamicMesh(MemoryOwner points, OctantId octantId) { int numberOfPointsInMesh; numberOfPointsInMesh = points.Length; - var firstPos = (float3)pointAccessor.GetPositionFloat3_64(ref points[0]); + var firstPos = points.Span[0].Position; var vertices = new float3[numberOfPointsInMesh]; var triangles = new uint[numberOfPointsInMesh]; var colors = new uint[numberOfPointsInMesh]; @@ -161,61 +125,55 @@ public static Mesh CreateDynamicMeshPosD3ColF3LblB(PointAccessor for (int i = 0; i < points.Length; i++) { - var pos = (float3)pointAccessor.GetPositionFloat3_64(ref points[i]); + var pos = points.Span[i].Position; vertices[i] = pos; boundingBox |= vertices[i]; triangles[i] = (uint)i; - var col = pointAccessor.GetColorFloat3_32(ref points[i]);//points[i].Color; + var col = points.Span[i].Color; colors[i] = ColorToUInt((int)col.r, (int)col.g, (int)col.b, 255); - flags[i] = 1 << 30; - - //TODO: add labels correctly - var label = pointAccessor.GetLabelUInt_8(ref points[i]);//points[i].Label; + flags[i] = points.Span[i].Flags; } return new Mesh(triangles, vertices, null, null, null, null, null, null, colors, null, null, flags) { - Name = OctantId.OctantIdToPotreeName(octantId), MeshType = PrimitiveType.Points }; } /// - /// Returns meshes for point clouds of type . + /// Returns meshes for point clouds of type . /// - /// The point accessor allows access to the point data without casting to explicit a explicit point type."/> /// The lists of "raw" points. /// The id of the octant. - public static InstanceData CreateInstanceDataPosD3ColF3LblB(PointAccessor pointAccessor, TPoint[] points, OctantId octantId) + public static InstanceData CreateInstanceData(MemoryOwner points, OctantId octantId) { int numberOfPointsInMesh; numberOfPointsInMesh = points.Length; - var firstPos = (float3)pointAccessor.GetPositionFloat3_64(ref points[0]); + var firstPos = points.Span[0].Position; var vertices = new float3[numberOfPointsInMesh]; var triangles = new ushort[numberOfPointsInMesh]; var colors = new float4[numberOfPointsInMesh]; var boundingBox = new AABBf(firstPos, firstPos); + var flags = new uint[numberOfPointsInMesh]; for (int i = 0; i < points.Length; i++) { - var pos = (float3)pointAccessor.GetPositionFloat3_64(ref points[i]); + var pos = points.Span[i].Position; vertices[i] = pos; boundingBox |= vertices[i]; triangles[i] = (ushort)i; - colors[i] = new float4(pointAccessor.GetColorFloat3_32(ref points[i]) / 256, 1.0f); - - //TODO: add labels correctly - var label = pointAccessor.GetLabelUInt_8(ref points[i]);//points[i].Label; + colors[i] = new float4(points.Span[i].Color.xyz / 265, points.Span[i].Color.w); + flags[i] = points.Span[i].Flags; } + // TODO: Add flags to InstanceData return new InstanceData(points.Length, vertices, null, null, colors) { - Name = octantId.ToString() }; } @@ -233,48 +191,6 @@ private static uint ColorToUInt(int r, int g, int b, int a) return (uint)((b << 16) | (g << 8) | (r << 0) | (a << 24)); } - /// - /// Converts a color, saved as an uint, to float4. - /// - /// The color. - private static float4 UintToColor(uint col) - { - float4 c = new(); - c.b = (byte)((col) & 0xFF); - c.g = (byte)((col >> 8) & 0xFF); - c.r = (byte)((col >> 16) & 0xFF); - c.a = (byte)((col >> 24) & 0xFF); - - return c; - } - - /// - /// Converts a color, saved as an uint, to float3. - /// - /// The color. - private static uint ColorToUint(float3 col) - { - uint packedR = (uint)(col.r * 255); - uint packedG = (uint)(col.g * 255) << 8; - uint packedB = (uint)(col.b * 255) << 16; - - return packedR + packedG + packedB; - } - - /// - /// Converts a color, saved as float4, to uint. - /// - /// The color. - private static uint ColorToUint(float4 col) - { - uint packedR = (uint)(col.r * 255); - uint packedG = (uint)(col.g * 255) << 8; - uint packedB = (uint)(col.b * 255) << 16; - uint packedA = (uint)(col.a * 255) << 24; - - return packedR + packedG + packedB + packedA; - } - #endregion } } \ No newline at end of file diff --git a/src/PointCloud/Core/PointCloudDataHandler.cs b/src/PointCloud/Core/PointCloudDataHandler.cs index 1dcfcf8bd..a4be588e6 100644 --- a/src/PointCloud/Core/PointCloudDataHandler.cs +++ b/src/PointCloud/Core/PointCloudDataHandler.cs @@ -1,11 +1,12 @@ -using Fusee.Base.Core; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Base.Core; using Fusee.Engine.Core; using Fusee.Engine.Core.Scene; using Fusee.PointCloud.Common; -using Fusee.PointCloud.Core.Accessors; using Microsoft.Extensions.Caching.Memory; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -21,7 +22,7 @@ namespace Fusee.PointCloud.Core /// Delegate that allows to inject the loading method of the PointReader - loads the points from file. /// /// Unique ID of an octant. - public delegate TPoint[] LoadPointsHandler(OctantId guid); + public delegate MemoryOwner LoadPointsHandler(OctantId guid); /// /// Delegate for a method that tries to get the mesh(es) of an octant. If they are not cached yet, they should be created an added to the _gpuDataCache. @@ -48,44 +49,50 @@ namespace Fusee.PointCloud.Core /// Generic delegate to inject a method that nows how to actually create a GpuMesh or InstanceData for the given point type. /// /// - /// Generic that describes the point type. - /// The that can be used to access the point data without casting the points. /// The point cloud points as generic array. + /// /// - public delegate TGpuData CreateGpuData(PointAccessor ptAccessor, TPoint[] points, OctantId octantId); + public delegate TGpuData CreateGpuData(MemoryOwner points, OctantId octantId); /// /// Manages the caching and loading of point and mesh data. /// - /// - /// - public class PointCloudDataHandler : PointCloudDataHandlerBase where TPoint : new() where TGpuData : IDisposable + /// Generic for the point/mesh type. + public class PointCloudDataHandler : PointCloudDataHandlerBase, IDisposable where TGpuData : IDisposable { + /// + /// If we encounter an error during our loading process, this event is being triggered + /// + public EventHandler? OnLoadingErrorEvent; + + + private HashSet _meshesToUpdate = new(); + /// /// Caches loaded points. /// - private readonly MemoryCache _pointCache; + private MemoryCache> _pointCache; /// /// Caches loaded points. /// - private readonly MemoryCache> _gpuDataCache; + private MemoryCache> _gpuDataCache; - private readonly PointAccessor _pointAccessor; - private readonly CreateGpuData _createGpuDataHandler; - private readonly LoadPointsHandler _loadPointsHandler; + private readonly CreateGpuData _createGpuDataHandler; + private readonly LoadPointsHandler _loadPointsHandler; private const int _maxNumberOfDisposals = 1; private float _deltaTimeSinceLastDisposal; private readonly bool _doRenderInstanced; + private bool _disposed; + /// /// Creates a new instance. /// - /// The point accessor that allows to access the point data. /// Method that knows how to create a mesh for the explicit point type (see ). /// The method that is able to load the points from the hard drive/file. /// - public PointCloudDataHandler(PointAccessor pointAccessor, CreateGpuData createMeshHandler, LoadPointsHandler loadPointsHandler, bool doRenderInstanced = false) + public PointCloudDataHandler(CreateGpuData createMeshHandler, LoadPointsHandler loadPointsHandler, bool doRenderInstanced = false) { _pointCache = new(); _gpuDataCache = new() @@ -96,7 +103,6 @@ public PointCloudDataHandler(PointAccessor pointAccessor, CreateGpuData< _createGpuDataHandler = createMeshHandler; _loadPointsHandler = loadPointsHandler; - _pointAccessor = pointAccessor; _doRenderInstanced = doRenderInstanced; @@ -104,6 +110,47 @@ public PointCloudDataHandler(PointAccessor pointAccessor, CreateGpuData< DisposeQueue = new Dictionary>((8 ^ 8) / 8); _gpuDataCache.HandleEvictedItem = OnItemEvictedFromCache; + + _pointCache.HandleEvictedItem += (object key, object? value, EvictionReason reason, object? state) => + { + if (value != null && value is MemoryOwner mo) + { + mo.Dispose(); + } + }; + + InvalidateCacheToken.IsDirtyPropertyChanged += (isDirty) => + { + if (isDirty) + { + //_meshesToUpdate = _gpuDataCache.GetKeys.ToHashSet(); + _meshesToUpdate = _pointCache.GetKeys.ToHashSet(); + } + }; + + } + + private GpuDataState DoUpdateGpuData(OctantId octantId, ref IEnumerable gpuData) + { + if (_pointCache.TryGetValue(octantId, out var points)) + { + if (UpdateGpuDataCache != null) + { + UpdateGpuDataCache.Invoke(ref gpuData, points); + } + else + { + if (!_doRenderInstanced) + gpuData = MeshMaker.CreateMeshes(points, _createGpuDataHandler, octantId); + else + gpuData = MeshMaker.CreateInstanceData(points, _createGpuDataHandler, octantId); + } + return GpuDataState.Changed; + } + + //No points in cache - cannot update (point loading is triggered in VisibilityTester) + return GpuDataState.None; + } /// @@ -112,28 +159,73 @@ public PointCloudDataHandler(PointAccessor pointAccessor, CreateGpuData< /// else look in the point cache, if there are points create a mesh and add to the _meshCache. /// /// The unique id of an octant. + /// Allows inserting a condition, if true the mesh will be updated. This is an addition to + /// State of the gpu data in it's life cycle. /// - public override IEnumerable GetGpuData(OctantId octantId) + public override IEnumerable? GetGpuData(OctantId octantId, Func? doUpdateIf, out GpuDataState gpuDataState) { if (_gpuDataCache.TryGetValue(octantId, out var gpuData)) + { + var doUpdate = doUpdateIf != null ? doUpdateIf.Invoke() : false; + if (_meshesToUpdate.Contains(octantId) || doUpdate) + { + gpuDataState = DoUpdateGpuData(octantId, ref gpuData); + + if (gpuDataState != GpuDataState.None) + { + _gpuDataCache.AddOrUpdate(octantId, gpuData); + _meshesToUpdate.Remove(octantId); + if (_meshesToUpdate.Count == 0) + { + InvalidateCacheToken.IsDirty = false; + } + return gpuData; + } + + //Mesh remains in the _meshesToUpdate list but couldn't be updated because the points were missing. + _gpuDataCache.Remove(octantId); + return null; + } + else + gpuDataState = GpuDataState.Unchanged; + return gpuData; + } else if (DisposeQueue.TryGetValue(octantId, out gpuData)) { lock (LockDisposeQueue) { DisposeQueue.Remove(octantId); - _gpuDataCache.Add(octantId, gpuData); + } + + gpuDataState = DoUpdateGpuData(octantId, ref gpuData); + if (gpuDataState != GpuDataState.None) + { + _gpuDataCache.AddOrUpdate(octantId, gpuData); + _meshesToUpdate.Remove(octantId); + if (_meshesToUpdate.Count == 0) + { + InvalidateCacheToken.IsDirty = false; + } return gpuData; } + + _gpuDataCache.Remove(octantId); } else if (_pointCache.TryGetValue(octantId, out var points)) { if (!_doRenderInstanced) - gpuData = MeshMaker.CreateMeshes(_pointAccessor, points, _createGpuDataHandler, octantId); + gpuData = MeshMaker.CreateMeshes(points, _createGpuDataHandler, octantId); else - gpuData = MeshMaker.CreateInstanceData(_pointAccessor, points, _createGpuDataHandler, octantId); - _gpuDataCache.Add(octantId, gpuData); + gpuData = MeshMaker.CreateInstanceData(points, _createGpuDataHandler, octantId); + + _gpuDataCache.AddOrUpdate(octantId, gpuData); + gpuDataState = GpuDataState.New; + return gpuData; } + + gpuDataState = GpuDataState.None; + //no points yet, probably in loading queue return null; } @@ -176,32 +268,86 @@ public override void TriggerPointLoading(OctantId guid) { if (!LoadingQueue.Contains(guid) && LoadingQueue.Count <= MaxNumberOfNodesToLoad) { + if (_pointCache.TryGetValue(guid, out var points)) return; + lock (LockLoadingQueue) { LoadingQueue.Add(guid); } + _ = Task.Run(() => { - if (!_pointCache.TryGetValue(guid, out var points)) - { - points = _loadPointsHandler.Invoke(guid); - _pointCache.Add(guid, points); - } + points = _loadPointsHandler.Invoke(guid); + _pointCache.AddOrUpdate(guid, points); lock (LockLoadingQueue) { LoadingQueue.Remove(guid); } + }).ContinueWith((finishedTask) => + { + // if an exception happened during loading process call the error event for futher handling of the situation + if (finishedTask.Exception != null) + { + OnLoadingErrorEvent?.Invoke(this, new ErrorEventArgs(finishedTask.Exception)); + } + }); } } - private void OnItemEvictedFromCache(object guid, object meshes, EvictionReason reason, object state) + private void OnItemEvictedFromCache(object guid, object? meshes, EvictionReason reason, object? state) { lock (LockDisposeQueue) { - DisposeQueue.Add((OctantId)guid, (IEnumerable)meshes); + if (meshes == null) return; + DisposeQueue.TryAdd((OctantId)guid, (IEnumerable)meshes); + } + } + + /// + /// Dispose(bool disposing) executes in two distinct scenarios. + /// If disposing equals true, the method has been called directly + /// or indirectly by a user's code. Managed and unmanaged resources + /// can be disposed. + /// If disposing equals false, the method has been called by the + /// runtime from inside the finalizer and you should not reference + /// other objects. Only unmanaged resources can be disposed. + /// + /// + protected override void Dispose(bool disposing) + { + // Check to see if Dispose has already been called. + if (!_disposed) + { + // If disposing equals true, dispose all managed + // and unmanaged resources. + if (disposing) + { + // Dispose managed resources. + } + + // Call the appropriate methods to clean up + // unmanaged resources here. + _gpuDataCache.Dispose(); + + lock (LockDisposeQueue) + { + foreach (var item in DisposeQueue) + { + foreach (var d in item.Value) + { + d.Dispose(); + } + } + } + + _pointCache.Dispose(); + LoadingQueue.Clear(); + + // Note disposing has been done. + _disposed = true; } } } diff --git a/src/PointCloud/Core/PointCloudDataHandlerBase.cs b/src/PointCloud/Core/PointCloudDataHandlerBase.cs new file mode 100644 index 000000000..caf2a5eda --- /dev/null +++ b/src/PointCloud/Core/PointCloudDataHandlerBase.cs @@ -0,0 +1,136 @@ +// Ignore Spelling: kvp + +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.PointCloud.Common; +using System; +using System.Collections.Generic; + +namespace Fusee.PointCloud.Core +{ + /// + /// Manages the caching and loading of point and mesh data. + /// + public abstract class PointCloudDataHandlerBase : IDisposable where TGpuData : IDisposable + { + /// + /// Token, that allows to invalidate the complete GpuData cache. + /// + public InvalidateGpuDataCache InvalidateCacheToken { get; } = new(); + + /// + /// Used to manage gpu pressure when disposing of a large quantity of meshes. + /// + public float DisposeRate = 1 / 3f; + + /// + /// Number of nodes that will be loaded, starting with the one with the biggest screen projected size to ensure no octant is loaded that will be invisible in a few frames. + /// Load the five biggest nodes (screen projected size) as proposed in Schütz' thesis. + /// + protected int MaxNumberOfNodesToLoad = 5; + + /// + /// Contains nodes that are queued for loading in the background. + /// + protected List LoadingQueue = new(); + + /// + /// Contains meshes that are marked for disposal. + /// + protected Dictionary> DisposeQueue = new(); + + /// + /// Locking object for the loading queue. + /// + protected object LockLoadingQueue = new(); + /// + /// Locking object for the dispose queue. + /// + protected object LockDisposeQueue = new(); + + /// + /// First looks in the mesh cache, if there are meshes return, + /// else look in the DisposeQueue, if there are meshes return, + /// else look in the point cache, if there are points create a mesh and add to the MeshCache. + /// + /// The unique id of an octant. + /// Allows inserting a condition, if true the mesh will be updated. + /// State of the gpu data in it's life cycle. + public abstract IEnumerable? GetGpuData(OctantId guid, Func? doUpdateIf, out GpuDataState gpuDataState); + + /// + /// Loads points from the hard drive if they are neither in the loading queue nor in the PointCahce. + /// + /// The octant for which the points should be loaded. + public abstract void TriggerPointLoading(OctantId guid); + + /// + /// Disposes of unused meshes, if needed. Depends on the dispose rate and the expiration frequency of the MeshCache. + /// + public abstract void ProcessDisposeQueue(); + + /// + /// Allows to update meshes with data from the points. + /// + public UpdateGpuData, MemoryOwner>? UpdateGpuDataCache; + + private bool _disposed = false; + + /// + /// Implement IDisposable. + /// Do not make this method virtual. + /// A derived class should not be able to override this method. + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// + /// Dispose(bool disposing) executes in two distinct scenarios. + /// If disposing equals true, the method has been called directly + /// or indirectly by a user's code. Managed and unmanaged resources + /// can be disposed. + /// If disposing equals false, the method has been called by the + /// runtime from inside the finalizer and you should not reference + /// other objects. Only unmanaged resources can be disposed. + /// + protected virtual void Dispose(bool disposing) + { + // Check to see if Dispose has already been called. + if (!_disposed) + { + // If disposing equals true, dispose all managed + // and unmanaged resources. + if (disposing) + { + // Dispose managed resources. + + } + + // Call the appropriate methods to clean up + // unmanaged resources here. + foreach (var kvp in DisposeQueue) + { + foreach (var val in kvp.Value) + { + val.Dispose(); + } + } + _disposed = true; + } + } + + /// + /// Use C# finalizer syntax for finalization code. + /// This finalizer will run only if the Dispose method + /// does not get called. + /// It gives your base class the opportunity to finalize. + /// Do not provide finalizer in types derived from this class. + /// + ~PointCloudDataHandlerBase() + { + Dispose(disposing: false); + } + } +} \ No newline at end of file diff --git a/src/PointCloud/Core/PointCloudOctant.cs b/src/PointCloud/Core/PointCloudOctant.cs index bbbc84acd..a415e0648 100644 --- a/src/PointCloud/Core/PointCloudOctant.cs +++ b/src/PointCloud/Core/PointCloudOctant.cs @@ -96,7 +96,7 @@ public double Size /// The size (in all three dimensions) of this octant. /// /// The octants child octants. - public PointCloudOctant(double3 center, double size, OctantId octId, PointCloudOctant[] children = null) + public PointCloudOctant(double3 center, double size, OctantId octId, PointCloudOctant[]? children = null) { Center = center; Size = size; @@ -132,7 +132,7 @@ public void ComputeScreenProjectedSize(double3 camPos, int screenHeight, float f var distance = (translatedCenter - camPos).Length; if (translatedCenter == camPos) distance = 0.0001f; - var slope = (float)System.Math.Tan(fov / 2d); + var slope = System.Math.Abs((float)System.Math.Tan(fov / 2d)); var maxRad = System.Math.Max(System.Math.Max(scaledRad.x, scaledRad.y), scaledRad.z); ProjectedScreenSize = screenHeight / 2d * maxRad / (slope * distance); @@ -193,6 +193,30 @@ public bool Intersects(double3 point) (point.z >= Min.z && point.z <= Max.z); } + /// + /// Returns true if a sphere is completely inside or is intersecting this + /// see: https://web.archive.org/web/19991129023147/http://www.gamasutra.com/features/19991018/Gomez_4.htm + /// + /// world coordinate of the sphere + /// the sphere radius + /// + public bool InsideOrIntersectingSphere(double3 center, float radius) + { + double minValue = 0; + for (var i = 0; i < 3; i++) + { + if (center[i] < Min[i]) + { + minValue += System.Math.Sqrt(center[i] - Min[i]); + } + else if (center[i] > Max[i]) + { + minValue += System.Math.Sqrt(center[i] - Max[i]); + } + } + return minValue <= (radius * radius); + } + /// /// Checks if a given ray originates in, or intersects octant. /// diff --git a/src/PointCloud/Core/Scene/PointCloudComponent.cs b/src/PointCloud/Core/Scene/PointCloudComponent.cs index a1bd6cd33..d11b4074e 100644 --- a/src/PointCloud/Core/Scene/PointCloudComponent.cs +++ b/src/PointCloud/Core/Scene/PointCloudComponent.cs @@ -1,4 +1,4 @@ -using Fusee.Engine.Core.Scene; +using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using Fusee.PointCloud.Common; @@ -32,10 +32,10 @@ public class PointCloudComponent : SceneComponent, IPointCloud /// /// Reference to the Camera whose properties are used to control the visibility of point cloud chunks (octants). /// - public Camera Camera; + public Camera? Camera; /// - /// Instantiates the . + /// Instantiates the . /// public PointCloudComponent(IPointCloudImpBase imp, RenderMode renderMode = RenderMode.StaticMesh) { diff --git a/src/PointCloud/Core/Scene/PointCloudPickerModule.cs b/src/PointCloud/Core/Scene/PointCloudPickerModule.cs new file mode 100644 index 000000000..c836636cf --- /dev/null +++ b/src/PointCloud/Core/Scene/PointCloudPickerModule.cs @@ -0,0 +1,215 @@ +using CommunityToolkit.Diagnostics; +using Fusee.Base.Core; +using Fusee.Engine.Core; +using Fusee.Engine.Core.Scene; +using Fusee.Math.Core; +using Fusee.PointCloud.Common; +using Fusee.Xene; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using static Fusee.Engine.Core.ScenePicker; + +namespace Fusee.PointCloud.Core.Scene +{ + /// + /// Result of a point cloud pick operation + /// + public class PointCloudPickResult : PickResult + { + /// + /// The point mesh. + /// + public Mesh? Mesh; + /// + /// The index of the hit vertex, in this case the point index. + /// + public int VertIdx; + /// + /// The of the in which the found point lies. + /// + public OctantId OctantId; + + /// + /// The distance between ray (origin: mouse position) and hit result (point) + /// + public float2 DistanceToRay; + } + + /// + /// Point cloud picker module. Inject to pick s + /// + public class PointCloudPickerModule : IPickerModule + { + private PickerState? _state; + private readonly PointCloudOctree? _octree; + private readonly IPointCloudImp? _pcImp; + private readonly float _pointSpacing; + + /// + /// The pick result after picking. + /// + public List PickResults { get; set; } = new(); + + internal struct MinPickValue + { + internal float2 Distance; + internal Mesh Mesh; + internal int VertIdx; + internal OctantId OctantId; + } + + /// + /// Determines visible points of a point cloud (using the components ) and renders them. + /// + /// The point cloud component. + [VisitMethod] + public void RenderPointCloud(PointCloudComponent pointCloud) + { + PickResults.Clear(); + if (!pointCloud.Active) return; + + Guard.IsNotNull(_pcImp); + Guard.IsNotNull(_octree); + Guard.IsNotNull(_state); + + if (float.IsInfinity(_state.PickPosClip.x) || float.IsInfinity(_state.PickPosClip.y)) + return; + + var proj = _state.CurrentCameraResult.Camera.GetProjectionMat(_state.ScreenSize.x, _state.ScreenSize.y, out _); + var view = _state.CurrentCameraResult.View; + var rayD = new RayD(new double2(_state.PickPosClip.x, _state.PickPosClip.y), (double4x4)view, (double4x4)proj); + + var tmpList = new List(); + var allHitBoxes = PickOctantRecursively((PointCloudOctant)_octree.Root, rayD, tmpList).ToList(); + + if (allHitBoxes == null || allHitBoxes.Count == 0) return; + + var currentRes = new ConcurrentBag(); + + Parallel.ForEach(_pcImp.GpuDataToRender, (mesh) => + { + foreach (var box in allHitBoxes) + { + if (!mesh.BoundingBox.Intersects(new AABBf((float3)box.Min, (float3)box.Max))) continue; + + var currentMin = new MinPickValue + { + Distance = float2.One * float.MaxValue + }; + + Guard.IsNotNull(mesh.Vertices); + + for (var i = 0; i < mesh.Vertices.Length; i++) + { + var dist = SphereRayIntersection((float3)rayD.Origin, (float3)rayD.Direction, mesh.Vertices[i], _pointSpacing * 0.5f); + if (dist.x < 0 || dist.y < 0) continue; + + if (dist.x <= currentMin.Distance.x && dist.y <= currentMin.Distance.y) + { + currentMin.Distance = dist; + currentMin.Mesh = mesh; + currentMin.VertIdx = i; + currentMin.OctantId = box.OctId; + //break; // <- check if break after first result is enough even for sparse point clouds + } + } + + if (currentMin.Mesh == null) continue; + currentRes.Add(currentMin); + } + }); + + if (currentRes == null || currentRes.IsEmpty) return; + + + var mvp = proj * view * _state.Model; + + foreach (var r in currentRes) + { + var pickRes = new PointCloudPickResult + { + Node = null, + Projection = proj, + View = view, + Model = _state.Model, + ClipPos = float4x4.TransformPerspective(mvp, r.Mesh.Vertices[r.VertIdx]), + DistanceToRay = r.Distance, + Mesh = r.Mesh, + VertIdx = r.VertIdx, + OctantId = r.OctantId + }; + + PickResults.Add(pickRes); + } + } + + + /// + /// Calculates the intersection distance between a ray and a sphere. + /// + /// + /// + /// Center point of sphere/point + /// Radius of sphere with center point of + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static float2 SphereRayIntersection(float3 ro, float3 rd, float3 ce, float ra) + { + var oc = ro - ce; + var b = float3.Dot(oc, rd); + var c = float3.Dot(oc, oc) - ra * ra; + var h = b * b - c; + if (h < 0.0f) return new float2(-1.0f); // no intersection + h = MathF.Sqrt(h); + if (float.IsNaN(h) || float.IsInfinity(h)) return new float2(-1.0f); + return new float2(-b - h, -b + h); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private List PickOctantRecursively(PointCloudOctant node, RayD ray, List list) + { + list.Add(node); + if (node.Children[0] != null) + { + foreach (var child in node.Children.Cast()) + { + if (child?.IsVisible == true && child.IntersectRay(ray)) + { + PickOctantRecursively(child, ray, list); + } + } + } + return list; + } + + /// + /// Inject this to a to be able to pick s. + /// The actual point and data needs to be present a priori, however it's type is polymorph, therefore we need to inject those data, too. + /// + /// The of the . + /// The , needs to be of type + /// The spacing between points. For Potree use the metadata spacing component * 0.1f
e. g. Spacing = 2.18f, pass 0.218f to this ctor. + public PointCloudPickerModule(IPointCloudOctree octree, IPointCloudImp pcImp, float pointSpacing) + { + if (pcImp == null) + Diagnostics.Warn("No per point picking possible, no PointCloud type loaded"); + + _octree = (PointCloudOctree)octree; + _pcImp = pcImp; + _pointSpacing = pointSpacing; + } + + /// + /// Set the current from external (this is done automatically by the method. + /// + /// + public void SetState(PickerState state) + { + _state = state; + } + } +} \ No newline at end of file diff --git a/src/PointCloud/Core/Scene/PointCloudRenderModule.cs b/src/PointCloud/Core/Scene/PointCloudRenderModule.cs index b5eebeea8..f319cd3fa 100644 --- a/src/PointCloud/Core/Scene/PointCloudRenderModule.cs +++ b/src/PointCloud/Core/Scene/PointCloudRenderModule.cs @@ -1,3 +1,5 @@ +// Ignore Spelling: fov + using Fusee.Base.Core; using Fusee.Engine.Core; using Fusee.Engine.Core.Primitives; @@ -42,6 +44,9 @@ public void UpdateContext(RenderContext rc) throw new ArgumentNullException(nameof(rc)); _rc = rc; + + // prevent rendering with + _rc.AllowDirtyMeshs = false; } /// @@ -110,19 +115,19 @@ public void RenderPointCloud(PointCloudComponent pointCloud) switch (pointCloud.RenderMode) { case RenderMode.StaticMesh: - foreach (var mesh in ((IPointCloudImp)pointCloud.PointCloudImp).GpuDataToRender) + foreach (var mesh in ((IPointCloudImp)pointCloud.PointCloudImp).GpuDataToRender) { _rc.Render(mesh, _isForwardModule); } break; case RenderMode.Instanced: - foreach (var instanceData in ((IPointCloudImp)pointCloud.PointCloudImp).GpuDataToRender) + foreach (var instanceData in ((IPointCloudImp)pointCloud.PointCloudImp).GpuDataToRender) { _rc.Render(quad, instanceData, _isForwardModule); } break; case RenderMode.DynamicMesh: - foreach (var mesh in ((IPointCloudImp)pointCloud.PointCloudImp).GpuDataToRender) + foreach (var mesh in ((IPointCloudImp)pointCloud.PointCloudImp).GpuDataToRender) { _rc.Render(mesh, null, _isForwardModule); } diff --git a/src/PointCloud/Core/VisibilityTester.cs b/src/PointCloud/Core/VisibilityTester.cs index e7998a4e5..51c66120d 100644 --- a/src/PointCloud/Core/VisibilityTester.cs +++ b/src/PointCloud/Core/VisibilityTester.cs @@ -1,3 +1,4 @@ +using CommunityToolkit.Diagnostics; using Fusee.Engine.Core; using Fusee.Math.Core; using Fusee.PointCloud.Common; @@ -49,7 +50,7 @@ public float3 CamPos /// /// Current camera frustum - set by the SceneRenderer if a PointCloud Component is visited. /// - public FrustumF RenderFrustum { get; set; } + public FrustumF? RenderFrustum { get; set; } /// /// The octree structure of the point cloud. @@ -178,6 +179,7 @@ private void DetermineVisibilityForNode(PointCloudOctant node) //If node does not intersect the viewing frustum or is smaller than the minimal projected size: //Return -> will not be added to _visibleNodesOrderedByProjectionSize -> traversal of this branch stops. + Guard.IsNotNull(RenderFrustum); if (!node.InsideOrIntersectingFrustum(RenderFrustum, translation, scale) || node.ProjectedScreenSize < _minScreenProjectedSize) { node.IsVisible = false; diff --git a/src/PointCloud/Core/VisualizationPoint.cs b/src/PointCloud/Core/VisualizationPoint.cs new file mode 100644 index 000000000..9da7e18ca --- /dev/null +++ b/src/PointCloud/Core/VisualizationPoint.cs @@ -0,0 +1,41 @@ +using Fusee.Math.Core; +using System; +using System.Runtime.InteropServices; + +namespace Fusee.PointCloud.Core +{ + /// + /// This point is used for visualization purposes. + /// It is read from a file and converted to mesh data. + /// + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public struct VisualizationPoint + { + /// + /// How Flags should be converted by ToString. + /// + public static Func FlagsParser = (flags) => flags.ToString(); + + /// + /// The position of a point. + /// + public float3 Position; + + /// + /// The color (r,g,b,a) of a point. + /// + public float4 Color; + + /// + /// Flags have to be interpreted manually or they will be ignored. + /// + + public uint Flags; + + /// + public override string ToString() + { + return $"Position: {Position} - Color: {Color} - Flags: {FlagsParser(Flags)}"; + } + } +} \ No newline at end of file diff --git a/src/PointCloud/Las/Desktop/Fusee.PointCloud.Las.Desktop.csproj b/src/PointCloud/Las/Desktop/Fusee.PointCloud.Las.Desktop.csproj deleted file mode 100644 index 9a7450b5d..000000000 --- a/src/PointCloud/Las/Desktop/Fusee.PointCloud.Las.Desktop.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - - net7.0 - $(DefineConstants);PLATFORM_DESKTOP - $(OutputPath)\$(RootNamespace).xml - - - - - - - - - - - - - - - - - - diff --git a/src/PointCloud/Las/Desktop/LasPointReader.cs b/src/PointCloud/Las/Desktop/LasPointReader.cs deleted file mode 100644 index 77f3212b9..000000000 --- a/src/PointCloud/Las/Desktop/LasPointReader.cs +++ /dev/null @@ -1,270 +0,0 @@ -using Fusee.Base.Imp.Desktop; -using Fusee.Math.Core; -using Fusee.PointCloud.Common; -using Fusee.PointCloud.Common.Accessors; -using Fusee.PointCloud.Core.Accessors; -using System; -using System.IO; -using System.Runtime.InteropServices; -using System.Threading.Tasks; - -namespace Fusee.PointCloud.Las.Desktop -{ - /// - /// A reader for points in LAZ or LAS format - /// - public class LasPointReader : IDisposable, IPointReader - { - public IPointAccessor PointAccessor { get; private set; } - - /// - /// The point cloud meta data, usually stored in the header of the las file. - /// - public LasMetaInfo MetaInfo { get; private set; } - - private IntPtr _ptrToLASClass = new(); - private string _filePath; - - public IPointCloud GetPointCloudComponent(RenderMode renderMode) - { - OpenFile(_filePath); - - var pointType = PointType; - switch (pointType) - { - case PointType.Undefined: - case PointType.PosD3: - case PointType.PosD3ColF3InUs: - case PointType.PosD3InUs: - case PointType.PosD3ColF3: - case PointType.PosD3LblB: - case PointType.PosD3NorF3ColF3InUs: - case PointType.PosD3NorF3InUs: - case PointType.PosD3NorF3ColF3: - case PointType.PosD3ColF3LblB: - case PointType.PosD3ColF3InUsLblB: - default: - break; - } - - throw new NotImplementedException(); - } - - public IPointCloudOctree GetOctree() - { - //convert to potree octree - throw new NotImplementedException(); - } - - public TPoint[] LoadNodeData(OctantId id) where TPoint : new() - { - throw new NotImplementedException(); - } - - private void OpenFile(string filename) - { - EmbeddedResourcesDllHandler.LoadEmbeddedDll("libLASlib.dll", "Fusee.PointCloud.Las.Desktop.Natives.libLASlib.dll"); - - // Open file - OpenLASFile(filename, ref _ptrToLASClass); - - if (_ptrToLASClass == IntPtr.Zero) - throw new FileNotFoundException($"{filename} not found!"); - - // Read header - var header = new LasInternalHeader(); - - GetHeader(_ptrToLASClass, ref header); - - MetaInfo = new LasMetaInfo - { - Filename = filename, - OffsetX = header.OffsetX, - OffsetY = header.OffsetY, - OffsetZ = header.OffsetZ, - PointCount = header.PointCnt, - PointDataFormat = header.PointDataFormat, - ScaleFactorX = header.ScaleFactorX, - ScaleFactorY = header.ScaleFactorY, - ScaleFactorZ = header.ScaleFactorZ - }; - } - - /// - /// A LASPointReader can open point files encoded by the las format v. 1.4 with the following extensions: - /// - *.asc - /// - *.bil - /// - *.bin - /// - *.dtm - /// - *.las - /// - *.ply - /// - *.qfit - /// - *.shp - /// - *.txt - /// - *.laz - /// - /// The path to a las encoded file. - public LasPointReader(string filePath) - { - _filePath = filePath; - OpenFile(filePath); - } - - /// - /// A LASPointReader can open point files encoded by the las format v. 1.4 with the following extensions: - /// - *.asc - /// - *.bil - /// - *.bin - /// - *.dtm - /// - *.las - /// - *.ply - /// - *.qfit - /// - *.shp - /// - *.txt - /// - *.laz - /// - public LasPointReader() { } - - /// - /// Reads the given amount of points from stream - /// - /// - /// - /// - /// - public TPoint[] ReadNPoints(int n, IPointAccessor pa) where TPoint : new() - { - if (_ptrToLASClass == IntPtr.Zero) - throw new FileNotFoundException("No file was specified yet. Call 'OpenFile' first"); - var points = new TPoint[n]; - for (var i = 0; i < points.Length; i++) - { - if (!ReadNextPoint(ref points[i], pa)) break; - } - return points; - } - - /// - /// Reads the next point and writes it to the given point - /// - /// - /// - /// - public bool ReadNextPoint(ref TPoint point, IPointAccessor pa) where TPoint : new() - { - if (point == null) - throw new ArgumentOutOfRangeException("No writable point found!"); - - var hasNextPoint = true; - ReadNextPoint(_ptrToLASClass, ref hasNextPoint); - if (!hasNextPoint) return false; - - var currentPoint = new LasInternalPoint(); - GetPoint(_ptrToLASClass, ref currentPoint); - - var typedAccessor = (PointAccessor)pa; - - if ((MetaInfo.PointDataFormat == 2 || MetaInfo.PointDataFormat == 3) && typedAccessor.ColorType == PointColorType.Float3) - typedAccessor.SetColorFloat3_32(ref point, new float3(currentPoint.R, currentPoint.G, currentPoint.B)); - - //TODO: Complete - //if (typedAccessor.PositionType == PointPositionType.Double3) - // -> always the case right now - typedAccessor.SetPositionFloat3_64(ref point, new double3(currentPoint.X * MetaInfo.ScaleFactorX, currentPoint.Y * MetaInfo.ScaleFactorY, currentPoint.Z * MetaInfo.ScaleFactorZ)); - - //if (currentFormat.HasIntensity && typedAccessor.IntensityType == PointIntensityType.UInt_16) - // -> always true right now! - typedAccessor.SetIntensityUInt_16(ref point, currentPoint.Intensity); - - // -> never the case right now! - //if (currentFormat.HasClassification && typedAccessor.LabelType == PointLabelType.UInt_8) - //{ - // //TODO: HACK!! label was somehow written to UserData and not to classification - // if (currentPoint.Classification != 0) - // typedAccessor.SetLabelUInt_8(ref point, currentPoint.Classification); - // else - // typedAccessor.SetLabelUInt_8(ref point, currentPoint.UserData); - //} - - return true; - } - - public PointType PointType - { - get - { - //TODO: Complete - switch (MetaInfo.PointDataFormat) - { - case 0: - case 1: - return PointType.PosD3InUs; - case 2: - case 3: - return PointType.PosD3ColF3InUs; - default: - throw new ArgumentException($"Point data format with byte {MetaInfo.PointDataFormat} not recognized!"); - } - } - } - - public Task LoadPointsForNodeAsync(string guid, IPointAccessor pointAccessor) where TPoint : new() - { - throw new NotImplementedException(); - } - - public TPoint[] LoadNodeData(string id, IPointAccessor pointAccessor) where TPoint : new() - { - throw new NotImplementedException(); - } - - #region IDisposable Support - private bool disposedValue = false; // To detect redundant calls - - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - // dispose managed state (managed objects). - } - - Delete(ref _ptrToLASClass); - _ptrToLASClass = IntPtr.Zero; - - disposedValue = true; - } - } - - ~LasPointReader() - { - // Do not change this code. Put cleanup code in Dispose(bool disposing) above. - Dispose(false); - } - - // This code added to correctly implement the disposable pattern. - public void Dispose() - { - // Do not change this code. Put cleanup code in Dispose(bool disposing) above. - Dispose(true); - GC.SuppressFinalize(this); - } - #endregion - - [DllImport("libLASlib", EntryPoint = "CS_OpenLasFile")] - private static extern IntPtr OpenLASFile(string filename, ref IntPtr lasFileHandle); - - [DllImport("libLASlib", EntryPoint = "CS_GetHeader")] - private static extern void GetHeader(IntPtr lasFileHandle, ref LasInternalHeader header); - - [DllImport("libLASlib", EntryPoint = "CS_ReadNextPoint")] - private static extern void ReadNextPoint(IntPtr lasFileHandle, ref bool nextPoint); - - [DllImport("libLASlib", EntryPoint = "CS_GetPoint")] - private static extern void GetPoint(IntPtr lasFileHandle, ref LasInternalPoint csPoint); - - [DllImport("libLASlib", EntryPoint = "CS_Delete")] - private static extern void Delete(ref IntPtr lasFileHandle); - } -} \ No newline at end of file diff --git a/src/PointCloud/Las/Shared/Fusee.PointCloud.Las.Shared.projitems b/src/PointCloud/Las/Shared/Fusee.PointCloud.Las.Shared.projitems deleted file mode 100644 index b3fe29c14..000000000 --- a/src/PointCloud/Las/Shared/Fusee.PointCloud.Las.Shared.projitems +++ /dev/null @@ -1,16 +0,0 @@ - - - - $(MSBuildAllProjects);$(MSBuildThisFileFullPath) - true - dcc7da71-3e2e-476c-8391-1f9651637503 - - - Fusee.PointCloud.FileReader.Las.Shared - - - - - - - \ No newline at end of file diff --git a/src/PointCloud/Las/Shared/Fusee.PointCloud.Las.Shared.shproj b/src/PointCloud/Las/Shared/Fusee.PointCloud.Las.Shared.shproj deleted file mode 100644 index 9fb031a46..000000000 --- a/src/PointCloud/Las/Shared/Fusee.PointCloud.Las.Shared.shproj +++ /dev/null @@ -1,13 +0,0 @@ - - - - dcc7da71-3e2e-476c-8391-1f9651637503 - 14.0 - - - - - - - - diff --git a/src/PointCloud/Las/Shared/LasInternalHeader.cs b/src/PointCloud/Las/Shared/LasInternalHeader.cs deleted file mode 100644 index 28c2a3efa..000000000 --- a/src/PointCloud/Las/Shared/LasInternalHeader.cs +++ /dev/null @@ -1,18 +0,0 @@ - -namespace Fusee.PointCloud.Las.Desktop -{ - internal struct LasInternalHeader - { - public byte PointDataFormat; - - public long PointCnt; - - public double ScaleFactorX; - public double ScaleFactorY; - public double ScaleFactorZ; - - public double OffsetX; - public double OffsetY; - public double OffsetZ; - } -} \ No newline at end of file diff --git a/src/PointCloud/Las/Shared/LasInternalPoint.cs b/src/PointCloud/Las/Shared/LasInternalPoint.cs deleted file mode 100644 index 48ec112e5..000000000 --- a/src/PointCloud/Las/Shared/LasInternalPoint.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace Fusee.PointCloud.Las.Desktop -{ - internal struct LasInternalPoint - { - public int X; - public int Y; - public int Z; - - public ushort Intensity; - - public byte Classification; - public byte UserData; - - public ushort R; - public ushort G; - public ushort B; - } -} \ No newline at end of file diff --git a/src/PointCloud/Las/Shared/LasMetaInfo.cs b/src/PointCloud/Las/Shared/LasMetaInfo.cs deleted file mode 100644 index 2f32d5a4f..000000000 --- a/src/PointCloud/Las/Shared/LasMetaInfo.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Fusee.PointCloud.Las.Desktop -{ - public struct LasMetaInfo - { - public string Filename; - - public byte PointDataFormat; - - public long PointCount { get; set; } - - public double ScaleFactorX; - public double ScaleFactorY; - public double ScaleFactorZ; - - public double OffsetX; - public double OffsetY; - public double OffsetZ; - } -} \ No newline at end of file diff --git a/src/PointCloud/Potree/Fusee.PointCloud.Potree.csproj b/src/PointCloud/Potree/Fusee.PointCloud.Potree.csproj index 52c49bd00..74038f875 100644 --- a/src/PointCloud/Potree/Fusee.PointCloud.Potree.csproj +++ b/src/PointCloud/Potree/Fusee.PointCloud.Potree.csproj @@ -1,21 +1,20 @@ - - netstandard2.1;net7.0 - $(OutputPath)\$(RootNamespace).xml - enable - + + netstandard2.1;net7.0 + $(OutputPath)\$(RootNamespace).xml + enable + - - - - - - - - - - - + + + + + + + + + + diff --git a/src/PointCloud/Potree/Potree2Cloud.cs b/src/PointCloud/Potree/Potree2Cloud.cs index 1f94d52be..c84e8db4c 100644 --- a/src/PointCloud/Potree/Potree2Cloud.cs +++ b/src/PointCloud/Potree/Potree2Cloud.cs @@ -1,4 +1,6 @@ -using Fusee.Engine.Core; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Base.Core; +using Fusee.Engine.Core; using Fusee.Math.Core; using Fusee.PointCloud.Common; using Fusee.PointCloud.Core; @@ -9,8 +11,10 @@ namespace Fusee.PointCloud.Potree /// /// Non-point-type-specific implementation of Potree2 clouds. /// - public class Potree2Cloud : IPointCloudImp + public class Potree2Cloud : IPointCloudImp { + public InvalidateGpuDataCache InvalidateGpuDataCache { get; } = new(); + /// /// The complete list of meshes that can be rendered. /// @@ -82,7 +86,6 @@ public float UpdateRate /// public float3 Size => new((float)VisibilityTester.Octree.Root.Size); - private readonly GetMeshes _getMeshes; private bool _doUpdate = true; /// @@ -92,12 +95,22 @@ public Potree2Cloud(PointCloudDataHandlerBase dataHandler, IPointCloudO { GpuDataToRender = new List(); DataHandler = dataHandler; + DataHandler.UpdateGpuDataCache = UpdateGpuDataCache; VisibilityTester = new VisibilityTester(octree, dataHandler.TriggerPointLoading); - _getMeshes = dataHandler.GetGpuData; } /// - /// Uses the and to update the visible meshes. + /// Allows to update meshes with data from the points. + /// + /// The meshes that have to be updated. + /// The points with the desired values. + public void UpdateGpuDataCache(ref IEnumerable meshes, MemoryOwner points) + { + Diagnostics.Warn("Not implemented. Cache will not be updated."); + } + + /// + /// Uses the and to update the visible meshes. /// Called every frame. /// /// The camera's field of view. @@ -129,7 +142,7 @@ public void Update(float fov, int viewportHeight, FrustumF renderFrustum, float3 { if (!guid.Valid) continue; - var meshes = _getMeshes(guid); + var meshes = DataHandler.GetGpuData(guid, null, out _); if (meshes == null) continue; //points for this octant aren't loaded yet. diff --git a/src/PointCloud/Potree/Potree2CloudDynamic.cs b/src/PointCloud/Potree/Potree2CloudDynamic.cs index a9eef4718..74f28d238 100644 --- a/src/PointCloud/Potree/Potree2CloudDynamic.cs +++ b/src/PointCloud/Potree/Potree2CloudDynamic.cs @@ -1,18 +1,24 @@ -using Fusee.Engine.Core.Scene; +using CommunityToolkit.Diagnostics; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using Fusee.PointCloud.Common; using Fusee.PointCloud.Core; using System; using System.Collections.Generic; -using System.Linq; namespace Fusee.PointCloud.Potree { /// /// Non-point-type-specific implementation of Potree2 clouds. /// - public class Potree2CloudDynamic : IPointCloudImp + public class Potree2CloudDynamic : IPointCloudImp { + /// + /// Object for handling the invalidation of the gpu data cache. + /// + public InvalidateGpuDataCache InvalidateGpuDataCache { get => DataHandler.InvalidateCacheToken; } + /// /// The complete list of meshes that can be rendered. /// @@ -84,7 +90,16 @@ public float UpdateRate /// public float3 Size => new((float)VisibilityTester.Octree.Root.Size); - private readonly GetDynamicMeshes _getMeshes; + /// + /// Action that is run on every mesh that is determined as newly visible. + /// + public Action? NewMeshAction; + + /// + /// Action that is run on every mesh that was updated. + /// + public Action? UpdatedMeshAction; + private bool _doUpdate = true; /// @@ -94,22 +109,45 @@ public Potree2CloudDynamic(PointCloudDataHandlerBase dataHandler, IPointCl { GpuDataToRender = new List(); DataHandler = dataHandler; + DataHandler.UpdateGpuDataCache = UpdateGpuDataCache; VisibilityTester = new VisibilityTester(octree, dataHandler.TriggerPointLoading); - _getMeshes = dataHandler.GetGpuData; } /// - /// Action that is run on every mesh that is loaded to be visible. + /// Allows to update meshes with data from the points. /// - public Action NewMeshAction; + /// The meshes that have to be updated. + /// The points with the desired values. + public void UpdateGpuDataCache(ref IEnumerable meshes, MemoryOwner points) + { + var countStartSlice = 0; + + foreach (var mesh in meshes) + { + mesh.Name = string.Empty; + if (mesh.Flags == null) continue; + var slice = points.Span.Slice(countStartSlice, mesh.Flags.Length); + + for (int i = 0; i < slice.Length; i++) + { + var pt = slice[i]; + if (mesh.Flags[i] != pt.Flags) + mesh.Flags[i] = pt.Flags; + } + countStartSlice += mesh.Flags.Length; + } + } /// - /// Determins if new Meshes should be loaded. + /// Determines if new Meshes should be loaded. /// public bool LoadNewMeshes { get; set; } = true; + + private List _visibleOctantsCache = new(); + /// - /// Uses the and to update the visible meshes. + /// Uses the and to update the visible meshes. /// Called every frame. /// /// The camera's field of view. @@ -137,35 +175,51 @@ public void Update(float fov, int viewportHeight, FrustumF renderFrustum, float3 VisibilityTester.Model = modelMat; VisibilityTester.Update(); + GpuDataToRender.Clear(); - var meshes = new List(); + var currentOctants = new List(); foreach (var guid in VisibilityTester.VisibleNodes) { if (!guid.Valid) continue; - var guidMeshes = _getMeshes(guid); - - if (guidMeshes == null) continue; //points for this octant aren't loaded yet. + var guidMeshes = DataHandler.GetGpuData(guid, () => !_visibleOctantsCache.Contains(guid), out GpuDataState meshStatus); - meshes.AddRange(guidMeshes); - } - - if (NewMeshAction != null) - { - var newMeshes = meshes.Except(GpuDataToRender); - - if (newMeshes.Any()) + switch (meshStatus) { - foreach (var mesh in newMeshes) - { - NewMeshAction(mesh); - } + //Octants that are now visible but the points for this octant aren't loaded yet. + //Nothing to do here. + case GpuDataState.None: + continue; + //Octants that are now visible and the meshes are newly created. + //They we have to call "NewMeshAction" when they are loaded. + case GpuDataState.New: + Guard.IsNotNull(guidMeshes); //If this is null we have an internal error in DataHandler.GetMeshes/DoUpdate + foreach (var mesh in guidMeshes) + { + NewMeshAction?.Invoke(mesh); + } + break; + //Octants that are now visible and the existing meshes where updated. + case GpuDataState.Changed: + Guard.IsNotNull(guidMeshes); //If this is null we have an internal error in DataHandler.GetMeshes/DoUpdate + foreach (var mesh in guidMeshes) + { + UpdatedMeshAction?.Invoke(mesh); + } + break; + case GpuDataState.Unchanged: + Guard.IsNotNull(guidMeshes); //If this is null we have an internal error in DataHandler.GetMeshes/DoUpdate + break; + default: + throw new ArgumentException($"Invalid mesh status {meshStatus}."); } + + GpuDataToRender.AddRange(guidMeshes); + currentOctants.Add(guid); } - GpuDataToRender.Clear(); - GpuDataToRender.AddRange(meshes); + _visibleOctantsCache = new(currentOctants); } } } \ No newline at end of file diff --git a/src/PointCloud/Potree/Potree2CloudInstanced.cs b/src/PointCloud/Potree/Potree2CloudInstanced.cs index 830e397c4..b270e10b1 100644 --- a/src/PointCloud/Potree/Potree2CloudInstanced.cs +++ b/src/PointCloud/Potree/Potree2CloudInstanced.cs @@ -1,4 +1,6 @@ -using Fusee.Engine.Core.Scene; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Base.Core; +using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using Fusee.PointCloud.Common; using Fusee.PointCloud.Core; @@ -9,8 +11,10 @@ namespace Fusee.PointCloud.Potree /// /// Non-point-type-specific implementation of Potree2 clouds. /// - public class Potree2CloudInstanced : IPointCloudImp + public class Potree2CloudInstanced : IPointCloudImp { + public InvalidateGpuDataCache InvalidateGpuDataCache { get; } = new(); + /// /// The complete list of meshes that can be rendered. /// @@ -82,7 +86,6 @@ public float UpdateRate /// public float3 Size => new((float)VisibilityTester.Octree.Root.Size); - private readonly GetInstanceData _getInstanceData; private bool _doUpdate = true; /// @@ -92,12 +95,22 @@ public Potree2CloudInstanced(PointCloudDataHandlerBase dataHandler { GpuDataToRender = new List(); DataHandler = dataHandler; + DataHandler.UpdateGpuDataCache = UpdateGpuDataCache; VisibilityTester = new VisibilityTester(octree, dataHandler.TriggerPointLoading); - _getInstanceData = dataHandler.GetGpuData; } /// - /// Uses the and to update the visible meshes. + /// Allows to update meshes with data from the points. + /// + /// The meshes that have to be updated. + /// The points with the desired values. + public void UpdateGpuDataCache(ref IEnumerable meshes, MemoryOwner points) + { + Diagnostics.Warn("Not implemented. Cache will not be updated."); + } + + /// + /// Uses the and to update the visible meshes. /// Called every frame. /// /// The camera's field of view. @@ -129,7 +142,7 @@ public void Update(float fov, int viewportHeight, FrustumF renderFrustum, float3 { if (!guid.Valid) continue; - var instanceData = _getInstanceData(guid); + var instanceData = DataHandler.GetGpuData(guid, null, out _); if (instanceData == null) continue; //points for this octant aren't loaded yet. diff --git a/src/PointCloud/Potree/Potree2LAS.cs b/src/PointCloud/Potree/Potree2LAS.cs new file mode 100644 index 000000000..6ecad5c39 --- /dev/null +++ b/src/PointCloud/Potree/Potree2LAS.cs @@ -0,0 +1,550 @@ +using CommunityToolkit.Diagnostics; +using Fusee.Base.Core; +using Fusee.PointCloud.Common; +using Fusee.PointCloud.Potree.V2.Data; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +namespace Fusee.PointCloud.Potree +{ + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + internal struct LASHeader + { + public LASHeader() { } + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + internal char[] FileSignature = new char[] { 'L', 'A', 'S', 'F' }; + + internal ushort FileSourceID = 0; + internal ushort GlobalEncoding = 0; + + internal uint GUIDData1 = 0; + internal ushort GUIDData2 = 0; + internal ushort GUIDData3 = 0; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] + internal byte[] GUIData4 = new byte[8]; + + internal byte VersionMajor = 1; + internal byte VersionMinor = 4; + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 32)] + internal byte[] SystemIdentifier = new byte[32]; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 32)] + internal byte[] GeneratingSoftware = new byte[32]; + + internal ushort FileCreationDayOfYear = (ushort)DateTime.Now.Day; + internal ushort FileCreationYear = (ushort)DateTime.Now.Year; + internal ushort HeaderSize = 375; + internal uint OffsetToPointData = 375; + internal uint NumberOfVariableLengthRecords = 0; + internal byte PointDataRecordFormat = 2; + internal ushort PointDataRecordLength = 0; + internal uint LegacyNbrOfPoints = 0; + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 5)] + internal uint[] LeacyNbrOfPointsByRtn = new uint[5]; + + internal double ScaleFactorX = 0; + internal double ScaleFactorY = 0; + internal double ScaleFactorZ = 0; + + internal double OffsetX = 0; + internal double OffsetY = 0; + internal double OffsetZ = 0; + + internal double MaxX = 0; + internal double MinX = 0; + + internal double MaxY = 0; + internal double MinY = 0; + + internal double MaxZ = 0; + internal double MinZ = 0; + + internal ulong StartOfWaveformPacket = 0; + internal ulong StartOfFirstExtendend = 0; + internal uint NbrOfExtendedVariableLength = 0; + + internal ulong NumberOfPtRecords = 0; + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 15)] + internal ulong[] NbrOfPointsByReturn = new ulong[15]; + } + + /// + /// LAS point type + /// + internal enum LASPointType : byte + { + /// + /// 0 + /// + Zero = 0x0, + /// + /// 1 + /// + One = 0x1, + /// + /// 2 + /// + Two = 0x2, + /// + /// 3 + /// + Three = 0x3, + /// + /// 4 + /// + Four = 0x4, + /// + /// 5 + /// + Five = 0x5, + /// + /// 6 + /// + Six = 0x6, + /// + /// 7 + /// + Seven = 0x7, + /// + /// 8 + /// + Eight = 0x8, + /// + /// 9 + /// + Nine = 0x9, + /// + /// 10 + /// + Ten = 0x10, + } + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + internal struct VariableLengthRecordHeader + { + public VariableLengthRecordHeader() { } + + public ushort Reserved = 0; + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] UserId = new byte[16]; + + public ushort RecordId = 0; + public ushort RecordLengthAfterHeader = 0; + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 32)] + public byte[] Description = new byte[32]; + } + + internal struct InternalVariableLengthRecord + { + public int Type; + public double[] Max; + public double[] Min; + public string Name; + public string Description; + + } + + /// + /// Struct for the Specification Defined VLR "Extra Bytes". This has always 192 bytes. + /// + [StructLayout(LayoutKind.Sequential, Pack = 1)] + internal struct LasExtraBytes + { + public LasExtraBytes() { } + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 2)] + public byte[] reserved = new byte[2]; // 2 bytes + + public byte data_type = 0; // 1 byte + + public byte options = 0; // 1 byte + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 32)] + public byte[] name = new byte[32]; // 32 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + public byte[] unused = new byte[4]; // 4 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] + public byte[] no_data = new byte[8]; // 8 bytes anytype + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] deprecated1 = new byte[16]; // 16 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] + public byte[] min = new byte[8]; // 8 bytes anytype + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] deprecated2 = new byte[16]; // 16 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] + public byte[] max = new byte[8]; // 8 bytes anytype + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] deprecated3 = new byte[16]; // 16 bytes + + public double scale = 0; // 8 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] deprecated4 = new byte[16]; // 16 bytes + + public double offset = 0; // 8 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] deprecated5 = new byte[16]; // 16 bytes + + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 32)] + public byte[] description = new byte[32]; // 32 bytes + } + + /// + /// This class provides methods to convert and saves to LAS 1.4 specification + /// + public class Potree2LAS : IPointWriter, IDisposable + { + /// + /// Path to save the las file to + /// + public FileInfo SavePath { get; private set; } + + /// + /// Metadata (scale, offset) + /// + public IPointWriterMetadata Metadata { get; private set; } + + private readonly Stream _fileStream; + private bool disposedValue; + private LASHeader _header; + private readonly PotreeData _potreeData; + + private readonly List _vlrh = new(); + private readonly List _extraByteDesc = new(); + + /// + /// Generate a writer instance, pass save path, Potree data and optional the point type to write + /// + /// + /// + /// + public Potree2LAS(FileInfo savePath, PotreeData potreeData) + { + Guard.IsNotNull(savePath); + Guard.IsNotNull(potreeData); + Guard.IsTrue(savePath.Extension == ".las"); + + if (savePath.Exists) + { + Diagnostics.Warn($"{savePath.FullName} does already exists. Overwriting ..."); + savePath.Delete(); + } + + SavePath = savePath; + Metadata = potreeData.Metadata; + _fileStream = SavePath.OpenWrite(); + _potreeData = potreeData; + ParseAndFillHeader(); + } + + private void ParseAndFillHeader() + { + var doy = DateTime.Now.DayOfYear; + var year = DateTime.Now.Year; + var size = Metadata.PointSize - 1; + + // make sure we can cast to and do not lose information + Guard.IsLessThan(doy, ushort.MaxValue); + Guard.IsLessThan(year, ushort.MaxValue); + Guard.IsLessThan(size, ushort.MaxValue); + + // automatic point guessing + var ptSize = _potreeData.Metadata.OffsetToExtraBytes == - 1 ? size : (_potreeData.Metadata.OffsetToExtraBytes - 1); + LASPointType ptType = ptSize switch + { + 20 => LASPointType.Zero, + 28 => LASPointType.One, + 26 => LASPointType.Two, + 34 => LASPointType.Three, + 57 => LASPointType.Four, + 63 => LASPointType.Five, + 30 => LASPointType.Six, + 36 => LASPointType.Seven, + 38 => LASPointType.Eight, + 59 => LASPointType.Nine, + 67 => LASPointType.Ten, + _ => throw new NotImplementedException($"Potree point type of size {ptSize} not supported."), + }; + + // Note: AABB is not y/z flipped, offset and scale is. + _header = new LASHeader + { + PointDataRecordLength = (ushort)size, + FileCreationDayOfYear = (ushort)doy, + FileCreationYear = (ushort)year, + MaxX = Metadata.AABB.max.x, + MaxY = Metadata.AABB.max.y, + MaxZ = Metadata.AABB.max.z, + MinX = Metadata.AABB.min.x, + MinY = Metadata.AABB.min.y, + MinZ = Metadata.AABB.min.z, + OffsetX = Metadata.Offset.x, + OffsetY = Metadata.Offset.z, + OffsetZ = Metadata.Offset.y, + ScaleFactorX = Metadata.Scale.x, + ScaleFactorY = Metadata.Scale.z, + ScaleFactorZ = Metadata.Scale.y, + NumberOfPtRecords = (ulong)Metadata.PointCount, + LegacyNbrOfPoints = (uint)Metadata.PointCount, + PointDataRecordFormat = (byte)ptType + }; + + _header.LeacyNbrOfPointsByRtn[0] = (uint)Metadata.PointCount; + _header.NbrOfPointsByReturn[0] = (ulong)Metadata.PointCount; + + var generatingSoftware = Encoding.UTF8.GetBytes($"Fusee v.{Assembly.GetExecutingAssembly().GetName().Version}\0"); + Guard.IsLessThan(generatingSoftware.Length, _header.GeneratingSoftware.Length); + Array.Copy(generatingSoftware, _header.GeneratingSoftware, generatingSoftware.Length); + + if (_potreeData.Metadata.OffsetToExtraBytes != -1) + { + var offset = 0; + var sizeOfExtraBytesAfterHeader = 0; // extra variable for vlr entries + _header.NumberOfVariableLengthRecords++; + + // we have extra bytes to append to each point + // check how many and which + // parse them and add them to the header + // set something for the write method + foreach (var attribute in _potreeData.Metadata.AttributesList) + { + if (attribute == null) continue; + if (offset >= _potreeData.Metadata.OffsetToExtraBytes) + { + + var desc = Encoding.ASCII.GetBytes(attribute.Description.Append('\0').ToArray()); + var name = Encoding.ASCII.GetBytes(attribute.Name.Append('\0').ToArray()); + + var extraByteType = attribute.Type switch + { + //see Las Specification + "uint8" => 1, // uchar + "int8" => 2, // char + "uint16" => 3, // ushort + "int16" => 4, // short + "uint32" => 5, // ulong + "int32" => 6, // long + "int64" => 7, // longlong + "uint64" => 8, // ulonglong, + "float" => 9, // float + "double" => 10 // double +, + _ => throw new ArgumentException("Invalid data type!") + }; + + var currentExtra = new LasExtraBytes + { + data_type = (byte)extraByteType + }; + + Guard.IsLessThan(desc.Length, currentExtra.description.Length); + Guard.IsLessThan(name.Length, currentExtra.name.Length); + + Array.Copy(desc, currentExtra.description, desc.Length); + Array.Copy(name, currentExtra.name, name.Length); + + var extraByteMarshalMin = attribute.Type switch + { + //see Las Specification + "uint8" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (sbyte)x).ToArray()), + "int8" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (byte)x).ToArray()), + "uint16" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (ushort)x).ToArray()), + "int16" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (short)x).ToArray()), + "uint32" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (ulong)x).ToArray()), + "int32" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (long)x).ToArray()), + "int64" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (long)x).ToArray()), + "uint64" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (ulong)x).ToArray()), + "float" => MemoryMarshal.AsBytes(attribute.MinList.Select(x => (float)x).ToArray()), + "double" => MemoryMarshal.AsBytes(attribute.MinList.ToArray()), + _ => throw new ArgumentException("Invalid data type!") + }; + + var extraByteMarshalMax = attribute.Type switch + { + //see Las Specification + "uint8" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (sbyte)x).ToArray()), + "int8" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (byte)x).ToArray()), + "uint16" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (ushort)x).ToArray()), + "int16" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (short)x).ToArray()), + "uint32" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (ulong)x).ToArray()), + "int32" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (long)x).ToArray()), + "int64" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (long)x).ToArray()), + "uint64" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (ulong)x).ToArray()), + "float" => MemoryMarshal.AsBytes(attribute.MaxList.Select(x => (float)x).ToArray()), + "double" => MemoryMarshal.AsBytes(attribute.MaxList.ToArray()), + _ => throw new ArgumentException("Invalid data type!") + }; + + var min = extraByteMarshalMin.ToArray(); + var max = extraByteMarshalMax.ToArray(); + Array.Copy(min, currentExtra.min, min.Length); + Array.Copy(max, currentExtra.max, max.Length); + + _extraByteDesc.Add(currentExtra); + + _header.OffsetToPointData += 192; + // each description is 192 bytes + sizeOfExtraBytesAfterHeader += 192; + } + + offset += attribute.Size; + } + + // add the actual variable record header + var vlr = new VariableLengthRecordHeader + { + RecordLengthAfterHeader = (ushort)sizeOfExtraBytesAfterHeader, // offset with all extraBytes + RecordId = 4 + }; + + var description = Encoding.UTF8.GetBytes("Extra Bytes Record\0"); + var userId = Encoding.UTF8.GetBytes("LASF_Spec\0"); + + Guard.IsLessThan(description.Length, vlr.Description.Length); + Guard.IsLessThan(userId.Length, vlr.UserId.Length); + Array.Copy(description, vlr.Description, description.Length); + Array.Copy(userId, vlr.UserId, userId.Length); + + // this is just for the LAS header, we should check if we can already + // build an internal list to update later when writing the actual extra bytes + _vlrh.Add(vlr); + + _header.OffsetToPointData += 54; + // LAS 1.4 Spec: Each Variable Length Record Header is 54 bytes in length. + // add, too. Complete size: header + variableLengthRecord + n * extraBytes (192bytes) + } + + // Initialize unmanged memory to hold the struct. + var headerPtr = Marshal.AllocHGlobal(Marshal.SizeOf()); + try + { + // Copy the struct to unmanaged memory. + Marshal.StructureToPtr(_header, headerPtr, false); + var dest = new byte[Marshal.SizeOf()]; + Marshal.Copy(headerPtr, dest, 0, Marshal.SizeOf()); + _fileStream.Write(dest); + } + finally + { + // Free the unmanaged memory. + Marshal.FreeHGlobal(headerPtr); + } + + // append all variable length record header + foreach (var vlr in _vlrh) + { + var vlrPtr = Marshal.AllocHGlobal(Marshal.SizeOf()); + try + { + // Copy the struct to unmanaged memory. + Marshal.StructureToPtr(vlr, vlrPtr, false); + var dest = new byte[Marshal.SizeOf()]; + Marshal.Copy(vlrPtr, dest, 0, Marshal.SizeOf()); + _fileStream.Write(dest); + } + finally + { + // Free the unmanaged memory. + Marshal.FreeHGlobal(vlrPtr); + } + } + + foreach (var extraByte in _extraByteDesc) + { + var extraBytePtr = Marshal.AllocHGlobal(Marshal.SizeOf()); + try + { + // Copy the struct to unmanaged memory. + Marshal.StructureToPtr(extraByte, extraBytePtr, false); + var dest = new byte[Marshal.SizeOf()]; + Marshal.Copy(extraBytePtr, dest, 0, Marshal.SizeOf()); + _fileStream.Write(dest); + } + finally + { + // Free the unmanaged memory. + Marshal.FreeHGlobal(extraBytePtr); + } + } + } + + /// + /// This methods starts the LASfile write progress. + /// + public void Write() + { + Guard.IsNotNull(_header); + + // advance to end of stream + _fileStream.Seek(0, SeekOrigin.End); + var fileLength = (long)Metadata.PointCount * (long)Metadata.PointSize; + + // set complete length before writing, this generates the full file + // writing operations are much faster afterwards + _fileStream.SetLength(fileLength); + + using var inputStream = _potreeData.OctreeMappedFile.CreateViewStream(); + + Span tmpArray = stackalloc byte[_potreeData.Metadata.PointSize]; + + // DO NOT USE stream.Length as the MemoryMappedStream aligns with the page size + for (long i = 0U; i < fileLength; i += Metadata.PointSize) + { + // we need to shrink each point back (skip byte 16) + // extra bytes and everything is already included + inputStream.Read(tmpArray); + _fileStream.Write(tmpArray[..15]); // pos(12) + intensity (2) + returnStuff (1) + _fileStream.Write(tmpArray[16..Metadata.PointSize]); // skip array pos [15], byte 16 + } + + inputStream.Close(); + } + + /// + /// Dispose the of this class. + /// + /// + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + // close the file stream + _fileStream.Dispose(); + } + + disposedValue = true; + } + } + + /// + /// Dispose + /// + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/Data/PotreeData.cs b/src/PointCloud/Potree/V2/Data/PotreeData.cs new file mode 100644 index 000000000..c2827dd34 --- /dev/null +++ b/src/PointCloud/Potree/V2/Data/PotreeData.cs @@ -0,0 +1,124 @@ +using CommunityToolkit.Diagnostics; +using Fusee.PointCloud.Common; +using System; +using System.Diagnostics; +using System.IO; +using System.IO.MemoryMappedFiles; + +namespace Fusee.PointCloud.Potree.V2.Data +{ + /// + /// Contains information about the Potree file's meta data and hierarchy/octree. + /// + public class PotreeData : IDisposable + { + /// + /// The hierarchy as linear list of s. + /// + public PotreeHierarchy Hierarchy; + + /// + /// The meta data of the file. + /// + public PotreeMetadata Metadata; + + /// + /// Returns a reference to the memory mapped file for octree.bin. + /// + internal MemoryMappedFile OctreeMappedFile { get; } + + /// + /// Returns a reference to the memory mapped file view accessor for reading. + /// + internal MemoryMappedViewAccessor ReadViewAccessor { get; } + + /// + /// Returns a reference to the memory mapped file view accessor for writing. + /// + internal MemoryMappedViewAccessor WriteViewAccessor { get; } + + + /// + /// Creats a new instance of PotreeData + /// + /// + /// + public PotreeData(PotreeHierarchy potreeHierarchy, PotreeMetadata potreeMetadata) + { + Hierarchy = potreeHierarchy; + Metadata = potreeMetadata; + + var path = Path.Combine(Metadata.FolderPath, Potree2Consts.OctreeFileName); + + OctreeMappedFile = MemoryMappedFile.CreateFromFile(path, FileMode.Open); + ReadViewAccessor = OctreeMappedFile.CreateViewAccessor(); + WriteViewAccessor = OctreeMappedFile.CreateViewAccessor(); + + Guard.IsTrue(CheckAllFileStreamsValidity()); + } + + /// + /// Returns the node for a given . + /// + /// + /// + public PotreeNode? GetNode(OctantId octantId) + { + return Hierarchy.Nodes.Find(n => n.OctantId == octantId); + } + + /// + /// Returns the node for a given name of a . + /// + /// + /// + public PotreeNode? GetNode(string nodeName) + { + var octantId = new OctantId(nodeName); + + return GetNode(octantId); + } + + /// + /// Checks if all filestreams are open and valid + /// + /// if any errors are present + public bool CheckAllFileStreamsValidity() + { + return !OctreeMappedFile.SafeMemoryMappedFileHandle.IsInvalid && + !OctreeMappedFile.SafeMemoryMappedFileHandle.IsClosed && + ReadViewAccessor.CanRead && WriteViewAccessor.CanWrite; + } + + #region IDisposable + + private bool disposedValue; + + /// + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + ReadViewAccessor.Flush(); + WriteViewAccessor.Flush(); + ReadViewAccessor.Dispose(); + WriteViewAccessor.Dispose(); + OctreeMappedFile.Dispose(); + } + + disposedValue = true; + } + } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + #endregion IDisposable + } +} \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/Data/PotreeMetadata.cs b/src/PointCloud/Potree/V2/Data/PotreeMetadata.cs index 99f762a57..e40c05929 100644 --- a/src/PointCloud/Potree/V2/Data/PotreeMetadata.cs +++ b/src/PointCloud/Potree/V2/Data/PotreeMetadata.cs @@ -1,4 +1,5 @@ using Fusee.Math.Core; +using Fusee.PointCloud.Common; using Newtonsoft.Json; using System.Collections.Generic; @@ -6,7 +7,7 @@ namespace Fusee.PointCloud.Potree.V2.Data { - public class PotreeSettingsHierarchy + public class PotreeSettingsHierarchy : IPointWriterHierarchy { public int FirstChunkSize { get; set; } public int StepSize { get; set; } @@ -37,26 +38,29 @@ public class PotreeSettingsAttribute [JsonProperty(PropertyName = "min")] public List MinList { get; set; } - [JsonIgnore] - public double3 Min => new(MinList[0], MinList[1], MinList[2]); [JsonProperty(PropertyName = "max")] public List MaxList { get; set; } - [JsonIgnore] - public double3 Max => new(MaxList[0], MaxList[1], MaxList[2]); [JsonIgnore] public int AttributeOffset { get; set; } + + [JsonIgnore] + public bool IsExtraByte { get; set; } = false; } - public class PotreeMetadata + + public class PotreeMetadata : IPointWriterMetadata { public string Version { get; set; } public string Name { get; set; } public string Description { get; set; } - public int Points { get; set; } + [JsonProperty(PropertyName = "points")] + public int PointCount { get; set; } + public int OffsetToExtraBytes { get; set; } = -1; public string Projection { get; set; } - public PotreeSettingsHierarchy Hierarchy { get; set; } + + public IPointWriterHierarchy Hierarchy { get; set; } [JsonProperty(PropertyName = "offset")] public List OffsetList { get; set; } @@ -70,6 +74,9 @@ public class PotreeMetadata public double Spacing { get; set; } public PotreeSettingsBoundingBox BoundingBox { get; set; } + + public AABBd AABB => new(BoundingBox.Min, BoundingBox.Max); + public string Encoding { get; set; } [JsonProperty(PropertyName = "attributes")] @@ -82,6 +89,10 @@ public class PotreeMetadata [JsonIgnore] public string FolderPath { get; set; } + + [JsonIgnore] + public float4x4 PrincipalAxisRotation { get; set; } + } } diff --git a/src/PointCloud/Potree/V2/Data/PotreeNode.cs b/src/PointCloud/Potree/V2/Data/PotreeNode.cs index 84d9f58b1..cda17dd87 100644 --- a/src/PointCloud/Potree/V2/Data/PotreeNode.cs +++ b/src/PointCloud/Potree/V2/Data/PotreeNode.cs @@ -1,4 +1,7 @@ -using Fusee.Math.Core; +using CommunityToolkit.Diagnostics; +using Fusee.Math.Core; +using Fusee.PointCloud.Common; +using Newtonsoft.Json; using System; #pragma warning disable CS1591 @@ -11,7 +14,22 @@ public PotreeNode() { } - public string Name = ""; + [JsonIgnore] + private string _name; + + public string Name + { + get { return _name; } + set + { + Guard.IsNotNull(value); + _name = value; + OctantId = new OctantId(value); + } + } + + [JsonIgnore] + public OctantId OctantId; public AABBd Aabb { get; set; } public PotreeNode Parent { get; set; } diff --git a/src/PointCloud/Potree/V2/Data/PotreePoint.cs b/src/PointCloud/Potree/V2/Data/PotreePoint.cs deleted file mode 100644 index d03d8be07..000000000 --- a/src/PointCloud/Potree/V2/Data/PotreePoint.cs +++ /dev/null @@ -1,17 +0,0 @@ -using Fusee.Math.Core; - -namespace Fusee.PointCloud.Potree.V2.Data -{ - public class PotreePoint - { - public double3 Position; - public short Intensity; - public byte ReturnNumber; - public byte NumberOfReturns; - public byte Classification; - public byte ScanAngleRank; - public byte UserData; - public byte PointSourceId; - public float3 Color; - } -} \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/Potree2AccessBase.cs b/src/PointCloud/Potree/V2/Potree2AccessBase.cs new file mode 100644 index 000000000..894d410f4 --- /dev/null +++ b/src/PointCloud/Potree/V2/Potree2AccessBase.cs @@ -0,0 +1,165 @@ +using CommunityToolkit.Diagnostics; +using Fusee.PointCloud.Potree.V2.Data; + +namespace Fusee.PointCloud.Potree.V2 +{ + /// + /// This is the base class for accessing Potree files + /// + public abstract class Potree2AccessBase + { + /// + /// The + /// + public PotreeData? PotreeData { get; set; } + + /// + /// Constructs a new instance of Potree2AccessBase + /// + /// + public Potree2AccessBase(PotreeData potreeData) + { + PotreeData = potreeData; + } + + /// + /// Constructs a new instance of Potree2AccessBase + /// + protected Potree2AccessBase() { } + + /// + /// Returns the raw data of a + /// + /// + /// + internal byte[] ReadRawNodeData(PotreeNode node) + { + Guard.IsLessThanOrEqualTo(node.NumPoints, int.MaxValue); + Guard.IsNotNull(PotreeData); + + var potreePointSize = (int)node.NumPoints * PotreeData.Metadata.PointSize; + var pointArray = new byte[potreePointSize]; + PotreeData.ReadViewAccessor.ReadArray(node.ByteOffset, pointArray, 0, potreePointSize); + + return pointArray; + } + + #region Metadata caching + + /// + /// Save if metadata has already been cached + /// + protected bool _isMetadataCached = false; + + /// + /// Offset in bytes to the position value in bytes in raw Potree stream + /// + protected int offsetPosition = -1; + /// + /// Offset in bytes to the intensity value in bytes in raw Potree stream + /// + protected int offsetIntensity = -1; + /// + /// Offset in bytes to the return number value in bytes in raw Potree stream + /// + protected int offsetReturnNumber = -1; + /// + /// Offset in bytes to the number of returns value in bytes in raw Potree stream + /// + protected int offsetNumberOfReturns = -1; + /// + /// Offset in bytes to the classification value in bytes in raw Potree stream + /// + protected int offsetClassification = -1; + /// + /// Offset in bytes to the scan angle rank value in bytes in raw Potree stream + /// + protected int offsetScanAngleRank = -1; + /// + /// Offset in bytes to the user data value in bytes in raw Potree stream + /// + protected int offsetUserData = -1; + /// + /// Offset in bytes to the point source id value in bytes in raw Potree stream + /// + protected int offsetPointSourceId = -1; + /// + /// Offset in bytes to the color value in bytes in raw Potree stream + /// + protected int offsetColor = -1; + + /// + /// Read and cache metadata from the metadata.json file + /// + protected void CacheMetadata(bool force = false) + { + Guard.IsNotNull(PotreeData); + + if (!_isMetadataCached || force) + { + offsetPosition = -1; + offsetIntensity = -1; + offsetReturnNumber = -1; + offsetNumberOfReturns = -1; + offsetClassification = -1; + offsetScanAngleRank = -1; + offsetUserData = -1; + offsetPointSourceId = -1; + offsetColor = -1; + + if (PotreeData.Metadata.Attributes.TryGetValue("position", out var position)) + { + offsetPosition = position.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("intensity", out var intensity)) + { + offsetIntensity = intensity.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("return number", out var returnNumber)) + { + offsetReturnNumber = returnNumber.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("number of returns", out var numberOfReturns)) + { + offsetNumberOfReturns = numberOfReturns.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("classification", out var classification)) + { + offsetClassification = classification.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("scan angle rank", out var scanAngleRank)) + { + offsetScanAngleRank = scanAngleRank.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("user data", out var userData)) + { + offsetUserData = userData.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("point source id", out var pointSourceId)) + { + offsetPointSourceId = pointSourceId.AttributeOffset; + } + if (PotreeData.Metadata.Attributes.TryGetValue("rgb", out var rgb)) + { + offsetColor = rgb.AttributeOffset; + } + + int pointSize = 0; + + if (PotreeData.Metadata != null) + { + foreach (var metaAttributeItem in PotreeData.Metadata.AttributesList) + { + pointSize += metaAttributeItem.Size; + } + + PotreeData.Metadata.PointSize = pointSize; + } + + _isMetadataCached = true; + } + } + + #endregion Metadata caching + } +} \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/Potree2Reader.cs b/src/PointCloud/Potree/V2/Potree2Reader.cs index 3f4e6483c..bcd8df0c0 100644 --- a/src/PointCloud/Potree/V2/Potree2Reader.cs +++ b/src/PointCloud/Potree/V2/Potree2Reader.cs @@ -1,27 +1,68 @@ +using CommunityToolkit.Diagnostics; +using CommunityToolkit.HighPerformance; +using CommunityToolkit.HighPerformance.Buffers; +using Fusee.Base.Core; using Fusee.Engine.Core; using Fusee.Engine.Core.Scene; using Fusee.Math.Core; using Fusee.PointCloud.Common; -using Fusee.PointCloud.Common.Accessors; using Fusee.PointCloud.Core; -using Fusee.PointCloud.Core.Accessors; using Fusee.PointCloud.Core.Scene; using Fusee.PointCloud.Potree.V2.Data; +using Newtonsoft.Json; using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; using System.IO; +using System.Linq; +using System.Runtime.InteropServices; namespace Fusee.PointCloud.Potree.V2 { + /// + /// Delegate for a method that knows how to parse a slice of a point's extra bytes to a valid uint. + /// + /// + /// + public delegate uint HandleReadExtraBytes(Span bytes); + /// /// Reads Potree V2 files and is able to create a point cloud scene component, that can be rendered. /// - public class Potree2Reader : Potree2RwBase, IPointReader + public class Potree2Reader : Potree2AccessBase, IPointReader { /// - /// Initializes a Potree 2 reader for the given Potree dataset + /// Pass method how to handle the extra bytes, resulting uint will be passed into . + /// + public HandleReadExtraBytes? HandleReadExtraBytes { get; set; } + + /// + /// If any errors during load occur this event is being called + /// + public EventHandler? OnPointCloudReadError; + + /// + /// Specify the byte offset for one point until the extra byte data is reached + /// + public int OffsetToExtraBytes = -1; + + /// + /// Generate a new instance of . + /// + /// + public Potree2Reader(string filepath) + { + ReadNewFile(filepath); + } + + /// + /// Generate a new instance of . /// /// - public Potree2Reader(ref PotreeData potreeData) : base(ref potreeData) { } + public Potree2Reader(PotreeData potreeData) : base(potreeData) + { + ReadFile(potreeData); + } /// /// Returns a renderable point cloud component. @@ -34,25 +75,25 @@ public IPointCloud GetPointCloudComponent(RenderMode renderMode = RenderMode.Sta default: case RenderMode.StaticMesh: { - var dataHandler = new PointCloudDataHandler( - (PointAccessor)PointAccessor, MeshMaker.CreateMeshPosD3ColF3LblB, - LoadNodeData); + var dataHandler = new PointCloudDataHandler(MeshMaker.CreateStaticMesh, + LoadVisualizationPointData); + dataHandler.OnLoadingErrorEvent += OnPointCloudReadError; var imp = new Potree2Cloud(dataHandler, GetOctree()); return new PointCloudComponent(imp, renderMode); } case RenderMode.Instanced: { - var dataHandlerInstanced = new PointCloudDataHandler( - (PointAccessor)PointAccessor, MeshMaker.CreateInstanceDataPosD3ColF3LblB, - LoadNodeData, true); + var dataHandlerInstanced = new PointCloudDataHandler(MeshMaker.CreateInstanceData, + LoadVisualizationPointData, true); + dataHandlerInstanced.OnLoadingErrorEvent += OnPointCloudReadError; var imp = new Potree2CloudInstanced(dataHandlerInstanced, GetOctree()); return new PointCloudComponent(imp, renderMode); } case RenderMode.DynamicMesh: { - var dataHandlerDynamic = new PointCloudDataHandler( - (PointAccessor)PointAccessor, MeshMaker.CreateDynamicMeshPosD3ColF3LblB, - LoadNodeData, true); + var dataHandlerDynamic = new PointCloudDataHandler(MeshMaker.CreateDynamicMesh, + LoadVisualizationPointData); + dataHandlerDynamic.OnLoadingErrorEvent += OnPointCloudReadError; var imp = new Potree2CloudDynamic(dataHandlerDynamic, GetOctree()); return new PointCloudComponent(imp, renderMode); } @@ -65,280 +106,434 @@ public IPointCloud GetPointCloudComponent(RenderMode renderMode = RenderMode.Sta /// public IPointCloudOctree GetOctree() { - int pointSize = 0; + Guard.IsNotNull(PotreeData); + Guard.IsNotNull(PotreeData.Metadata); + + var center = PotreeData.Hierarchy.Root.Aabb.Center; + var size = PotreeData.Hierarchy.Root.Aabb.Size.y; + var maxLvl = PotreeData.Metadata.Hierarchy.Depth; + + var octree = new PointCloudOctree(center, size, maxLvl); + + MapChildNodesRecursive(octree.Root, PotreeData.Hierarchy.Root); + + return octree; + } + + /// + /// Reads the points for a specific octant of type . + /// + /// Id of the octant. + /// + public MemoryOwner LoadVisualizationPointData(OctantId id) + { + Guard.IsNotNull(PotreeData); + var node = PotreeData.GetNode(id); + + // if node is null the hierarchy is broken and we look for an octant that isn't there... + Guard.IsNotNull(node); - if (_potreeData.Metadata != null) + return LoadVisualizationPoint(node); + } + + private MemoryOwner LoadVisualizationPoint(PotreeNode node) + { + Guard.IsLessThanOrEqualTo(node.NumPoints, int.MaxValue); + //if (HandleExtraBytes != null) + // Guard.IsGreaterThan(OffsetToExtraBytes, 0); + Guard.IsNotNull(PotreeData); + + var pointArray = ReadRawNodeData(node); + + var returnMemory = MemoryOwner.Allocate((int)node.NumPoints); + + var pointCount = 0; + + for (var i = 0; i < pointArray.Length; i += PotreeData.Metadata.PointSize) { - foreach (var metaAttributeItem in _potreeData.Metadata.AttributesList) + var posSlice = new Span(pointArray).Slice(i + offsetPosition, Marshal.SizeOf() * 3); + var pos = MemoryMarshal.Cast(posSlice); + + double x = pos[0] * PotreeData.Metadata.Scale.x; + double y = pos[1] * PotreeData.Metadata.Scale.y; + double z = pos[2] * PotreeData.Metadata.Scale.z; + + float3 position = new((float)x, (float)y, (float)z); + position = (float4x4)Potree2Consts.YZflip * position; + + var posSpan = MemoryMarshal.Cast(position.ToArray()); + + Span colorSlice; + Span rgb; + float4 color; + if (offsetColor != -1) { - pointSize += metaAttributeItem.Size; + colorSlice = new Span(pointArray).Slice(i + offsetColor, Marshal.SizeOf() * 3); + rgb = MemoryMarshal.Cast(colorSlice); + + color = float4.Zero; + + color.r = ((byte)(rgb[0] > 255 ? rgb[0] / 256 : rgb[0])); + color.g = ((byte)(rgb[1] > 255 ? rgb[1] / 256 : rgb[1])); + color.b = ((byte)(rgb[2] > 255 ? rgb[2] / 256 : rgb[2])); + color.a = 1; + } + else if (offsetIntensity != -1) + { + var attrib = PotreeData.Metadata.Attributes["intensity"]; + colorSlice = new Span(pointArray).Slice(i + offsetIntensity, Marshal.SizeOf()); + rgb = MemoryMarshal.Cast(colorSlice); + color = float4.Zero; + + color.r = (float)((rgb[0] - attrib.MinList[0]) / (attrib.MaxList[0] - attrib.MinList[0]) * 1f); + color.g = color.r; + color.b = color.r; + color.a = 1; + } + else + { + color = float4.UnitW; } - _potreeData.Metadata.PointSize = pointSize; + var colorSpan = MemoryMarshal.Cast(color.ToArray()); + + uint flags = 0; + Span extraBytesSpan = null; + if (PotreeData.Metadata.OffsetToExtraBytes != -1 && PotreeData.Metadata.OffsetToExtraBytes != 0) + { + var extraByteSize = PotreeData.Metadata.PointSize - PotreeData.Metadata.OffsetToExtraBytes; + extraBytesSpan = pointArray.AsSpan().Slice(i + PotreeData.Metadata.OffsetToExtraBytes, extraByteSize); + } + if (HandleReadExtraBytes != null) + { + try + { + flags = HandleReadExtraBytes(extraBytesSpan); + } + catch (Exception e) + { + OnPointCloudReadError?.Invoke(this, new ErrorEventArgs(e)); + } + } + + var flagsSpan = MemoryMarshal.Cast(new uint[] { flags }); + + var currentMemoryPt = MemoryMarshal.Cast(returnMemory.Span.Slice(pointCount, 1)); + posSpan.CopyTo(currentMemoryPt[..]); + colorSpan.CopyTo(currentMemoryPt[posSpan.Length..]); + flagsSpan.CopyTo(currentMemoryPt.Slice(posSpan.Length + colorSpan.Length, Marshal.SizeOf())); + + pointCount++; } - var center = _potreeData.Hierarchy.Root.Aabb.Center; - var size = _potreeData.Hierarchy.Root.Aabb.Size.y; - var maxLvl = _potreeData.Metadata.Hierarchy.Depth; + return returnMemory; + } - var octree = new PointCloudOctree(center, size, maxLvl); + private static void MapChildNodesRecursive(IPointCloudOctant octreeNode, PotreeNode potreeNode) + { + octreeNode.NumberOfPointsInNode = (int)potreeNode.NumPoints; - MapChildNodesRecursive(octree.Root, _potreeData.Hierarchy.Root); + for (int i = 0; i < potreeNode.Children.Length; i++) + { + if (potreeNode.Children[i] != null) + { + var potreeChild = potreeNode.Children[i]; - return octree; + var octant = new PointCloudOctant(potreeNode.Children[i].Aabb.Center, potreeNode.Children[i].Aabb.Size.y, new OctantId(potreeChild.Name)); + + if (potreeChild.NodeType == NodeType.LEAF) + { + octant.IsLeaf = true; + } + + MapChildNodesRecursive(octant, potreeChild); + + octreeNode.Children[i] = octant; + } + } } /// - /// Returns the points for one octant as generic array. + /// Reads a potree file. /// - /// The generic point type. - /// The unique id of the octant. - /// - public TPoint[] LoadNodeData(OctantId id) where TPoint : new() + /// Path to the file. + /// Meta and octree data of the potree file. + public PotreeData ReadNewFile(string path) { - TPoint[] points = null; + (var Metadata, var Hierarchy) = LoadHierarchy(path); - var node = FindNode(ref _potreeData.Hierarchy, id); + PotreeData = new PotreeData(Hierarchy, Metadata); - if (node != null) + foreach (var item in PotreeData.Metadata.Attributes.Values) { - points = LoadNodeData(node); + PotreeData.Metadata.PointSize += item.Size; + if (PotreeData.Metadata.OffsetToExtraBytes > -1 && PotreeData.Metadata.PointSize > PotreeData.Metadata.OffsetToExtraBytes) + item.IsExtraByte = true; + else + item.IsExtraByte = false; } - return points; + CacheMetadata(true); + + PotreeData.Metadata.PrincipalAxisRotation = CalculatePrincipalAxis(PotreeData); + + return PotreeData; } - public TPoint[] LoadNodeData(PotreeNode potreeNode) where TPoint : new() + + + /// + /// Calculate the principal axis rotation from given data + /// - Load root node + /// - Convert to covariance matrix + /// - Calculate principal axis + /// + /// The potree data + /// + private float4x4 CalculatePrincipalAxis(PotreeData data) { - TPoint[] points = null; + using var allPoints = LoadVisualizationPoint(data.Hierarchy.Root); + using var positions = MemoryOwner.Allocate(allPoints.Length); + + var allPtsBytes = allPoints.Span.AsBytes(); - if (potreeNode != null) + // convert all points to byte, slice the position value (stride/offset) and copy to position array + for (var i = 0; i < allPoints.Length; i++) { - points = ReadNodeData(potreeNode); - potreeNode.IsLoaded = true; + positions.Span[i] = allPoints.Span[i].Position; } - return points; + // dangerous, undefined behavior -> do not use the array values after this method + // this is irrelevant, as we use only a local variable + try + { + var eigen = new Eigen(positions.DangerousGetArray().Array); + return (float4x4)eigen.RotationMatrix; + } + catch (ArithmeticException ex) + { + Diagnostics.Warn(ex); + return float4x4.Identity; + } } - private TPoint[] ReadNodeData(PotreeNode node) where TPoint : new() + + + /// + /// Changes the potree data package that is currently bound to the reader. So a reader can be used for multiple data packages, this avoids rereading the potree data like in . + /// + /// Meta and octree data of the potree file. + public void ReadFile(PotreeData potreeData) { - var points = new TPoint[node.NumPoints]; - for (int i = 0; i < node.NumPoints; i++) - { - points[i] = new TPoint(); - } + PotreeData = potreeData; - var binaryReader = new BinaryReader(File.OpenRead(OctreeFilePath)); + CacheMetadata(true); + } + + #region LoadHierarchy + + private (PotreeMetadata, PotreeHierarchy) LoadHierarchy(string folderPath) + { + var metadataFilePath = Path.Combine(folderPath, Potree2Consts.MetadataFileName); + var hierarchyFilePath = Path.Combine(folderPath, Potree2Consts.HierarchyFileName); + + Guard.IsTrue(File.Exists(metadataFilePath), metadataFilePath); + Guard.IsTrue(File.Exists(metadataFilePath), hierarchyFilePath); - // Commented code is to read the entire Potree2 file format. Since we don't use everything atm unused - // things are commented for performance. - for (int i = 0; i < node.NumPoints; i++) + var Metadata = LoadPotreeMetadata(metadataFilePath); + var Hierarchy = new PotreeHierarchy() { - if (offsetPosition > -1) + Root = new() { - binaryReader.BaseStream.Position = node.ByteOffset + offsetPosition + i * _potreeData.Metadata.PointSize; + Name = "r", + } + }; - double x = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.x; - double y = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.y; - double z = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.z; + Metadata.Attributes = GetAttributesDict(Metadata.AttributesList); - double3 position = new(x, y, z); - position = Potree2Consts.YZflip * position; + Metadata.FolderPath = folderPath; - ((PointAccessor)PointAccessor).SetPositionFloat3_64(ref points[i], position); - } + CalculateAttributeOffsets(ref Metadata); - //if (offsetIntensity > -1) - //{ - // binaryReader.BaseStream.Position = node.ByteOffset + offsetIntensity + i * _potreeData.Metadata.PointSize; - // Int16 intensity = binaryReader.ReadInt16(); - //} - //if (offsetReturnNumber > -1) - //{ - // binaryReader.BaseStream.Position = node.ByteOffset + offsetReturnNumber + i * _potreeData.Metadata.PointSize; - // byte returnNumber = binaryReader.ReadByte(); - //} - //if (offsetNumberOfReturns > -1) - //{ - // binaryReader.BaseStream.Position = node.ByteOffset + offsetNumberOfReturns + i * _potreeData.Metadata.PointSize; - // byte numberOfReturns = binaryReader.ReadByte(); - //} - - if (offsetClassification > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetClassification + i * _potreeData.Metadata.PointSize; + Hierarchy.Root.Aabb = new AABBd(Metadata.BoundingBox.Min, Metadata.BoundingBox.Max); - byte label = binaryReader.ReadByte(); + var data = File.ReadAllBytes(hierarchyFilePath); - ((PointAccessor)PointAccessor).SetLabelUInt_8(ref points[i], label); - } + Guard.IsNotNull(data, nameof(data)); - //else if (offsetScanAngleRank > -1) - //{ - // binaryReader.BaseStream.Position = node.ByteOffset + offsetScanAngleRank + i * _potreeData.Metadata.PointSize; - // byte scanAngleRank = binaryReader.ReadByte(); - //} - //else if (offsetUserData > -1) - //{ - // binaryReader.BaseStream.Position = node.ByteOffset + offsetUserData + i * _potreeData.Metadata.PointSize; - // byte userData = binaryReader.ReadByte(); - //} - //else if (offsetPointSourceId > -1) - //{ - // binaryReader.BaseStream.Position = node.ByteOffset + offsetPointSourceId + i * _potreeData.Metadata.PointSize; - // byte pointSourceId = binaryReader.ReadByte(); - //} - - if (offsetColor > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetColor + i * _potreeData.Metadata.PointSize; + LoadHierarchyRecursive(ref Hierarchy.Root, ref data, 0, Metadata.Hierarchy.FirstChunkSize); - ushort r = binaryReader.ReadUInt16(); - ushort g = binaryReader.ReadUInt16(); - ushort b = binaryReader.ReadUInt16(); + Hierarchy.Nodes = new(); + Hierarchy.Root.Traverse(n => Hierarchy.Nodes.Add(n)); - float3 color = float3.Zero; + FlipYZAxis(Metadata, Hierarchy); - color.r = ((byte)(r > 255 ? r / 256 : r)); - color.g = ((byte)(g > 255 ? g / 256 : g)); - color.b = ((byte)(b > 255 ? b / 256 : b)); + // adapt the global AABB after conversion, this works with the current LAS writer + Metadata.BoundingBox.MinList = new List(3) { Hierarchy.Root.Aabb.min.x + Metadata.Offset.x, Hierarchy.Root.Aabb.min.z + Metadata.Offset.z, Hierarchy.Root.Aabb.min.y + Metadata.Offset.y }; + Metadata.BoundingBox.MaxList = new List(3) { Hierarchy.Root.Aabb.max.x + Metadata.Offset.x, Hierarchy.Root.Aabb.max.z + Metadata.Offset.z, Hierarchy.Root.Aabb.max.y + Metadata.Offset.y }; - ((PointAccessor)PointAccessor).SetColorFloat3_32(ref points[i], color); - } - } + return (Metadata, Hierarchy); + } - binaryReader.Close(); - binaryReader.Dispose(); + private static PotreeMetadata LoadPotreeMetadata(string metadataFilepath) + { + var settings = new JsonSerializerSettings(); + settings.Converters.Add(new ConvertIPointWriterHierarchy()); + var metaData = File.ReadAllText(metadataFilepath); + var potreeData = JsonConvert.DeserializeObject(metaData, settings); + Guard.IsNotNull(potreeData, nameof(potreeData)); - return points; + return potreeData; } - public TPotreePoint[] ReadRawPoints(OctantId oid) where TPotreePoint : PotreePoint, new() + + internal class ConvertIPointWriterHierarchy : JsonConverter { - var node = FindNode(ref _potreeData.Hierarchy, oid); + public override bool CanConvert(Type objectType) + { + return objectType == typeof(IPointWriterHierarchy); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + return serializer.Deserialize(reader, typeof(PotreeSettingsHierarchy)); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value); + } + } + - var points = new TPotreePoint[node.NumPoints]; - Array.Fill(points, new TPotreePoint()); - var binaryReader = new BinaryReader(File.OpenRead(OctreeFilePath)); + private static void LoadHierarchyRecursive(ref PotreeNode root, ref byte[] data, long offset, long size) + { + int bytesPerNode = 22; + int numNodes = (int)(size / bytesPerNode); - for (int i = 0; i < node.NumPoints; i++) + var nodes = new List(numNodes) { - if (offsetPosition > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetPosition + i * _potreeData.Metadata.PointSize; + root + }; - double x = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.x; - double y = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.y; - double z = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.z; + for (int i = 0; i < numNodes; i++) + { + var currentNode = nodes[i]; + if (currentNode == null) + currentNode = new PotreeNode(); - double3 position = new(x, y, z); - position = Potree2Consts.YZflip * position; + ulong offsetNode = (ulong)offset + (ulong)(i * bytesPerNode); - points[i].Position = position; - } + var nodeType = data[offsetNode + 0]; + int childMask = BitConverter.ToInt32(data, (int)offsetNode + 1); + var numPoints = BitConverter.ToUInt32(data, (int)offsetNode + 2); + var byteOffset = BitConverter.ToInt64(data, (int)offsetNode + 6); + var byteSize = BitConverter.ToInt64(data, (int)offsetNode + 14); - if (offsetIntensity > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetIntensity + i * _potreeData.Metadata.PointSize; - Int16 intensity = binaryReader.ReadInt16(); + currentNode.NodeType = (NodeType)nodeType; + currentNode.NumPoints = numPoints; + currentNode.ByteOffset = byteOffset; + currentNode.ByteSize = byteSize; - points[i].Intensity = intensity; - } - if (offsetReturnNumber > -1) + if (currentNode.NodeType == NodeType.PROXY) { - binaryReader.BaseStream.Position = node.ByteOffset + offsetReturnNumber + i * _potreeData.Metadata.PointSize; - byte returnNumber = binaryReader.ReadByte(); - - points[i].ReturnNumber = returnNumber; + LoadHierarchyRecursive(ref currentNode, ref data, byteOffset, byteSize); } - if (offsetNumberOfReturns > -1) + else { - binaryReader.BaseStream.Position = node.ByteOffset + offsetNumberOfReturns + i * _potreeData.Metadata.PointSize; - byte numberOfReturns = binaryReader.ReadByte(); + for (int childIndex = 0; childIndex < 8; childIndex++) + { + bool childExists = (1 << childIndex & childMask) != 0; - points[i].NumberOfReturns = numberOfReturns; - } + if (!childExists) + { + continue; + } - if (offsetClassification > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetClassification + i * _potreeData.Metadata.PointSize; + string childName = currentNode.Name + childIndex.ToString(); - byte label = binaryReader.ReadByte(); + PotreeNode child = new() + { + Aabb = ChildAABB(currentNode.Aabb, childIndex), + Name = childName + }; + currentNode.Children[childIndex] = child; + child.Parent = currentNode; - points[i].Classification = label; + nodes.Add(child); + } } + } - else if (offsetScanAngleRank > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetScanAngleRank + i * _potreeData.Metadata.PointSize; - byte scanAngleRank = binaryReader.ReadByte(); + static AABBd ChildAABB(AABBd aabb, int index) + { - points[i].ScanAngleRank = scanAngleRank; + double3 min = aabb.min; + double3 max = aabb.max; + + double3 size = max - min; + + if ((index & 0b0001) > 0) + { + min.z += size.z / 2; } - else if (offsetUserData > -1) + else { - binaryReader.BaseStream.Position = node.ByteOffset + offsetUserData + i * _potreeData.Metadata.PointSize; - byte userData = binaryReader.ReadByte(); + max.z -= size.z / 2; + } - points[i].UserData = userData; + if ((index & 0b0010) > 0) + { + min.y += size.y / 2; } - else if (offsetPointSourceId > -1) + else { - binaryReader.BaseStream.Position = node.ByteOffset + offsetPointSourceId + i * _potreeData.Metadata.PointSize; - byte pointSourceId = binaryReader.ReadByte(); - - points[i].PointSourceId = pointSourceId; + max.y -= size.y / 2; } - if (offsetColor > -1) + if ((index & 0b0100) > 0) { - binaryReader.BaseStream.Position = node.ByteOffset + offsetColor + i * _potreeData.Metadata.PointSize; - - ushort r = binaryReader.ReadUInt16(); - ushort g = binaryReader.ReadUInt16(); - ushort b = binaryReader.ReadUInt16(); - - float3 color = float3.Zero; - - color.r = ((byte)(r > 255 ? r / 256 : r)); - color.g = ((byte)(g > 255 ? g / 256 : g)); - color.b = ((byte)(b > 255 ? b / 256 : b)); - - points[i].Color = color; + min.x += size.x / 2; + } + else + { + max.x -= size.x / 2; } - } - - binaryReader.Close(); - binaryReader.Dispose(); - return points; + return new AABBd(min, max); + } } - private static void MapChildNodesRecursive(IPointCloudOctant octreeNode, PotreeNode potreeNode) + private void FlipYZAxis(PotreeMetadata potreeMetadata, PotreeHierarchy potreeHierarchy) { - octreeNode.NumberOfPointsInNode = (int)potreeNode.NumPoints; - - for (int i = 0; i < potreeNode.Children.Length; i++) + for (int i = 0; i < potreeHierarchy.Nodes.Count; i++) { - if (potreeNode.Children[i] != null) - { - var potreeChild = potreeNode.Children[i]; - - var octant = new PointCloudOctant(potreeNode.Children[i].Aabb.Center, potreeNode.Children[i].Aabb.Size.y, new OctantId(potreeChild.Name)); + var node = potreeHierarchy.Nodes[i]; + node.Aabb = new AABBd(Potree2Consts.YZflip * (node.Aabb.min - potreeMetadata.Offset), Potree2Consts.YZflip * (node.Aabb.max - potreeMetadata.Offset)); + } + potreeMetadata.OffsetList = new List(3) { potreeMetadata.Offset.x, potreeMetadata.Offset.z, potreeMetadata.Offset.y }; + potreeMetadata.ScaleList = new List(3) { potreeMetadata.Scale.x, potreeMetadata.Scale.z, potreeMetadata.Scale.y }; + } - if (potreeChild.NodeType == NodeType.LEAF) - { - octant.IsLeaf = true; - } + private static void CalculateAttributeOffsets(ref PotreeMetadata potreeMetadata) + { + var attributeOffset = 0; - MapChildNodesRecursive(octant, potreeChild); + for (int i = 0; i < potreeMetadata.AttributesList.Count; i++) + { + potreeMetadata.AttributesList[i].AttributeOffset = attributeOffset; - octreeNode.Children[i] = octant; - } + attributeOffset += potreeMetadata.AttributesList[i].Size; } } + + private static Dictionary GetAttributesDict(List attributes) + { + return attributes.ToDictionary(x => x.Name, x => x); + } + + #endregion LoadHierarchy } } \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/Potree2RwBase.cs b/src/PointCloud/Potree/V2/Potree2RwBase.cs deleted file mode 100644 index bd15c01d6..000000000 --- a/src/PointCloud/Potree/V2/Potree2RwBase.cs +++ /dev/null @@ -1,107 +0,0 @@ -using Fusee.PointCloud.Common; -using Fusee.PointCloud.Common.Accessors; -using Fusee.PointCloud.Core.Accessors; -using Fusee.PointCloud.Potree.V2.Data; -using System.IO; - -namespace Fusee.PointCloud.Potree.V2 -{ - public abstract class Potree2RwBase - { - protected PotreeData _potreeData; - - protected bool cachedMetadata = false; - - protected int offsetPosition = -1; - protected int offsetIntensity = -1; - protected int offsetReturnNumber = -1; - protected int offsetNumberOfReturns = -1; - protected int offsetClassification = -1; - protected int offsetScanAngleRank = -1; - protected int offsetUserData = -1; - protected int offsetPointSourceId = -1; - protected int offsetColor = -1; - - protected string OctreeFilePath => Path.Combine(_potreeData.Metadata.FolderPath, Potree2Consts.OctreeFileName); - - public Potree2RwBase(ref PotreeData potreeData) - { - _potreeData = potreeData; - PointAccessor = new PosD3ColF3LblBAccessor(); - - CacheMetadata(); - } - - /// - /// Returns the point type. - /// - public PointType PointType => PointType.PosD3ColF3LblB; - - /// - /// A PointAccessor allows access to the point information (position, color, ect.) without casting it to a specific . - /// - public IPointAccessor PointAccessor { get; protected set; } - - protected void CacheMetadata() - { - if (!cachedMetadata) - { - if (_potreeData.Metadata.Attributes.ContainsKey("position")) - { - offsetPosition = _potreeData.Metadata.Attributes["position"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("intensity")) - { - offsetIntensity = _potreeData.Metadata.Attributes["intensity"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("return number")) - { - offsetReturnNumber = _potreeData.Metadata.Attributes["return number"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("number of returns")) - { - offsetNumberOfReturns = _potreeData.Metadata.Attributes["number of returns"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("classification")) - { - offsetClassification = _potreeData.Metadata.Attributes["classification"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("scan angle rank")) - { - offsetScanAngleRank = _potreeData.Metadata.Attributes["scan angle rank"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("user data")) - { - offsetUserData = _potreeData.Metadata.Attributes["user data"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("point source id")) - { - offsetPointSourceId = _potreeData.Metadata.Attributes["point source id"].AttributeOffset; - } - if (_potreeData.Metadata.Attributes.ContainsKey("rgb")) - { - offsetColor = _potreeData.Metadata.Attributes["rgb"].AttributeOffset; - } - - int pointSize = 0; - - if (_potreeData.Metadata != null) - { - foreach (var metaAttributeItem in _potreeData.Metadata.AttributesList) - { - pointSize += metaAttributeItem.Size; - } - - _potreeData.Metadata.PointSize = pointSize; - } - - cachedMetadata = true; - } - } - - public static PotreeNode FindNode(ref PotreeHierarchy potreeHierarchy, OctantId id) - { - return potreeHierarchy.Nodes.Find(n => n.Name == OctantId.OctantIdToPotreeName(id)); - } - } -} \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/Potree2Writer.cs b/src/PointCloud/Potree/V2/Potree2Writer.cs index 2e623d918..6328d9e40 100644 --- a/src/PointCloud/Potree/V2/Potree2Writer.cs +++ b/src/PointCloud/Potree/V2/Potree2Writer.cs @@ -1,351 +1,81 @@ -using Fusee.Math.Core; +using CommunityToolkit.Diagnostics; +using CommunityToolkit.HighPerformance.Buffers; using Fusee.PointCloud.Common; +using Fusee.PointCloud.Core; using Fusee.PointCloud.Potree.V2.Data; using System; -using System.IO; namespace Fusee.PointCloud.Potree.V2 { - public class Potree2Writer : Potree2RwBase + /// + /// Delegate for a method that knows how to parse a enxtra byte uint back to its byte representation. + /// + /// + /// + public delegate Span HandleWriteExtraBytes(uint flag); + + /// + /// Writes Potree data + /// + public class Potree2Writer : Potree2AccessBase { - public Potree2Writer(ref PotreeData potreeData) : base(ref potreeData) { } - /// - /// Directly writes the action of one given set of selectors to disk. + /// Pass method how to handle the extra bytes, resulting uint will be passed into . /// - /// - /// - /// - /// - /// - public (long octants, long points) Write(Predicate nodeSelector, Predicate pointSelector, Action action, bool dryrun = false) - { - long octantCount = 0; - long pointsCount = 0; - - using (Stream readStream = File.Open(OctreeFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (Stream writeStream = File.Open(OctreeFilePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) - { - BinaryReader binaryReader = new BinaryReader(readStream); - BinaryWriter binaryWriter = new BinaryWriter(writeStream); - - foreach (var node in _potreeData.Hierarchy.Nodes) - { - if (nodeSelector(node)) - { - octantCount++; - - var point = new PotreePoint(); - - for (int i = 0; i < node.NumPoints; i++) - { - if (offsetPosition > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetPosition + i * _potreeData.Metadata.PointSize; - - double x = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.x; - double y = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.y; - double z = binaryReader.ReadInt32() * _potreeData.Metadata.Scale.z; - - double3 position = new(x, y, z); - position = Potree2Consts.YZflip * position; - - point.Position = position; - } - - if (offsetIntensity > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetIntensity + i * _potreeData.Metadata.PointSize; - point.Intensity = binaryReader.ReadInt16(); - } - - if (offsetReturnNumber > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetReturnNumber + i * _potreeData.Metadata.PointSize; - point.ReturnNumber = binaryReader.ReadByte(); - } - - if (offsetNumberOfReturns > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetNumberOfReturns + i * _potreeData.Metadata.PointSize; - point.NumberOfReturns = binaryReader.ReadByte(); - } - - if (offsetClassification > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetClassification + i * _potreeData.Metadata.PointSize; - point.Classification = binaryReader.ReadByte(); - } - - if (offsetScanAngleRank > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetScanAngleRank + i * _potreeData.Metadata.PointSize; - point.ScanAngleRank = binaryReader.ReadByte(); - } - - if (offsetUserData > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetUserData + i * _potreeData.Metadata.PointSize; - point.UserData = binaryReader.ReadByte(); - } - - if (offsetPointSourceId > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetPointSourceId + i * _potreeData.Metadata.PointSize; - point.PointSourceId = binaryReader.ReadByte(); - } - - if (offsetColor > -1) - { - binaryReader.BaseStream.Position = node.ByteOffset + offsetColor + i * _potreeData.Metadata.PointSize; - - ushort r = binaryReader.ReadUInt16(); - ushort g = binaryReader.ReadUInt16(); - ushort b = binaryReader.ReadUInt16(); - - float3 color = float3.Zero; - - color.r = ((byte)(r > 255 ? r / 256 : r)); - color.g = ((byte)(g > 255 ? g / 256 : g)); - color.b = ((byte)(b > 255 ? b / 256 : b)); - - point.Color = color; - } - - if (pointSelector(point)) - { - action(point); - - if (!dryrun) - { - if (offsetPosition > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetPosition + i * _potreeData.Metadata.PointSize; - - var position = Potree2Consts.YZflip * point.Position; - - int x = Convert.ToInt32(position.x / _potreeData.Metadata.Scale.x); - int y = Convert.ToInt32(position.y / _potreeData.Metadata.Scale.y); - int z = Convert.ToInt32(position.z / _potreeData.Metadata.Scale.z); - - binaryWriter.Write(x); - binaryWriter.Write(y); - binaryWriter.Write(z); - } - - if (offsetIntensity > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetIntensity + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.Intensity); - } - - if (offsetReturnNumber > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetReturnNumber + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.ReturnNumber); - } - - if (offsetNumberOfReturns > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetNumberOfReturns + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.NumberOfReturns); - } - - if (offsetClassification > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetClassification + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.Classification); - } - - if (offsetScanAngleRank > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetScanAngleRank + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.ScanAngleRank); - } - - if (offsetUserData > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetUserData + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.UserData); - } - - if (offsetPointSourceId > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetPointSourceId + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.PointSourceId); - } - - if (offsetColor > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetColor + i * _potreeData.Metadata.PointSize; + public HandleWriteExtraBytes? HandleWriteExtraBytes { get; set; } - ushort r = Convert.ToUInt16(point.Color.r * 256); - ushort g = Convert.ToUInt16(point.Color.g * 256); - ushort b = Convert.ToUInt16(point.Color.b * 256); - - binaryWriter.Write(r); - binaryWriter.Write(g); - binaryWriter.Write(b); - } - } - - pointsCount++; - } - } - } - } + /// + /// Generate a instance. + /// + public Potree2Writer(PotreeData potreeData) : base(potreeData) { } - binaryWriter.Close(); - binaryReader.Dispose(); + /// + /// Writes for a node to disk. + /// + /// + /// + /// + public void WriteVisualizationPoint(OctantId octantId, MemoryOwner visualizationPoints, PotreeSettingsAttribute potreeSettingsAttribute) + { + Guard.IsNotNull(PotreeData); + var node = PotreeData.GetNode(octantId); - binaryReader.Close(); - binaryReader.Dispose(); - } + // if node is null the hierarchy is broken and we look for an octant that isn't there... + Guard.IsNotNull(node); - return (octantCount, pointsCount); + WriteVisualizationPoint(node, visualizationPoints, potreeSettingsAttribute); } - public (long octants, long points) Label(Predicate nodeSelector, Predicate pointSelector, byte Label, bool dryrun = false) + private void WriteVisualizationPoint(PotreeNode potreeNode, MemoryOwner visualizationPoints, PotreeSettingsAttribute potreeSettingsAttribute) { - long octantCount = 0; - long pointsCount = 0; + Guard.IsLessThanOrEqualTo(potreeNode.NumPoints, int.MaxValue); + Guard.IsNotNull(PotreeData); + Guard.IsNotNull(HandleWriteExtraBytes); - using (Stream readStream = File.Open(OctreeFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (Stream writeStream = File.Open(OctreeFilePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) - { - BinaryReader binaryReader = new BinaryReader(readStream); - BinaryWriter binaryWriter = new BinaryWriter(writeStream); + var pointArray = ReadRawNodeData(potreeNode); - double3 point = double3.Zero; + var visualizationArray = visualizationPoints.Span; + var visualizationIdx = 0; - foreach (var node in _potreeData.Hierarchy.Nodes) - { - if (nodeSelector(node)) - { - octantCount++; - - for (int i = 0; i < node.NumPoints; i++) - { - binaryReader.BaseStream.Position = node.ByteOffset + 0 + i * _potreeData.Metadata.PointSize; - - point.x = (binaryReader.ReadInt32() * _potreeData.Metadata.Scale.x); - point.z = (binaryReader.ReadInt32() * _potreeData.Metadata.Scale.y); - point.y = (binaryReader.ReadInt32() * _potreeData.Metadata.Scale.z); - - if (pointSelector(point)) - { - if (!dryrun) - { - binaryWriter.BaseStream.Position = node.ByteOffset + 16 + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(Label); - } - - pointsCount++; - } - } - } - } + for (int i = 0; i < pointArray.Length; i += PotreeData.Metadata.PointSize) + { + var attributeSlice = new Span(pointArray).Slice(i + potreeSettingsAttribute.AttributeOffset, potreeSettingsAttribute.Size); - binaryWriter.Close(); - binaryReader.Dispose(); + HandleWriteExtraBytes(visualizationArray[visualizationIdx].Flags).CopyTo(attributeSlice); - binaryReader.Close(); - binaryReader.Dispose(); + visualizationIdx++; } - return (octantCount, pointsCount); + WriteRawNodeData(potreeNode, pointArray); } - public void WriteRawPoints(OctantId oid, TPotreePoint[] points) where TPotreePoint : PotreePoint + private void WriteRawNodeData(PotreeNode potreeNode, byte[] rawNodeData) { - var node = FindNode(ref _potreeData.Hierarchy, oid); - - if (points.Length != node.NumPoints) - { - //TODO: (throw) correct error - throw new Exception(); - } + Guard.IsNotNull(PotreeData); + Guard.IsNotNull(rawNodeData); - using (Stream writeStream = File.Open(OctreeFilePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) - { - BinaryWriter binaryWriter = new BinaryWriter(writeStream); - - for (int i = 0; i < points.Length; i++) - { - var point = points[i]; - - if (offsetPosition > -1) - { - // TODO: Fix position conversion - //binaryWriter.BaseStream.Position = node.ByteOffset + offsetPosition + i * _potreeData.Metadata.PointSize; - - //var position = Potree2Consts.YZflip * point.Position; - - //binaryWriter.Write(position.x); - //binaryWriter.Write(position.y); - //binaryWriter.Write(position.z); - } - - if (offsetIntensity > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetIntensity + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.Intensity); - } - - if (offsetReturnNumber > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetReturnNumber + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.ReturnNumber); - } - - if (offsetNumberOfReturns > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetNumberOfReturns + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.NumberOfReturns); - } - - if (offsetClassification > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetClassification + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.Classification); - } - - if (offsetScanAngleRank > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetScanAngleRank + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.ScanAngleRank); - } - - if (offsetUserData > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetUserData + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.UserData); - } - - if (offsetPointSourceId > -1) - { - binaryWriter.BaseStream.Position = node.ByteOffset + offsetPointSourceId + i * _potreeData.Metadata.PointSize; - binaryWriter.Write(point.PointSourceId); - } - - if (offsetColor > -1) - { - // TODO: Fix color conversion - //binaryWriter.BaseStream.Position = node.ByteOffset + offsetColor + i * _potreeData.Metadata.PointSize; - - //ushort r = (ushort)MathF.Floor(point.Color.r >= 1f ? 255 : point.Color.r * 256f); - //ushort g = (ushort)MathF.Floor(point.Color.g >= 1f ? 255 : point.Color.g * 256f); - //ushort b = (ushort)MathF.Floor(point.Color.b >= 1f ? 255 : point.Color.b * 256f); - - //binaryWriter.Write(r); - //binaryWriter.Write(g); - //binaryWriter.Write(b); - } - } - - binaryWriter.Close(); - binaryWriter.Dispose(); - } + var potreePointSize = (int)potreeNode.NumPoints * PotreeData.Metadata.PointSize; + PotreeData.WriteViewAccessor.WriteArray(potreeNode.ByteOffset, rawNodeData, 0, potreePointSize); } } } \ No newline at end of file diff --git a/src/PointCloud/Potree/V2/PotreeData.cs b/src/PointCloud/Potree/V2/PotreeData.cs deleted file mode 100644 index 16b3aa9f6..000000000 --- a/src/PointCloud/Potree/V2/PotreeData.cs +++ /dev/null @@ -1,209 +0,0 @@ -using CommunityToolkit.Diagnostics; -using Fusee.Math.Core; -using Fusee.PointCloud.Potree.V2.Data; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace Fusee.PointCloud.Potree.V2 -{ - /// - /// Contains information about the Potree file's meta data and hierarchy/octree. - /// - public class PotreeData - { - /// - /// The hierarchy as linear list of s. - /// - public PotreeHierarchy Hierarchy; - - /// - /// The meta data of the file. - /// - public PotreeMetadata Metadata; - - public PotreeData(string folderPath) - { - Guard.IsNotNullOrWhiteSpace(folderPath, nameof(folderPath)); - - LoadHierarchy(folderPath); - } - - private void LoadHierarchy(string folderPath) - { - var metadataFilePath = Path.Combine(folderPath, Potree2Consts.MetadataFileName); - var hierarchyFilePath = Path.Combine(folderPath, Potree2Consts.HierarchyFileName); - - Guard.IsTrue(File.Exists(metadataFilePath), metadataFilePath); - Guard.IsTrue(File.Exists(metadataFilePath), hierarchyFilePath); - - Metadata = LoadPotreeMetadata(metadataFilePath); - Hierarchy = new() - { - Root = new() - { - Name = "r", - } - }; - - Metadata.Attributes = GetAttributesDict(Metadata.AttributesList); - - Metadata.FolderPath = folderPath; - - CalculateAttributeOffsets(ref Metadata); - - Hierarchy.Root.Aabb = new AABBd(Metadata.BoundingBox.Min, Metadata.BoundingBox.Max); - - var data = File.ReadAllBytes(hierarchyFilePath); - - Guard.IsNotNull(data, nameof(data)); - - LoadHierarchyRecursive(ref Hierarchy.Root, ref data, 0, Metadata.Hierarchy.FirstChunkSize); - - Hierarchy.Nodes = new(); - Hierarchy.Root.Traverse(n => Hierarchy.Nodes.Add(n)); - - FlipYZAxis(); - - Metadata.BoundingBox.MinList = new List(3) { Hierarchy.Root.Aabb.min.x, Hierarchy.Root.Aabb.min.y, Hierarchy.Root.Aabb.min.z }; - Metadata.BoundingBox.MaxList = new List(3) { Hierarchy.Root.Aabb.max.x, Hierarchy.Root.Aabb.max.y, Hierarchy.Root.Aabb.max.z }; - } - - private static PotreeMetadata LoadPotreeMetadata(string metadataFilepath) - { - var potreeData = JsonConvert.DeserializeObject(File.ReadAllText(metadataFilepath)); - - Guard.IsNotNull(potreeData, nameof(potreeData)); - - return potreeData; - } - - private static void LoadHierarchyRecursive(ref PotreeNode root, ref byte[] data, long offset, long size) - { - int bytesPerNode = 22; - int numNodes = (int)(size / bytesPerNode); - - var nodes = new List(numNodes) - { - root - }; - - for (int i = 0; i < numNodes; i++) - { - var currentNode = nodes[i]; - if (currentNode == null) - currentNode = new PotreeNode(); - - ulong offsetNode = (ulong)offset + (ulong)(i * bytesPerNode); - - var nodeType = data[offsetNode + 0]; - int childMask = BitConverter.ToInt32(data, (int)offsetNode + 1); - var numPoints = BitConverter.ToUInt32(data, (int)offsetNode + 2); - var byteOffset = BitConverter.ToInt64(data, (int)offsetNode + 6); - var byteSize = BitConverter.ToInt64(data, (int)offsetNode + 14); - - currentNode.NodeType = (NodeType)nodeType; - currentNode.NumPoints = numPoints; - currentNode.ByteOffset = byteOffset; - currentNode.ByteSize = byteSize; - - if (currentNode.NodeType == NodeType.PROXY) - { - LoadHierarchyRecursive(ref currentNode, ref data, byteOffset, byteSize); - } - else - { - for (int childIndex = 0; childIndex < 8; childIndex++) - { - bool childExists = (1 << childIndex & childMask) != 0; - - if (!childExists) - { - continue; - } - - string childName = currentNode.Name + childIndex.ToString(); - - PotreeNode child = new() - { - Aabb = ChildAABB(currentNode.Aabb, childIndex), - Name = childName - }; - currentNode.Children[childIndex] = child; - child.Parent = currentNode; - - nodes.Add(child); - } - } - } - - static AABBd ChildAABB(AABBd aabb, int index) - { - - double3 min = aabb.min; - double3 max = aabb.max; - - double3 size = max - min; - - if ((index & 0b0001) > 0) - { - min.z += size.z / 2; - } - else - { - max.z -= size.z / 2; - } - - if ((index & 0b0010) > 0) - { - min.y += size.y / 2; - } - else - { - max.y -= size.y / 2; - } - - if ((index & 0b0100) > 0) - { - min.x += size.x / 2; - } - else - { - max.x -= size.x / 2; - } - - return new AABBd(min, max); - } - } - - private void FlipYZAxis() - { - for (int i = 0; i < Hierarchy.Nodes.Count; i++) - { - var node = Hierarchy.Nodes[i]; - node.Aabb = new AABBd(Potree2Consts.YZflip * (node.Aabb.min - Metadata.Offset), Potree2Consts.YZflip * (node.Aabb.max - Metadata.Offset)); - } - Metadata.OffsetList = new List(3) { Metadata.Offset.x, Metadata.Offset.z, Metadata.Offset.y }; - Metadata.ScaleList = new List(3) { Metadata.Scale.x, Metadata.Scale.z, Metadata.Scale.y }; - } - - private static void CalculateAttributeOffsets(ref PotreeMetadata potreeMetadata) - { - var attributeOffset = 0; - - for (int i = 0; i < potreeMetadata.AttributesList.Count; i++) - { - potreeMetadata.AttributesList[i].AttributeOffset = attributeOffset; - - attributeOffset += potreeMetadata.AttributesList[i].Size; - } - } - - private static Dictionary GetAttributesDict(List attributes) - { - return attributes.ToDictionary(x => x.Name, x => x); - } - } -} \ No newline at end of file diff --git a/src/Serialization/FusFile.cs b/src/Serialization/FusFile.cs index 6b2ddbe52..d69022edf 100644 --- a/src/Serialization/FusFile.cs +++ b/src/Serialization/FusFile.cs @@ -49,7 +49,7 @@ public struct FusHeader /// /// Polymorphic contents of a fus file. The actual contents is contained in a sub-class. Possible sub-classes are listed /// in decorators preceding this class declaration. This mechanism allows fus files - /// to contain different types of contents as well as different versions of contents. + /// to contain different types of contents as well as different versions of contents. /// [ProtoInclude(201, typeof(V1.FusScene))] [ProtoContract] @@ -71,7 +71,7 @@ public class FusFile public FusHeader Header; /// - /// The file contents. Check and cast to the concrete type to access it, e. g. using a C# 7.0 + /// The file contents. Check and cast to the concrete type to access it, e. g. using a C# 7.0 /// pattern matching in switch expression. /// [ProtoMember(2)] diff --git a/src/Serialization/Fusee.Serialization.csproj b/src/Serialization/Fusee.Serialization.csproj index 8c44d05f9..9d5784b30 100644 --- a/src/Serialization/Fusee.Serialization.csproj +++ b/src/Serialization/Fusee.Serialization.csproj @@ -1,4 +1,4 @@ - + netstandard2.1;net7.0 @@ -19,7 +19,7 @@ analyzers - + \ No newline at end of file diff --git a/src/Serialization/V1/FusMesh.cs b/src/Serialization/V1/FusMesh.cs index 501bf036b..10bc0f1f3 100644 --- a/src/Serialization/V1/FusMesh.cs +++ b/src/Serialization/V1/FusMesh.cs @@ -71,7 +71,7 @@ public class FusMesh : FusComponent /// The triangles. /// [ProtoMember(7)] - public uint[] Triangles; + public ushort[] Triangles; /// /// The bounding box of this geometry chunk. diff --git a/src/Serialization/V2/FusMesh.cs b/src/Serialization/V2/FusMesh.cs new file mode 100644 index 000000000..171acbcae --- /dev/null +++ b/src/Serialization/V2/FusMesh.cs @@ -0,0 +1,104 @@ +using Fusee.Math.Core; +using Fusee.Serialization.V1; +using ProtoBuf; + +namespace Fusee.Serialization.V2 +{ + /// + /// Contains 3D geometry information (Vertices, Triangles, Normals, UVs, ...). + /// + [ProtoContract] + public class FusMesh : FusComponent + { + #region Payload + /// + /// Gets and sets the vertices. + /// + /// + /// The vertices. + /// + [ProtoMember(1)] + public float3[] Vertices; + + /// + /// Gets and sets the color of a single vertex. + /// + /// + /// The color. + /// + [ProtoMember(2)] + public uint[] Colors; + + /// + /// Gets and sets the normals. + /// + /// + /// The normals.. + /// + [ProtoMember(3)] + public float3[] Normals; + + /// + /// Gets and sets the UV-coordinates. + /// + /// + /// The UV-coordinates. + /// + [ProtoMember(4)] + public float2[] UVs; + + /// + /// Gets and sets the boneweights. + /// + /// + /// The boneweights. + /// + [ProtoMember(5)] + public float4[] BoneWeights; + + /// + /// Gets and sets the boneindices. + /// + /// + /// The boneindices. + /// + [ProtoMember(6)] + public float4[] BoneIndices; + + /// + /// Gets and sets the triangles. + /// + /// + /// The triangles. + /// + [ProtoMember(7)] + public uint[] Triangles; + + /// + /// The bounding box of this geometry chunk. + /// + [ProtoMember(8)] + public AABBf BoundingBox; + + /// + /// The tangent of each triangle for normal mapping. + /// w-component is handedness + /// + [ProtoMember(9)] + public float4[] Tangents; + + /// + /// The bitangent of each triangle for normal mapping. + /// + [ProtoMember(10)] + public float3[] BiTangents; + + /// + /// The type of the mesh ??? + /// + [ProtoMember(11)] + public int MeshType = 0; + #endregion + } + +} \ No newline at end of file diff --git a/src/Serialization/V2/FusPickComponent.cs b/src/Serialization/V2/FusPickComponent.cs new file mode 100644 index 000000000..ba220bd65 --- /dev/null +++ b/src/Serialization/V2/FusPickComponent.cs @@ -0,0 +1,17 @@ +using ProtoBuf; + +namespace Fusee.Serialization.V2 +{ + /// + /// If is , the picking is being skiped for this and all successors. + /// + [ProtoContract] + public class FusPickComponent : V1.FusComponent + { + /// + /// If there is more than one PickComponent in one scene, the rendered output of the picker with a higher layer will be picked first above the output of a pick result with a lower layer. + /// + [ProtoMember(1)] + public int PickLayer; + } +} \ No newline at end of file diff --git a/src/Tests/AssetStorage/Desktop/Fusee.Tests.AssetStorage.Desktop.csproj b/src/Tests/AssetStorage/Desktop/Fusee.Tests.AssetStorage.Desktop.csproj index 54f3ed031..f80f3d055 100644 --- a/src/Tests/AssetStorage/Desktop/Fusee.Tests.AssetStorage.Desktop.csproj +++ b/src/Tests/AssetStorage/Desktop/Fusee.Tests.AssetStorage.Desktop.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\AssetStorage\Desktop\ @@ -24,9 +24,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Tests/Math/Core/Fusee.Tests.Math.Core.csproj b/src/Tests/Math/Core/Fusee.Tests.Math.Core.csproj index 9e7a11d22..e8f33545c 100644 --- a/src/Tests/Math/Core/Fusee.Tests.Math.Core.csproj +++ b/src/Tests/Math/Core/Fusee.Tests.Math.Core.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\Math @@ -6,9 +6,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Tests/Render/Desktop/Fusee.Tests.Render.Desktop.csproj b/src/Tests/Render/Desktop/Fusee.Tests.Render.Desktop.csproj index 509e269cd..755d0e5e9 100644 --- a/src/Tests/Render/Desktop/Fusee.Tests.Render.Desktop.csproj +++ b/src/Tests/Render/Desktop/Fusee.Tests.Render.Desktop.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\Render\Desktop\ @@ -26,10 +26,13 @@ - + - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Tests/Render/Desktop/Program.cs b/src/Tests/Render/Desktop/Program.cs index 711dc2c88..df19907d6 100644 --- a/src/Tests/Render/Desktop/Program.cs +++ b/src/Tests/Render/Desktop/Program.cs @@ -31,7 +31,7 @@ public static void Init(string arg) IO.IOImp = new Fusee.Base.Imp.Desktop.IOImp(); AssetStorage.Instance.Dispose(); - var baseDirOfExample = new Uri(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().GetName().CodeBase)); + var baseDirOfExample = new Uri(Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory)); FilePath = baseDirOfExample.LocalPath; var fap = new Fusee.Base.Imp.Desktop.FileAssetProvider(Path.Combine(baseDirOfExample.LocalPath, "Assets")); diff --git a/src/Tests/Render/Desktop/References/AdvancedUI.png b/src/Tests/Render/Desktop/References/AdvancedUI.png index bc1b67c02..fcffcc601 100644 Binary files a/src/Tests/Render/Desktop/References/AdvancedUI.png and b/src/Tests/Render/Desktop/References/AdvancedUI.png differ diff --git a/src/Tests/Render/Desktop/References/Camera.png b/src/Tests/Render/Desktop/References/Camera.png index 8d34d7ff3..dc8d08020 100644 Binary files a/src/Tests/Render/Desktop/References/Camera.png and b/src/Tests/Render/Desktop/References/Camera.png differ diff --git a/src/Tests/Render/Desktop/References/Deferred.png b/src/Tests/Render/Desktop/References/Deferred.png index 0f2c8e5b8..bc109e4f6 100644 Binary files a/src/Tests/Render/Desktop/References/Deferred.png and b/src/Tests/Render/Desktop/References/Deferred.png differ diff --git a/src/Tests/Render/Desktop/References/Fractal.png b/src/Tests/Render/Desktop/References/Fractal.png index 9268213d0..96fc83288 100644 Binary files a/src/Tests/Render/Desktop/References/Fractal.png and b/src/Tests/Render/Desktop/References/Fractal.png differ diff --git a/src/Tests/Render/Desktop/References/Labyrinth.png b/src/Tests/Render/Desktop/References/Labyrinth.png index 48b566404..3c627c7c4 100644 Binary files a/src/Tests/Render/Desktop/References/Labyrinth.png and b/src/Tests/Render/Desktop/References/Labyrinth.png differ diff --git a/src/Tests/Render/Desktop/References/Materials.png b/src/Tests/Render/Desktop/References/Materials.png index 25b166a13..e83107a81 100644 Binary files a/src/Tests/Render/Desktop/References/Materials.png and b/src/Tests/Render/Desktop/References/Materials.png differ diff --git a/src/Tests/Render/Desktop/References/Picking.png b/src/Tests/Render/Desktop/References/Picking.png index 9b1a76712..d15d977dd 100644 Binary files a/src/Tests/Render/Desktop/References/Picking.png and b/src/Tests/Render/Desktop/References/Picking.png differ diff --git a/src/Tests/Render/Desktop/References/PointCloudPotree2.png b/src/Tests/Render/Desktop/References/PointCloudPotree2.png index cc9ca8863..12cbd12dc 100644 Binary files a/src/Tests/Render/Desktop/References/PointCloudPotree2.png and b/src/Tests/Render/Desktop/References/PointCloudPotree2.png differ diff --git a/src/Tests/Render/Desktop/References/Simple.png b/src/Tests/Render/Desktop/References/Simple.png index 099d90202..aac80f449 100644 Binary files a/src/Tests/Render/Desktop/References/Simple.png and b/src/Tests/Render/Desktop/References/Simple.png differ diff --git a/src/Tests/Render/Desktop/References/ThreeDFont.png b/src/Tests/Render/Desktop/References/ThreeDFont.png index 32b874ad5..bcb975e7e 100644 Binary files a/src/Tests/Render/Desktop/References/ThreeDFont.png and b/src/Tests/Render/Desktop/References/ThreeDFont.png differ diff --git a/src/Tests/Scene/Components/Fusee.Tests.Scene.Components.csproj b/src/Tests/Scene/Components/Fusee.Tests.Scene.Components.csproj index f7dda8ecf..7ef53eaa4 100644 --- a/src/Tests/Scene/Components/Fusee.Tests.Scene.Components.csproj +++ b/src/Tests/Scene/Components/Fusee.Tests.Scene.Components.csproj @@ -1,4 +1,4 @@ - + net7.0 @@ -8,9 +8,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Tests/Serialization/V1/Fusee.Tests.Serialization.V1.csproj b/src/Tests/Serialization/V1/Fusee.Tests.Serialization.V1.csproj index 04c06af7f..89740ffa9 100644 --- a/src/Tests/Serialization/V1/Fusee.Tests.Serialization.V1.csproj +++ b/src/Tests/Serialization/V1/Fusee.Tests.Serialization.V1.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\Serialization\V1\ @@ -8,12 +8,12 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/Tests/Serialization/V1/SimpleConvertSceneGraph.cs b/src/Tests/Serialization/V1/SimpleConvertSceneGraph.cs index 00b69d516..c56f6dad3 100644 --- a/src/Tests/Serialization/V1/SimpleConvertSceneGraph.cs +++ b/src/Tests/Serialization/V1/SimpleConvertSceneGraph.cs @@ -127,15 +127,15 @@ public void V1_SimpleScene_Convert() } } - if (gtComp is Mesh mesh) + if (gtComp is FusMesh mesh) { Assert.Equal(mesh.Name, ((FusMesh)fusFileComp).Name); // Assert.Equal(mesh.BoundingBox, ((FusMesh)fusFileComp).BoundingBox); <- not yet calculated, is done after first frame - Assert.Equal(mesh.Colors0?.ToArray(), ((FusMesh)fusFileComp).Colors); + Assert.Equal(mesh.Colors?.ToArray(), ((FusMesh)fusFileComp).Colors); Assert.Equal(mesh.Vertices?.ToArray(), ((FusMesh)fusFileComp).Vertices); - Assert.Equal(mesh.Triangles?.ToArray(), ((FusMesh)fusFileComp).Triangles); + Assert.Equal(mesh.Triangles?.ToArray().Select(x => x).ToArray(), ((FusMesh)fusFileComp).Triangles); Assert.Equal(mesh.UVs?.ToArray(), ((FusMesh)fusFileComp).UVs); - Assert.Equal((int)mesh.MeshType, ((FusMesh)fusFileComp).MeshType); + Assert.Equal(mesh.MeshType, ((FusMesh)fusFileComp).MeshType); Assert.Equal(mesh.Tangents?.ToArray(), ((FusMesh)fusFileComp).Tangents); Assert.Equal(mesh.BiTangents?.ToArray(), ((FusMesh)fusFileComp).BiTangents); } @@ -216,7 +216,7 @@ public void V1_SimpleScene_Convert() //Assert.Equal(t.Name, ((Transform)sceneFileComp).Name); //Assert.Equal(t.Rotation, ((Transform)sceneFileComp).Rotation); //Assert.Equal(t.Scale, ((Transform)sceneFileComp).Scale); - //Assert.Equal(t.Translation, ((Transform)sceneFileComp).Translation); + //Assert.Equal(t.Translation, ((Transform)sceneFileComp).Translation); //} //if (gtComp is Bone bone) diff --git a/src/Tests/Serialization/V1/SimpleSerialization.cs b/src/Tests/Serialization/V1/SimpleSerialization.cs index 4f21bdebb..1a4bca4da 100644 --- a/src/Tests/Serialization/V1/SimpleSerialization.cs +++ b/src/Tests/Serialization/V1/SimpleSerialization.cs @@ -105,7 +105,7 @@ public static FusMesh CreateCuboid(float3 size) new float3 {x = -0.5f * size.x, y = -0.5f * size.y, z = -0.5f * size.z} }, - Triangles = new uint[] + Triangles = new ushort[] { // front face 0, 2, 1, 0, 3, 2, diff --git a/src/Tests/Xene/Fusee.Tests.Xene.csproj b/src/Tests/Xene/Fusee.Tests.Xene.csproj index e1ce294d6..de80f550c 100644 --- a/src/Tests/Xene/Fusee.Tests.Xene.csproj +++ b/src/Tests/Xene/Fusee.Tests.Xene.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\Xene\ @@ -6,9 +6,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Tests/Xene/SimpleXene.cs b/src/Tests/Xene/SimpleXene.cs index d0bbd02de..fceac8f65 100644 --- a/src/Tests/Xene/SimpleXene.cs +++ b/src/Tests/Xene/SimpleXene.cs @@ -197,12 +197,36 @@ public void FindNodesWhereComponentByTypeSingleRootExtension() ); } + + [Fact] + public void CallCurrentGetterTwiceOnIEnumeratorReturnedByViserator() + { + TestNode tree = CreateSimpleTestTree(); + + IEnumerable resultSet = tree.Viserate(); + + IEnumerator iterator = resultSet.GetEnumerator(); + + Assert.True(iterator.MoveNext()); + + // Call the getter of Current twice! + TestResult trCurrent1 = iterator.Current; + TestResult trCurrent2 = iterator.Current; + + Assert.Equal(trCurrent1, trCurrent2); + + // Besides the fact that both calls to Current.Get yield the same object, the main fact that is proven here + // is that calling Current.Get tiwce without a call to MoveNext in between works on our Viserator "_itemQueue". + } + + + [Fact] public void SingleNodeViserator() { TestNode tree = CreateSimpleTestTree(); - var resultSet = tree.Viserate(); + IEnumerable resultSet = tree.Viserate(); Assert.Collection(resultSet, resultItem => Assert.Equal(new TestResult { Depth = 1, Path = "/RootNode/[2]" }, resultItem), resultItem => Assert.Equal(new TestResult { Depth = 1, Path = "/RootNode/[6]" }, resultItem), @@ -217,7 +241,7 @@ public void ListOfRootsViserator() var twoRoots = TwoTestTrees(); - var resultSet = twoRoots.Viserate(); + IEnumerable resultSet = twoRoots.Viserate(); Assert.Collection(resultSet, resultItem => Assert.Equal(new TestResult { Depth = 1, Path = "/RootNode/[2]" }, resultItem), diff --git a/src/Tests/Xirkit/Core/Fusee.Tests.Xirkit.Core.csproj b/src/Tests/Xirkit/Core/Fusee.Tests.Xirkit.Core.csproj index f2e6fab8c..236bb3143 100644 --- a/src/Tests/Xirkit/Core/Fusee.Tests.Xirkit.Core.csproj +++ b/src/Tests/Xirkit/Core/Fusee.Tests.Xirkit.Core.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\Xirkit @@ -6,9 +6,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Tests/Xirkit/NestedAccess/Fusee.Tests.Xirkit.NestedAccess.csproj b/src/Tests/Xirkit/NestedAccess/Fusee.Tests.Xirkit.NestedAccess.csproj index fb1e3c024..5b0175f8b 100644 --- a/src/Tests/Xirkit/NestedAccess/Fusee.Tests.Xirkit.NestedAccess.csproj +++ b/src/Tests/Xirkit/NestedAccess/Fusee.Tests.Xirkit.NestedAccess.csproj @@ -1,4 +1,4 @@ - + net7.0 $(BaseOutputPath)\Tests\Xirkit\ @@ -6,9 +6,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Tools/BlenderScripts/addons/io_export_fus/requirements.txt b/src/Tools/BlenderScripts/addons/io_export_fus/requirements.txt index 5d63f1f5b..7c30f52f8 100644 --- a/src/Tools/BlenderScripts/addons/io_export_fus/requirements.txt +++ b/src/Tools/BlenderScripts/addons/io_export_fus/requirements.txt @@ -1,2 +1,2 @@ -protobuf==3.21.9 +protobuf==3.22.0 ptvsd==4.3.2 diff --git a/src/Tools/CmdLine/Fusee.Tools.CmdLine.csproj b/src/Tools/CmdLine/Fusee.Tools.CmdLine.csproj index 912622829..9caab5c54 100644 --- a/src/Tools/CmdLine/Fusee.Tools.CmdLine.csproj +++ b/src/Tools/CmdLine/Fusee.Tools.CmdLine.csproj @@ -50,7 +50,7 @@ - + diff --git a/src/Xene/Viserator.cs b/src/Xene/Viserator.cs index a719baf60..ab34e9278 100644 --- a/src/Xene/Viserator.cs +++ b/src/Xene/Viserator.cs @@ -1,4 +1,5 @@ -using System; +using Fusee.Engine.Common; +using System; using System.Collections; using System.Collections.Generic; @@ -9,8 +10,8 @@ namespace Fusee.Xene /// public static class ViseratorExtensions { - // Unfortunate construct, but there seems no other way. What we really needed here is a MixIn to make - // a INode or SceneContainer implement IEnumerable (afterwards). All C# offers us is to + // Unfortunate construct, but there seems no other way. What we really needed here is a MixIn to make + // a INode or SceneContainer implement IEnumerable (afterwards). All C# offers us is to // define ExtensionMethods returning an IEnumerable<>. // Thus we need some class to implement that: internal class ViseratorEnumerable : IEnumerable @@ -86,7 +87,7 @@ public ViseratorBase() { } - // Step2: initialize the instance + // Step2: initialize the instance /// /// Initializes this instance with the specified tree. /// @@ -105,9 +106,20 @@ protected internal virtual void Init(IEnumerable rootList) /// public bool MoveNext() { - if (_itemQueue.Count > 0) - return true; - return EnumMoveNext(); + switch (_itemQueue.Count) + { + case 0: + // Empty queue - enumeration has just begun. Start traversing + return EnumMoveNext(); + case 1: + // Exactly one item in queue: This is the current "Current". Since we want to move to the next, dequeue it and carry on traversing + _itemQueue.Dequeue(); + return EnumMoveNext(); + default: + // More than one item in the queue. Dequeue the current "Current" and signal that the loop can iterate over the next item. + _itemQueue.Dequeue(); + return true; + } } /// @@ -121,7 +133,8 @@ public void Reset() /// /// Gets the element in the collection at the current position of the enumerator. /// - public TItem Current => _itemQueue.Dequeue(); + // Peek only, don't Dequeue when Current is read. Otherwise it can only be read once (which is OK in foreach loops but NOT e.g. in other uses of IEnumerator). + public TItem Current => _itemQueue.Peek(); object IEnumerator.Current => Current; @@ -202,8 +215,8 @@ protected virtual void Dispose(bool disposing) /// time during traversal and some additional information of the tree object currently visited. /// /// To implement your own Viserator you should consider which state information the Viserator must keep track of. - /// Either you assemble your own State type by deriving from or choose to use one of - /// the standard state types like . Then you need to derive your own class from + /// Either you assemble your own State type by deriving from or choose to use one of + /// the standard state types like . Then you need to derive your own class from /// /// with the TState replaced by your choice of State and TItem replaced by the type of the items you want your Viserator to yield /// during the traversal. @@ -243,8 +256,12 @@ protected internal override void Init(IEnumerable rootList) /// Initializes a new instance of the class. /// /// The root list. - public Viserator(IEnumerable rootList) + /// Optional custom . Needs to be passed to base.ctor, + /// as the initialization and discovery of potential methods to visit is being done in + protected Viserator(IEnumerable rootList, IEnumerable customVisitorModules = null) { + if (customVisitorModules != null) + VisitorModules.AddRange(customVisitorModules); Init(rootList); } diff --git a/src/Xene/Visitor.cs b/src/Xene/Visitor.cs index 506be045d..4460eea2a 100644 --- a/src/Xene/Visitor.cs +++ b/src/Xene/Visitor.cs @@ -106,13 +106,13 @@ internal class VisitorSet public Dictionary> ModuleComponents = new(); } - // The list of visitor methods defined in a concrete child class of Visitor private VisitorSet _visitors; // The static list of all known sets of visitor methods // This is kept to avoid building the visitor map again and again different instances of the same visitor. - private static Dictionary _visitorMap; + // Removed this functionality as we now have the possibility to "side load" additional visitors even with the same VisitorType + // private static Dictionary _visitorMap; #endregion #region Public Traversal Methods @@ -411,6 +411,9 @@ protected bool EnumMoveNextNoComponent() #endregion + /// + /// The list of visitor modules defined during generation. + /// public List VisitorModules = new(); /// @@ -421,19 +424,12 @@ private void ScanForVisitors() if (_visitors != null) return; - if (_visitorMap == null) - _visitorMap = new Dictionary(); - - var myType = GetType(); - if (_visitorMap.TryGetValue(myType, out _visitors)) - return; - _visitors = new VisitorSet(); - AddVisitMethods(); foreach (var module in VisitorModules) { AddVisitMethodsFromModule(module.GetType()); } + AddVisitMethods(); } /// @@ -462,7 +458,6 @@ private void AddVisitMethodsFromModule(Type moduleType) _visitors.ModuleNodes.Add(paramType, VisitorCallerFactory.MakeNodeVistorForModule(methodInfo)); } } - _visitorMap.Add(moduleType, _visitors); } /// @@ -492,7 +487,6 @@ private void AddVisitMethods() _visitors.Nodes.Add(paramType, VisitorCallerFactory.MakeNodeVistor(methodInfo)); } } - _visitorMap.Add(type, _visitors); } private static bool IsVisitor(MethodInfo methodInfo)