From a15d71456620b484929f279e5752df31e44a1e7935906ecf9422cba984379ec5 Mon Sep 17 00:00:00 2001 From: DjordjeIlic Date: Sat, 30 May 2026 00:02:57 +0200 Subject: [PATCH 1/2] Spin offset area added --- Config/DefaultGame.ini | 6 ++++++ Content/Blueprints/BP_WhiteTravel.uasset | 4 ++-- Content/Levels/MainLevel.umap | 4 ++-- Content/Materials/MP_NDI_FEED.uasset | 3 +++ Content/Materials/MP_NDI_FEED_Video.uasset | 3 +++ Content/Materials/M_ConeArea.uasset | 3 +++ Content/Materials/M_NDIStream.uasset | 3 +++ Content/Materials/NewNDIMediaSource.uasset | 3 +++ PSC_SharkTable.uproject | 17 +++++++++++++++++ 9 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 Content/Materials/MP_NDI_FEED.uasset create mode 100644 Content/Materials/MP_NDI_FEED_Video.uasset create mode 100644 Content/Materials/M_ConeArea.uasset create mode 100644 Content/Materials/M_NDIStream.uasset create mode 100644 Content/Materials/NewNDIMediaSource.uasset diff --git a/Config/DefaultGame.ini b/Config/DefaultGame.ini index 823ea00..054e2dc 100644 --- a/Config/DefaultGame.ini +++ b/Config/DefaultGame.ini @@ -5,3 +5,9 @@ CommonButtonAcceptKeyHandling=TriggerClick [/Script/EngineSettings.GeneralProjectSettings] ProjectID=FF51FF9E47059C478A46B88BD6125B41 + +[/Script/PixelStreaming2Settings.PixelStreaming2PluginSettings] +EditorUseRemoteSignallingServer=True +ConnectionURL="ws://10.10.100.204:8888" +LogStats=True + diff --git a/Content/Blueprints/BP_WhiteTravel.uasset b/Content/Blueprints/BP_WhiteTravel.uasset index 861e133..002cbcc 100644 --- a/Content/Blueprints/BP_WhiteTravel.uasset +++ b/Content/Blueprints/BP_WhiteTravel.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5518139c74310ef74688fe5c3298bcc43b7c76aa0ed7fa90fd92166b4d3b3675 -size 576936 +oid sha256:0d2bf15a17d92409253e7813aad58812b0fa76af20345db5e5c99b6c8411e1ad +size 814719 diff --git a/Content/Levels/MainLevel.umap b/Content/Levels/MainLevel.umap index 0b56d7d..5facccf 100644 --- a/Content/Levels/MainLevel.umap +++ b/Content/Levels/MainLevel.umap @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:49ea983b13e3ad199ef75df478ec92a35cec5599ae10a0562c7754d02c686175 -size 163675 +oid sha256:aa53680f67ef94ec0d5cdd570d51d38fbe9985ad9bbf8af0050050f6699ee081 +size 166618 diff --git a/Content/Materials/MP_NDI_FEED.uasset b/Content/Materials/MP_NDI_FEED.uasset new file mode 100644 index 0000000..78f99c3 --- /dev/null +++ b/Content/Materials/MP_NDI_FEED.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b13f867a847aaea5446c094103977eb9ae7f5b15b3227e6406c5c62786683aa6 +size 1242 diff --git a/Content/Materials/MP_NDI_FEED_Video.uasset b/Content/Materials/MP_NDI_FEED_Video.uasset new file mode 100644 index 0000000..1050a9d --- /dev/null +++ b/Content/Materials/MP_NDI_FEED_Video.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:043fbc936686b8bbf54f79fa3d6ef429e05f3228e0559ac8b5d993a975f01fe5 +size 2444 diff --git a/Content/Materials/M_ConeArea.uasset b/Content/Materials/M_ConeArea.uasset new file mode 100644 index 0000000..6275f4a --- /dev/null +++ b/Content/Materials/M_ConeArea.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e59a122ec32c2f0717588fcb242463b0c1a1fefb9b8a104829a4ac098bb2f292 +size 11923 diff --git a/Content/Materials/M_NDIStream.uasset b/Content/Materials/M_NDIStream.uasset new file mode 100644 index 0000000..ff90065 --- /dev/null +++ b/Content/Materials/M_NDIStream.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:902528b5538da59ce17cb898c1b1d3fd2d0d8d26097f555dd6f584a3af1cccb9 +size 14547 diff --git a/Content/Materials/NewNDIMediaSource.uasset b/Content/Materials/NewNDIMediaSource.uasset new file mode 100644 index 0000000..14cc936 --- /dev/null +++ b/Content/Materials/NewNDIMediaSource.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d54ab67a5fb8f1c38d808207b903bbffdd499ed30ba442dfbcf7fab1678821 +size 2425 diff --git a/PSC_SharkTable.uproject b/PSC_SharkTable.uproject index 12f24a0..d73ddfd 100644 --- a/PSC_SharkTable.uproject +++ b/PSC_SharkTable.uproject @@ -10,6 +10,23 @@ "TargetAllowList": [ "Editor" ] + }, + { + "Name": "PixelStreaming2", + "Enabled": true + }, + { + "Name": "NDIMedia", + "Enabled": true, + "SupportedTargetPlatforms": [ + "Win64", + "Mac", + "Linux" + ] + }, + { + "Name": "BasemashVoiceCommander", + "Enabled": true } ] } \ No newline at end of file From 3b960967c426c8df7c0d6220cc5d96fd48af393cab36555c03649090076a29d0 Mon Sep 17 00:00:00 2001 From: DjordjeIlic Date: Mon, 8 Jun 2026 16:59:05 +0200 Subject: [PATCH 2/2] adittion of power calculation for shot distance and power UI --- .vsconfig | 19 + Config/DefaultGame.ini | 7 + Content/Blueprints/BP_AngleTravelLine.uasset | 4 +- Content/Blueprints/BP_GeneralManager.uasset | 4 +- .../BP_PlayerControllerOverride.uasset | 4 +- Content/Blueprints/BP_PowerSelection.uasset | 3 + Content/Blueprints/BP_SpinUI.uasset | 3 + Content/Blueprints/BP_WhiteTravel.uasset | 4 +- Content/Levels/MainLevel.umap | 4 +- Content/Materials/Black.uasset | 4 +- Content/Materials/MP_NDI_FEED.uasset | 3 - Content/Materials/MP_NDI_FEED_Video.uasset | 3 - Content/Materials/M_NDIStream.uasset | 4 +- Content/Materials/NewMediaPlayer.uasset | 3 + Content/Materials/NewMediaPlayer_Video.uasset | 3 + Content/Materials/NewNDIMediaSource.uasset | 4 +- .../Structs/CF_WhiteTravelAnglePower.uasset | 3 + Content/Structs/CF_WhiteTravelPower.uasset | 3 + Content/UI/Pictures/CueBal.uasset | 3 + Content/UI/Pictures/Curve_SpinMapping.uasset | 3 + .../Red_circle_frame_transparent_svg.uasset | 3 + Content/UI/Pictures/minus.uasset | 3 + Content/UI/Pictures/plus.uasset | 3 + Content/UI/WBP_Automatic.uasset | 4 +- Content/UI/WBP_LineLabel.uasset | 4 +- Content/UI/WBP_PowerSelection.uasset | 3 + Content/UI/WBP_Spin.uasset | 3 + PSC_SharkTable.uproject | 7 + .../BasemashVoiceCommander.uplugin | 35 + .../Config/FilterPlugin.ini | 8 + .../BasemashVoiceCommander.Build.cs | 33 + .../Private/BasemashVoiceCommanderModule.cpp | 22 + .../Private/PSVoiceAudioConsumer.cpp | 92 ++ .../Private/VoiceCaptureComponent.cpp | 569 +++++++++++ .../Private/VoiceCommandRegistry.cpp | 209 ++++ .../Private/VoiceCommandSubsystem.cpp | 608 +++++++++++ .../Private/VoiceCommanderSettings.cpp | 23 + .../Private/VoiceObjectEditorComponent.cpp | 961 ++++++++++++++++++ .../Public/BasemashVoiceCommanderLog.h | 8 + .../Public/PSVoiceAudioConsumer.h | 61 ++ .../Public/VoiceCaptureComponent.h | 163 +++ .../Public/VoiceCommandAction.h | 43 + .../Public/VoiceCommandExecutor.h | 28 + .../Public/VoiceCommandRegistry.h | 68 ++ .../Public/VoiceCommandSubsystem.h | 114 +++ .../Public/VoiceCommanderSettings.h | 104 ++ .../Public/VoiceObjectEditorComponent.h | 229 +++++ .../WebServers/get_ps_servers.bat | 215 ++++ .../WebServers/get_ps_servers.sh | 199 ++++ Source/PSC_SharkTable.Target.cs | 12 + Source/PSC_SharkTable/PSC_SharkTable.Build.cs | 10 + Source/PSC_SharkTable/PSC_SharkTable.cpp | 4 + Source/PSC_SharkTableEditor.Target.cs | 12 + UE5_BP_to_CPP_Packaging_Guide.md | 242 +++++ 54 files changed, 4164 insertions(+), 26 deletions(-) create mode 100644 .vsconfig create mode 100644 Content/Blueprints/BP_PowerSelection.uasset create mode 100644 Content/Blueprints/BP_SpinUI.uasset delete mode 100644 Content/Materials/MP_NDI_FEED.uasset delete mode 100644 Content/Materials/MP_NDI_FEED_Video.uasset create mode 100644 Content/Materials/NewMediaPlayer.uasset create mode 100644 Content/Materials/NewMediaPlayer_Video.uasset create mode 100644 Content/Structs/CF_WhiteTravelAnglePower.uasset create mode 100644 Content/Structs/CF_WhiteTravelPower.uasset create mode 100644 Content/UI/Pictures/CueBal.uasset create mode 100644 Content/UI/Pictures/Curve_SpinMapping.uasset create mode 100644 Content/UI/Pictures/Red_circle_frame_transparent_svg.uasset create mode 100644 Content/UI/Pictures/minus.uasset create mode 100644 Content/UI/Pictures/plus.uasset create mode 100644 Content/UI/WBP_PowerSelection.uasset create mode 100644 Content/UI/WBP_Spin.uasset create mode 100644 Plugins/BasemashVoiceCommander/BasemashVoiceCommander.uplugin create mode 100644 Plugins/BasemashVoiceCommander/Config/FilterPlugin.ini create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/BasemashVoiceCommander.Build.cs create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/BasemashVoiceCommanderModule.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/PSVoiceAudioConsumer.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCaptureComponent.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandRegistry.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandSubsystem.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommanderSettings.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceObjectEditorComponent.cpp create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/BasemashVoiceCommanderLog.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/PSVoiceAudioConsumer.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCaptureComponent.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandAction.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandExecutor.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandRegistry.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandSubsystem.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommanderSettings.h create mode 100644 Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceObjectEditorComponent.h create mode 100644 Samples/PixelStreaming2/WebServers/get_ps_servers.bat create mode 100644 Samples/PixelStreaming2/WebServers/get_ps_servers.sh create mode 100644 Source/PSC_SharkTable.Target.cs create mode 100644 Source/PSC_SharkTable/PSC_SharkTable.Build.cs create mode 100644 Source/PSC_SharkTable/PSC_SharkTable.cpp create mode 100644 Source/PSC_SharkTableEditor.Target.cs create mode 100644 UE5_BP_to_CPP_Packaging_Guide.md diff --git a/.vsconfig b/.vsconfig new file mode 100644 index 0000000..125273f --- /dev/null +++ b/.vsconfig @@ -0,0 +1,19 @@ +{ + "version": "1.0", + "components": [ + "Component.Unreal.Debugger", + "Component.Unreal.Ide", + "Microsoft.Net.Component.4.6.2.TargetingPack", + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL", + "Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64", + "Microsoft.VisualStudio.Component.VC.14.44.17.14.ATL", + "Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64", + "Microsoft.VisualStudio.Component.VC.Llvm.Clang", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "Microsoft.VisualStudio.Component.Windows11SDK.22621", + "Microsoft.VisualStudio.Workload.CoreEditor", + "Microsoft.VisualStudio.Workload.ManagedDesktop", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.Workload.NativeGame" + ] +} diff --git a/Config/DefaultGame.ini b/Config/DefaultGame.ini index 054e2dc..6eecd69 100644 --- a/Config/DefaultGame.ini +++ b/Config/DefaultGame.ini @@ -1,5 +1,9 @@ +[/Script/UnrealEd.ProjectPackagingSettings] +BuildTarget=PSC_SharkTable +StagingDirectory=(Path="") + [/Script/CommonUI.CommonUISettings] CommonButtonAcceptKeyHandling=TriggerClick @@ -11,3 +15,6 @@ EditorUseRemoteSignallingServer=True ConnectionURL="ws://10.10.100.204:8888" LogStats=True +[/Script/BasemashVoiceCommander.VoiceCommanderSettings] +GroqApiKey=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 + diff --git a/Content/Blueprints/BP_AngleTravelLine.uasset b/Content/Blueprints/BP_AngleTravelLine.uasset index 7a893c8..bf4da41 100644 --- a/Content/Blueprints/BP_AngleTravelLine.uasset +++ b/Content/Blueprints/BP_AngleTravelLine.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:53758a6455b5d503d323bc7adae6d12c13c55a66bcb71dbc5d7276e36adaf5ca -size 222407 +oid sha256:2536fbe85001f5a2ae17faec007b0fc6ae5a15915acc3bd6044e7713002c9e17 +size 222129 diff --git a/Content/Blueprints/BP_GeneralManager.uasset b/Content/Blueprints/BP_GeneralManager.uasset index ad139b5..400dbc2 100644 --- a/Content/Blueprints/BP_GeneralManager.uasset +++ b/Content/Blueprints/BP_GeneralManager.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e2dd31a47c0e69b38542449523da0c58704c69af0128e4a31832e719d5e9fa57 -size 1323399 +oid sha256:632cc7dc2f4f55d6c6c7a58f5cb63dc58052806caee5ed46d7a8ba90291a9d53 +size 1395378 diff --git a/Content/Blueprints/BP_PlayerControllerOverride.uasset b/Content/Blueprints/BP_PlayerControllerOverride.uasset index 5c9fd8e..e1dc14c 100644 --- a/Content/Blueprints/BP_PlayerControllerOverride.uasset +++ b/Content/Blueprints/BP_PlayerControllerOverride.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e0841afd4b2258762c8680fc1f2d9e5ea6753f3b54698ec3b732fe17bb2466fe -size 155108 +oid sha256:398d0cee921f1a928c46c51c6e74ef4c73f2c5171d91af839967e4bb2c46abe0 +size 202338 diff --git a/Content/Blueprints/BP_PowerSelection.uasset b/Content/Blueprints/BP_PowerSelection.uasset new file mode 100644 index 0000000..454fdf7 --- /dev/null +++ b/Content/Blueprints/BP_PowerSelection.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b1612114212d026ef58a58a246873ccc76b081e3473f5591dfdee1d03612860 +size 40895 diff --git a/Content/Blueprints/BP_SpinUI.uasset b/Content/Blueprints/BP_SpinUI.uasset new file mode 100644 index 0000000..8d629dd --- /dev/null +++ b/Content/Blueprints/BP_SpinUI.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39ad31036b77087220cd9bff2c9155800d99bcd48543a683fdb1786a80dd122 +size 69822 diff --git a/Content/Blueprints/BP_WhiteTravel.uasset b/Content/Blueprints/BP_WhiteTravel.uasset index 002cbcc..08de54f 100644 --- a/Content/Blueprints/BP_WhiteTravel.uasset +++ b/Content/Blueprints/BP_WhiteTravel.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d2bf15a17d92409253e7813aad58812b0fa76af20345db5e5c99b6c8411e1ad -size 814719 +oid sha256:d03a6f2d0aeedd9f2727f5e5076fc750f226147e530db7e9cdf66fac422034a7 +size 838099 diff --git a/Content/Levels/MainLevel.umap b/Content/Levels/MainLevel.umap index 5facccf..125b7ef 100644 --- a/Content/Levels/MainLevel.umap +++ b/Content/Levels/MainLevel.umap @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aa53680f67ef94ec0d5cdd570d51d38fbe9985ad9bbf8af0050050f6699ee081 -size 166618 +oid sha256:a8a21fca35588bea3679ebf27b8b6e993883a90564007faa948b2ec4507c7551 +size 172442 diff --git a/Content/Materials/Black.uasset b/Content/Materials/Black.uasset index b8128da..61555b8 100644 --- a/Content/Materials/Black.uasset +++ b/Content/Materials/Black.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d532995006f32af6d1a43f47a280927f5a39dd3a3678657646cf8aea99ccc396 -size 9948 +oid sha256:becb3a5acb4a083091219138f035b772cb085c40dffa35f706af87fbe6afb0bf +size 9267 diff --git a/Content/Materials/MP_NDI_FEED.uasset b/Content/Materials/MP_NDI_FEED.uasset deleted file mode 100644 index 78f99c3..0000000 --- a/Content/Materials/MP_NDI_FEED.uasset +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b13f867a847aaea5446c094103977eb9ae7f5b15b3227e6406c5c62786683aa6 -size 1242 diff --git a/Content/Materials/MP_NDI_FEED_Video.uasset b/Content/Materials/MP_NDI_FEED_Video.uasset deleted file mode 100644 index 1050a9d..0000000 --- a/Content/Materials/MP_NDI_FEED_Video.uasset +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:043fbc936686b8bbf54f79fa3d6ef429e05f3228e0559ac8b5d993a975f01fe5 -size 2444 diff --git a/Content/Materials/M_NDIStream.uasset b/Content/Materials/M_NDIStream.uasset index ff90065..4173929 100644 --- a/Content/Materials/M_NDIStream.uasset +++ b/Content/Materials/M_NDIStream.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:902528b5538da59ce17cb898c1b1d3fd2d0d8d26097f555dd6f584a3af1cccb9 -size 14547 +oid sha256:908cdaa354787c6a5d6097eccb341cbf1d56ca12aa47138ccbbcc29817a0f3de +size 14731 diff --git a/Content/Materials/NewMediaPlayer.uasset b/Content/Materials/NewMediaPlayer.uasset new file mode 100644 index 0000000..377dff5 --- /dev/null +++ b/Content/Materials/NewMediaPlayer.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6f690aee8fc2a982151cce7e47ffa150c129a9507a5953f15e66ae09ad6edb +size 1257 diff --git a/Content/Materials/NewMediaPlayer_Video.uasset b/Content/Materials/NewMediaPlayer_Video.uasset new file mode 100644 index 0000000..6b2dbc6 --- /dev/null +++ b/Content/Materials/NewMediaPlayer_Video.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95989f0756d6c6b00b187a0564b17600a826fe25d2ff30f38b1969449ed87022 +size 7067 diff --git a/Content/Materials/NewNDIMediaSource.uasset b/Content/Materials/NewNDIMediaSource.uasset index 14cc936..0a9a0cb 100644 --- a/Content/Materials/NewNDIMediaSource.uasset +++ b/Content/Materials/NewNDIMediaSource.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f2d54ab67a5fb8f1c38d808207b903bbffdd499ed30ba442dfbcf7fab1678821 -size 2425 +oid sha256:5750a74d05cd5f7696013401d1f2a681255e5fc83778f2340baefe6520ef7535 +size 2490 diff --git a/Content/Structs/CF_WhiteTravelAnglePower.uasset b/Content/Structs/CF_WhiteTravelAnglePower.uasset new file mode 100644 index 0000000..47ae104 --- /dev/null +++ b/Content/Structs/CF_WhiteTravelAnglePower.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca0e0a67bc48d4fde426a61bb714beb538ef17af4021f6424f33d38184287fa +size 4402 diff --git a/Content/Structs/CF_WhiteTravelPower.uasset b/Content/Structs/CF_WhiteTravelPower.uasset new file mode 100644 index 0000000..25e5a2e --- /dev/null +++ b/Content/Structs/CF_WhiteTravelPower.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81fad2ae7453944fb3db9148af12d3b162e3b27d187e4557e248cbf484928fdb +size 4696 diff --git a/Content/UI/Pictures/CueBal.uasset b/Content/UI/Pictures/CueBal.uasset new file mode 100644 index 0000000..b8f2bc9 --- /dev/null +++ b/Content/UI/Pictures/CueBal.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a9cda5e8fa86493369aa4d4f1fb4ccd3d2e38c404fa1f108699a6fa21633577 +size 213649 diff --git a/Content/UI/Pictures/Curve_SpinMapping.uasset b/Content/UI/Pictures/Curve_SpinMapping.uasset new file mode 100644 index 0000000..24df5e4 --- /dev/null +++ b/Content/UI/Pictures/Curve_SpinMapping.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d23552e15d2e8989535d52a2f69b1cfb32ff2e19f7e5e1fa83f87d57372817 +size 3361 diff --git a/Content/UI/Pictures/Red_circle_frame_transparent_svg.uasset b/Content/UI/Pictures/Red_circle_frame_transparent_svg.uasset new file mode 100644 index 0000000..92c4086 --- /dev/null +++ b/Content/UI/Pictures/Red_circle_frame_transparent_svg.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f47a0ed41389c70abf172511f2790831d124f10179225a0441b18ec4740f1002 +size 173850 diff --git a/Content/UI/Pictures/minus.uasset b/Content/UI/Pictures/minus.uasset new file mode 100644 index 0000000..0e56c64 --- /dev/null +++ b/Content/UI/Pictures/minus.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a814008265bea9570e1c64002476b3a03582faf9a76700e3254f674e79b49c7 +size 25318 diff --git a/Content/UI/Pictures/plus.uasset b/Content/UI/Pictures/plus.uasset new file mode 100644 index 0000000..4cfb405 --- /dev/null +++ b/Content/UI/Pictures/plus.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c4c563a3d7299546ae42752d9ae4ab47773038711e87909bc9815a7330d044c +size 27108 diff --git a/Content/UI/WBP_Automatic.uasset b/Content/UI/WBP_Automatic.uasset index 21a16dc..73f93f1 100644 --- a/Content/UI/WBP_Automatic.uasset +++ b/Content/UI/WBP_Automatic.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eb979ce24e0ded28db2b2a1b699bf3904eb80ae096e5e6cdebfe904ddf48a749 -size 43034 +oid sha256:267326b1de8bf7c96d374bc63ce2b36c50f2049319ee3da39cc589a1777bdcf2 +size 40859 diff --git a/Content/UI/WBP_LineLabel.uasset b/Content/UI/WBP_LineLabel.uasset index b89bd2b..93f5089 100644 --- a/Content/UI/WBP_LineLabel.uasset +++ b/Content/UI/WBP_LineLabel.uasset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a33330ef209b4e076540944503244c75216df0d64c9b85eceefc3a85122fbb06 -size 38579 +oid sha256:b8ef75fd031f058928bbfee1452a93eeb457ab4df65671fe607d4bbff70e30b2 +size 37541 diff --git a/Content/UI/WBP_PowerSelection.uasset b/Content/UI/WBP_PowerSelection.uasset new file mode 100644 index 0000000..0f88aa3 --- /dev/null +++ b/Content/UI/WBP_PowerSelection.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63ec6af21fa97680a7cdecd968d19088d93f5666fc9c5d702b45faa87f83a7e0 +size 387972 diff --git a/Content/UI/WBP_Spin.uasset b/Content/UI/WBP_Spin.uasset new file mode 100644 index 0000000..2658bb9 --- /dev/null +++ b/Content/UI/WBP_Spin.uasset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ed590bcbf73202391a53510491dccb0cdcf3c1e3897532e653953b6490bf010 +size 71108 diff --git a/PSC_SharkTable.uproject b/PSC_SharkTable.uproject index d73ddfd..bb8fc76 100644 --- a/PSC_SharkTable.uproject +++ b/PSC_SharkTable.uproject @@ -3,6 +3,13 @@ "EngineAssociation": "5.7", "Category": "", "Description": "", + "Modules": [ + { + "Name": "PSC_SharkTable", + "Type": "Runtime", + "LoadingPhase": "Default" + } + ], "Plugins": [ { "Name": "ModelingToolsEditorMode", diff --git a/Plugins/BasemashVoiceCommander/BasemashVoiceCommander.uplugin b/Plugins/BasemashVoiceCommander/BasemashVoiceCommander.uplugin new file mode 100644 index 0000000..b91cb60 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/BasemashVoiceCommander.uplugin @@ -0,0 +1,35 @@ +{ + "FileVersion": 3, + "Version": 2, + "VersionName": "0.1.1", + "FriendlyName": "Basemash Voice Commander", + "Description": "Voice-controlled object manipulation plugin. Local mic or Pixel Streaming 2 browser mic as audio source. Groq STT + LLM command parsing. Fully modular via Blueprint-registered actions.", + "Category": "Audio", + "CreatedBy": "Basemash", + "CreatedByURL": "", + "DocsURL": "", + "MarketplaceURL": "", + "SupportURL": "", + "CanContainContent": true, + "IsBetaVersion": true, + "IsExperimentalVersion": false, + "Installed": false, + "Modules": [ + { + "Name": "BasemashVoiceCommander", + "Type": "Runtime", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + { + "Name": "AudioCapture", + "Enabled": true + }, + { + "Name": "PixelStreaming2", + "Enabled": true, + "Optional": true + } + ] +} diff --git a/Plugins/BasemashVoiceCommander/Config/FilterPlugin.ini b/Plugins/BasemashVoiceCommander/Config/FilterPlugin.ini new file mode 100644 index 0000000..651cad0 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Config/FilterPlugin.ini @@ -0,0 +1,8 @@ +[FilterPlugin] +; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and +; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively. +; +; Examples: +; /README.txt +; /Extras/... +; /Binaries/ThirdParty/*.dll diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/BasemashVoiceCommander.Build.cs b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/BasemashVoiceCommander.Build.cs new file mode 100644 index 0000000..e990ebb --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/BasemashVoiceCommander.Build.cs @@ -0,0 +1,33 @@ +// Copyright (c) Basemash. All Rights Reserved. + +using UnrealBuildTool; + +public class BasemashVoiceCommander : ModuleRules +{ + public BasemashVoiceCommander(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + PrecompileForTargets = PrecompileTargetsType.Any; + + PublicDependencyModuleNames.AddRange(new string[] + { + "Core", + "CoreUObject", + "Engine", + "HTTP", + "Json", + "JsonUtilities", + "AudioCapture", + "AudioCaptureCore", + "AudioMixer", + "PixelStreaming2", + "PixelStreaming2Core", + "DeveloperSettings" + }); + + PrivateDependencyModuleNames.AddRange(new string[] + { + "Projects" + }); + } +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/BasemashVoiceCommanderModule.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/BasemashVoiceCommanderModule.cpp new file mode 100644 index 0000000..6dec60d --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/BasemashVoiceCommanderModule.cpp @@ -0,0 +1,22 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "BasemashVoiceCommanderLog.h" +#include "Modules/ModuleManager.h" + +DEFINE_LOG_CATEGORY(LogBasemashVoiceCommander); + +class FBasemashVoiceCommanderModule : public IModuleInterface +{ +public: + virtual void StartupModule() override + { + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("BasemashVoiceCommander module started.")); + } + + virtual void ShutdownModule() override + { + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("BasemashVoiceCommander module shut down.")); + } +}; + +IMPLEMENT_MODULE(FBasemashVoiceCommanderModule, BasemashVoiceCommander) diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/PSVoiceAudioConsumer.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/PSVoiceAudioConsumer.cpp new file mode 100644 index 0000000..c958dba --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/PSVoiceAudioConsumer.cpp @@ -0,0 +1,92 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "PSVoiceAudioConsumer.h" + +#include "Async/Async.h" +#include "BasemashVoiceCommanderLog.h" +#include "Engine/Engine.h" + +void UPSVoiceAudioConsumer::ConsumeRawPCM(const int16_t* AudioData, int InSampleRate, size_t NChannels, size_t NFrames) +{ + FScopeLock Lock(&CriticalSection); + + // First-data flag lets us verify audio actually flows from the new peer after switch. + // Kept inside the lock so the check+set is atomic versus ResetFirstDataFlag on the game thread. + const bool bFireFirstData = !bGotFirstData; + if (bFireFirstData) + { + bGotFirstData = true; + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("PSVoiceAudioConsumer: First audio data received! %d Hz, %d ch, %d frames"), + InSampleRate, + static_cast(NChannels), + static_cast(NFrames)); + AsyncTask(ENamedThreads::GameThread, + []() + { + if (GEngine != nullptr) + GEngine->AddOnScreenDebugMessage(80, 10.f, FColor::Green, TEXT("PS2: Audio data flowing!")); + }); + } + + if (!bIsBuffering) + { + return; + } + + SampleRate = InSampleRate; + NumChannels = static_cast(NChannels); + + // Convert signed 16-bit PCM to normalized float [-1, 1] for downstream voice processing. + const int32 TotalSamples = static_cast(NFrames * NChannels); + Buffer.Reserve(Buffer.Num() + TotalSamples); + for (int32 i = 0; i < TotalSamples; i++) + { + Buffer.Add(static_cast(AudioData[i]) / 32767.0f); + } +} + +void UPSVoiceAudioConsumer::OnAudioConsumerAdded() +{ + bConnected = true; + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("PSVoiceAudioConsumer: Registered with Pixel Streaming audio sink.")); +} + +void UPSVoiceAudioConsumer::OnAudioConsumerRemoved() +{ + // This runs on a PS2 worker thread. Reset flags immediately so downstream observers + // reading bConnected via FThreadSafeBool see the correct state right away. + bConnected = false; + bGotFirstData = false; + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("PSVoiceAudioConsumer: Removed from Pixel Streaming audio sink.")); + + // FSimpleDelegate is not thread-safe — the game-thread owner may Unbind() concurrently. + // Hop to the game thread before firing so Unbind/ExecuteIfBound can never race. + TWeakObjectPtr WeakThis(this); + AsyncTask(ENamedThreads::GameThread, + [WeakThis]() + { + if (UPSVoiceAudioConsumer* StrongThis = WeakThis.Get()) + { + StrongThis->OnRemovedDelegate.ExecuteIfBound(); + } + }); +} + +void UPSVoiceAudioConsumer::StartBuffering() +{ + FScopeLock Lock(&CriticalSection); + Buffer.Empty(); + bIsBuffering = true; +} + +void UPSVoiceAudioConsumer::StopBuffering(TArray& OutPCMData, int32& OutSampleRate, int32& OutNumChannels) +{ + FScopeLock Lock(&CriticalSection); + bIsBuffering = false; + OutPCMData = MoveTemp(Buffer); + OutSampleRate = SampleRate > 0 ? SampleRate : 48000; // Default to 48kHz (PS2 WebRTC native rate). + OutNumChannels = NumChannels > 0 ? NumChannels : 1; + Buffer.Empty(); +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCaptureComponent.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCaptureComponent.cpp new file mode 100644 index 0000000..d6e1d53 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCaptureComponent.cpp @@ -0,0 +1,569 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "VoiceCaptureComponent.h" + +#include "Async/Async.h" +#include "BasemashVoiceCommanderLog.h" +#include "Engine/GameInstance.h" +#include "Engine/World.h" +#include "GameFramework/Actor.h" +#include "Misc/FileHelper.h" +#include "Misc/Paths.h" +#include "PSVoiceAudioConsumer.h" +#include "TimerManager.h" +#include "VoiceCommandSubsystem.h" + +// Pixel Streaming 2 +#include "IPixelStreaming2AudioSink.h" +#include "IPixelStreaming2Module.h" +#include "IPixelStreaming2Streamer.h" +#include "PixelStreaming2Delegates.h" +#include "Templates/PointerVariants.h" + +UVoiceCaptureComponent::UVoiceCaptureComponent() { PrimaryComponentTick.bCanEverTick = false; } + +void UVoiceCaptureComponent::BeginPlay() +{ + Super::BeginPlay(); + + if (AudioSource == EVoiceAudioSource::PixelStreaming) { InitPixelStreamingAudio(); } + else { InitLocalMic(); } +} + +void UVoiceCaptureComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) +{ + // Always clear the PS2 retry timer even if we went down the LocalMic branch — + // AudioSource may have been toggled at runtime. + if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); } + + // Unsubscribe from OnAudioTrackOpenNative if we subscribed. + if (PSAudioTrackOpenHandle.IsValid()) + { + if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) + { + Delegates->OnAudioTrackOpenNative.Remove(PSAudioTrackOpenHandle); + } + PSAudioTrackOpenHandle.Reset(); + } + + if (AudioSource == EVoiceAudioSource::PixelStreaming) { CleanupPixelStreamingAudio(); } + else { CleanupLocalMic(); } + + Super::EndPlay(EndPlayReason); +} + +bool UVoiceCaptureComponent::IsAudioSourceReady() const +{ + if (AudioSource == EVoiceAudioSource::PixelStreaming) { return bPSAudioRegistered; } + return AudioCapture.IsStreamOpen(); +} + +// ============================================================================ +// Local Mic +// ============================================================================ + +void UVoiceCaptureComponent::InitLocalMic() +{ + Audio::FCaptureDeviceInfo DeviceInfo; + if (AudioCapture.GetCaptureDeviceInfo(DeviceInfo)) + { + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: Mic device: %s, SampleRate: %d, Channels: %d"), + *DeviceInfo.DeviceName, + DeviceInfo.PreferredSampleRate, + DeviceInfo.InputChannels); + } + + Audio::FAudioCaptureDeviceParams Params; + + // Capture callback runs on the audio thread — must be cheap and thread-safe. + // Capture a weak UObject pointer; a raw `this` would UAF if the component is GC'd + // before the audio stream fully drains on shutdown. + TWeakObjectPtr WeakThis(this); + Audio::FOnAudioCaptureFunction OnCapture = [WeakThis](const void* InBuffer, + int32 NumFrames, + int32 NumChannels, + int32 SampleRate, + double /*StreamTime*/, + bool /*bOverflow*/) + { + UVoiceCaptureComponent* StrongThis = WeakThis.Get(); + if (StrongThis == nullptr) { return; } + const float* InAudio = static_cast(InBuffer); + FScopeLock Lock(&StrongThis->RecordingCriticalSection); + if (StrongThis->bIsRecording) + { + StrongThis->CaptureSampleRate = SampleRate; + StrongThis->CaptureNumChannels = NumChannels; + StrongThis->RecordedPCMData.Append(InAudio, NumFrames * NumChannels); + } + }; + + if (AudioCapture.OpenAudioCaptureStream(Params, OnCapture, 1024)) + { + if (AudioCapture.StartStream()) + { + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Audio capture stream started.")); + } + else + { + UE_LOG( + LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: Failed to start audio capture stream.")); + } + } + else + { + UE_LOG(LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: Failed to open audio capture stream.")); + } +} + +void UVoiceCaptureComponent::CleanupLocalMic() +{ + if (AudioCapture.IsStreamOpen()) + { + AudioCapture.StopStream(); + AudioCapture.CloseStream(); + } +} + +// ============================================================================ +// Pixel Streaming Audio +// ============================================================================ + +void UVoiceCaptureComponent::InitPixelStreamingAudio() +{ + if (!IPixelStreaming2Module::IsAvailable()) + { + UE_LOG(LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: PixelStreaming2 module not available!")); + return; + } + + PSAudioConsumer = NewObject(this); + + // When the consumer is removed (peer disconnected), restart polling for a new peer. + // Use a weak pointer to the component so we don't fire on a destroyed component. + // NOTE: OnRemovedDelegate is now guaranteed to be invoked on the game thread by + // UPSVoiceAudioConsumer::OnAudioConsumerRemoved (see M2). The explicit AsyncTask + // here is defensive belt-and-braces; it is a no-op when already on the game thread. + TWeakObjectPtr WeakThis(this); + PSAudioConsumer->OnRemovedDelegate.BindLambda( + [WeakThis]() + { + AsyncTask(ENamedThreads::GameThread, + [WeakThis]() + { + UVoiceCaptureComponent* StrongThis = WeakThis.Get(); + if (StrongThis == nullptr) { return; } + + // m2: guard against stale "removed" notifications for a sink we already + // switched away from. If we've already re-attached to a new sink + // (PSAudioSink.Pin() is non-null and we're registered), this "removed" + // event is for the old peer — skip the reset + re-arm so we don't + // stomp the newly-established registration. + if (StrongThis->bPSAudioRegistered && StrongThis->PSAudioSink.Pin().IsValid()) + { + UE_LOG(LogBasemashVoiceCommander, + Verbose, + TEXT("VoiceCaptureComponent: Ignoring stale PS2 remove event; already " + "re-registered to a new sink.")); + return; + } + + StrongThis->bPSAudioRegistered = false; + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCaptureComponent: PS2 audio sink lost, restarting poll...")); + + if (UWorld* World = StrongThis->GetWorld()) + { + World->GetTimerManager().SetTimer(StrongThis->PSRetryTimerHandle, + StrongThis, + &UVoiceCaptureComponent::PSRetryRegistration, + 2.0f, + true, + 0.0f); + } + }); + }); + + // When a new browser peer opens an audio track, switch the consumer over to it. + if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) + { + PSAudioTrackOpenHandle = Delegates->OnAudioTrackOpenNative.AddLambda( + [WeakThis](FString StreamerId, FString PlayerId, bool bIsRemote) + { + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: PS2 AudioTrack opened. streamer=%s player=%s remote=%d"), + *StreamerId, + *PlayerId, + bIsRemote ? 1 : 0); + + if (!bIsRemote) + { + return; // Only browser (remote) mics are of interest. + } + + AsyncTask(ENamedThreads::GameThread, + [WeakThis, StreamerId, PlayerId]() + { + if (UVoiceCaptureComponent* StrongThis = WeakThis.Get()) + { + StrongThis->SwitchAudioToPlayer(StreamerId, PlayerId); + } + }); + }); + } + + // Kick off a 2-second polling cycle that looks for any connected peer with an audio sink. + if (UWorld* World = GetWorld()) + { + World->GetTimerManager().SetTimer( + PSRetryTimerHandle, this, &UVoiceCaptureComponent::PSRetryRegistration, 2.0f, true, 0.0f); + } + + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: PS2 audio init - polling for browser mic...")); +} + +void UVoiceCaptureComponent::PSRetryRegistration() +{ + // PS2 module is optional on some platforms; Get() asserts when !IsAvailable(). + // If PS2 went away (or never existed), bail and shut the retry timer down for good. + if (!IPixelStreaming2Module::IsAvailable()) + { + if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); } + return; + } + + if (bPSAudioRegistered) + { + if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); } + return; + } + + IPixelStreaming2Module& PSModule = IPixelStreaming2Module::Get(); + TArray StreamerIds = PSModule.GetStreamerIds(); + + for (const FString& StreamerId : StreamerIds) + { + TSharedPtr Streamer = PSModule.FindStreamer(StreamerId); + if (Streamer == nullptr) { continue; } + + TArray Players = Streamer->GetConnectedPlayers(); + for (const FString& PlayerId : Players) + { + TWeakPtr SinkWeak = Streamer->GetPeerAudioSink(PlayerId); + TSharedPtr Sink = SinkWeak.Pin(); + if (Sink != nullptr) + { + SwitchAudioToPlayer(StreamerId, PlayerId); + return; + } + } + + // Fallback: some backends expose an "unlistened" sink if no specific peer is ready. + { + TWeakPtr SinkWeak = Streamer->GetUnlistenedAudioSink(); + TSharedPtr Sink = SinkWeak.Pin(); + if (Sink != nullptr) + { + Sink->AddAudioConsumer(TWeakPtrVariant(PSAudioConsumer)); + PSAudioSink = SinkWeak; + bPSAudioRegistered = true; + + if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: PS2 registered via unlistened sink (streamer: %s)"), + *StreamerId); + return; + } + } + } +} + +void UVoiceCaptureComponent::CleanupPixelStreamingAudio() +{ + if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); } + + if (PSAudioConsumer != nullptr && bPSAudioRegistered) + { + if (TSharedPtr Sink = PSAudioSink.Pin()) + { + Sink->RemoveAudioConsumer(TWeakPtrVariant(PSAudioConsumer)); + } + + // M4: RemoveAudioConsumer does not guarantee ConsumeRawPCM on the audio thread + // has finished. Briefly take + release the consumer's critical section to ensure + // any in-flight callback has drained before we tear the component down. + PSAudioConsumer->Fence(); + + bPSAudioRegistered = false; + } + + // Break the consumer's lambda binding so no AsyncTask queued from it references this component. + // Safe to Unbind() here because UPSVoiceAudioConsumer::OnAudioConsumerRemoved now hops to the + // game thread before calling ExecuteIfBound (M2), so Unbind and ExecuteIfBound are serialized. + if (PSAudioConsumer != nullptr) { PSAudioConsumer->OnRemovedDelegate.Unbind(); } +} + +void UVoiceCaptureComponent::SwitchAudioToPlayer(const FString& StreamerId, const FString& PlayerId) +{ + if (PSAudioConsumer == nullptr) { return; } + + // Already attached to this peer — nothing to do. + if (bPSAudioRegistered && CurrentAudioPlayerId == PlayerId) { return; } + + IPixelStreaming2Module& PSModule = IPixelStreaming2Module::Get(); + TSharedPtr Streamer = PSModule.FindStreamer(StreamerId); + if (Streamer == nullptr) { return; } + + TWeakPtr NewSinkWeak = Streamer->GetPeerAudioSink(PlayerId); + TSharedPtr NewSink = NewSinkWeak.Pin(); + if (NewSink == nullptr) { return; } + + // Detach from the old sink before attaching to the new one. + if (bPSAudioRegistered) + { + if (TSharedPtr OldSink = PSAudioSink.Pin()) + { + OldSink->RemoveAudioConsumer(TWeakPtrVariant(PSAudioConsumer)); + } + // M4: Ensure any in-flight ConsumeRawPCM has drained before attaching to the new sink. + PSAudioConsumer->Fence(); + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: PS2 unregistered audio from player %s"), + *CurrentAudioPlayerId); + } + + NewSink->AddAudioConsumer(TWeakPtrVariant(PSAudioConsumer)); + PSAudioSink = NewSinkWeak; + bPSAudioRegistered = true; + CurrentAudioPlayerId = PlayerId; + + // Reset first-data flag so we can verify the new peer actually streams audio. + PSAudioConsumer->ResetFirstDataFlag(); + + if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: PS2 switched mic to player %s (streamer: %s)"), + *PlayerId, + *StreamerId); +} + +// ============================================================================ +// Recording lifecycle +// ============================================================================ + +void UVoiceCaptureComponent::StartRecording() +{ + if (bIsRecording) { return; } + bIsRecording = true; + + if (AudioSource == EVoiceAudioSource::PixelStreaming && PSAudioConsumer != nullptr) { PSAudioConsumer->StartBuffering(); } + else + { + FScopeLock Lock(&RecordingCriticalSection); + RecordedPCMData.Empty(); + } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: Recording started. Source: %s"), + AudioSource == EVoiceAudioSource::PixelStreaming ? TEXT("PixelStreaming") : TEXT("LocalMic")); +} + +void UVoiceCaptureComponent::StopRecordingAndSend() +{ + if (!bIsRecording) { return; } + bIsRecording = false; + + TArray PCMData; + int32 SampleRate = 0; + int32 NumChannels = 0; + + if (AudioSource == EVoiceAudioSource::PixelStreaming && PSAudioConsumer != nullptr) + { + PSAudioConsumer->StopBuffering(PCMData, SampleRate, NumChannels); + } + else + { + FScopeLock Lock(&RecordingCriticalSection); + PCMData = MoveTemp(RecordedPCMData); + SampleRate = CaptureSampleRate > 0 ? CaptureSampleRate : 48000; + NumChannels = CaptureNumChannels > 0 ? CaptureNumChannels : 1; + RecordedPCMData.Empty(); + } + + UE_LOG( + LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Recording stopped. Samples: %d"), PCMData.Num()); + + if (PCMData.Num() == 0) + { + UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceCaptureComponent: No audio data recorded.")); + OnError.Broadcast(TEXT("No audio recorded")); + return; + } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCaptureComponent: Audio format: %d Hz, %d channels"), + SampleRate, + NumChannels); + + // Stereo → mono downmix (simple average). Preserves mono buffers as-is. + TArray MonoData; + if (NumChannels == 2) + { + const int32 NumFrames = PCMData.Num() / 2; + MonoData.SetNum(NumFrames); + for (int32 i = 0; i < NumFrames; i++) + { + MonoData[i] = (PCMData[i * 2] + PCMData[i * 2 + 1]) * 0.5f; + } + } + else { MonoData = MoveTemp(PCMData); } + + // Silence detection: if the loudest sample is below the threshold, abort the request. + float MaxAmplitude = 0.f; + for (float Sample : MonoData) + { + MaxAmplitude = FMath::Max(MaxAmplitude, FMath::Abs(Sample)); + } + + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Max amplitude: %f"), MaxAmplitude); + + if (MaxAmplitude < SilenceThreshold) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCaptureComponent: Audio is silence (max %.6f < threshold %.6f)."), + MaxAmplitude, + SilenceThreshold); + OnError.Broadcast(TEXT("Audio was silence")); + return; + } + + TArray WavData = ConvertPCMToWav(MonoData, SampleRate, 1); + + if (bSaveDebugRecording) + { + const FString DebugPath = FPaths::ProjectSavedDir() / TEXT("debug_recording.wav"); + FFileHelper::SaveArrayToFile(WavData, *DebugPath); + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Saved debug WAV to %s"), *DebugPath); + } + + // Resolve the subsystem via the owning actor's game instance. + UVoiceCommandSubsystem* Subsystem = nullptr; + if (AActor* OwnerActor = GetOwner()) + { + if (UWorld* World = OwnerActor->GetWorld()) + { + if (UGameInstance* GI = World->GetGameInstance()) + { + Subsystem = GI->GetSubsystem(); + } + } + } + + if (Subsystem == nullptr) + { + UE_LOG(LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: UVoiceCommandSubsystem not found.")); + OnError.Broadcast(TEXT("VoiceCommandSubsystem unavailable")); + return; + } + + // Per-request callbacks — the subsystem invokes these only for this request, + // so events reach ONLY the component that originated the call. + FVoiceRequestCallbacks Callbacks; + Callbacks.Context = this; + Callbacks.OnTranscription.AddDynamic(this, &UVoiceCaptureComponent::HandleTranscription); + Callbacks.OnCommandParsed.AddDynamic(this, &UVoiceCaptureComponent::HandleCommandParsed); + Callbacks.OnError.AddDynamic(this, &UVoiceCaptureComponent::HandleError); + + Subsystem->SendAudioToGroq(WavData, ApiKeyOverride, Callbacks); +} + +// ============================================================================ +// Subsystem → component event fan-out +// ============================================================================ + +void UVoiceCaptureComponent::HandleTranscription(const FString& Text) { OnTranscription.Broadcast(Text); } + +void UVoiceCaptureComponent::HandleCommandParsed(const FString& Action, const FString& ParamsJson) +{ + OnCommandParsed.Broadcast(Action, ParamsJson); +} + +void UVoiceCaptureComponent::HandleError(const FString& ErrorMessage) { OnError.Broadcast(ErrorMessage); } + +// ============================================================================ +// WAV encoding (float PCM → 16-bit PCM WAV, byte-for-byte compatible with original) +// ============================================================================ + +TArray +UVoiceCaptureComponent::ConvertPCMToWav(const TArray& PCMData, int32 SampleRate, int32 NumChannels) const +{ + // Float [-1.0, 1.0] → int16 + TArray Int16Data; + Int16Data.SetNum(PCMData.Num()); + for (int32 i = 0; i < PCMData.Num(); i++) + { + const float Clamped = FMath::Clamp(PCMData[i], -1.0f, 1.0f); + Int16Data[i] = static_cast(Clamped * 32767.0f); + } + + const int32 DataSize = Int16Data.Num() * sizeof(int16); + const int32 FileSize = 44 + DataSize; // standard 44-byte WAV header + + TArray WavData; + WavData.SetNum(FileSize); + uint8* Ptr = WavData.GetData(); + + // RIFF header + FMemory::Memcpy(Ptr, "RIFF", 4); + Ptr += 4; + int32 ChunkSize = FileSize - 8; + FMemory::Memcpy(Ptr, &ChunkSize, 4); + Ptr += 4; + FMemory::Memcpy(Ptr, "WAVE", 4); + Ptr += 4; + + // fmt subchunk + FMemory::Memcpy(Ptr, "fmt ", 4); + Ptr += 4; + int32 SubChunk1Size = 16; + FMemory::Memcpy(Ptr, &SubChunk1Size, 4); + Ptr += 4; + int16 AudioFormat = 1; // PCM + FMemory::Memcpy(Ptr, &AudioFormat, 2); + Ptr += 2; + int16 Channels = static_cast(NumChannels); + FMemory::Memcpy(Ptr, &Channels, 2); + Ptr += 2; + FMemory::Memcpy(Ptr, &SampleRate, 4); + Ptr += 4; + int32 ByteRate = SampleRate * NumChannels * sizeof(int16); + FMemory::Memcpy(Ptr, &ByteRate, 4); + Ptr += 4; + int16 BlockAlign = static_cast(NumChannels * sizeof(int16)); + FMemory::Memcpy(Ptr, &BlockAlign, 2); + Ptr += 2; + int16 BitsPerSample = 16; + FMemory::Memcpy(Ptr, &BitsPerSample, 2); + Ptr += 2; + + // data subchunk + FMemory::Memcpy(Ptr, "data", 4); + Ptr += 4; + FMemory::Memcpy(Ptr, &DataSize, 4); + Ptr += 4; + FMemory::Memcpy(Ptr, Int16Data.GetData(), DataSize); + + return WavData; +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandRegistry.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandRegistry.cpp new file mode 100644 index 0000000..4df76e0 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandRegistry.cpp @@ -0,0 +1,209 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "VoiceCommandRegistry.h" + +#include "BasemashVoiceCommanderLog.h" +#include "Engine/GameInstance.h" +#include "Kismet/GameplayStatics.h" + +void UVoiceCommandRegistry::Initialize(FSubsystemCollectionBase& Collection) +{ + Super::Initialize(Collection); + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandRegistry: Initialized.")); +} + +void UVoiceCommandRegistry::Deinitialize() +{ + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandRegistry: Deinitialized (%d actions cleared)."), + Actions.Num()); + Actions.Empty(); + Super::Deinitialize(); +} + +void UVoiceCommandRegistry::RegisterAction(const FVoiceCommandAction& Action) +{ + if (Action.Name.IsEmpty()) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandRegistry::RegisterAction: Action has empty Name, ignoring.")); + return; + } + + // Protocol-reserved name; 'none' is always appended by BuildSystemPrompt and must not be overridden. + if (Action.Name.Equals(TEXT("none"), ESearchCase::IgnoreCase)) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandRegistry::RegisterAction: 'none' is a reserved protocol action, ignoring.")); + return; + } + + const bool bReplacing = Actions.Contains(Action.Name); + Actions.Add(Action.Name, Action); + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandRegistry: %s action '%s'."), + bReplacing ? TEXT("Replaced") : TEXT("Registered"), + *Action.Name); +} + +bool UVoiceCommandRegistry::RegisterSimpleAction(const FString& Name, + const FString& Description, + const FString& ParamSchema, + const TArray& Examples, + const FVoiceActionHandler& Handler) +{ + if (Name.IsEmpty() || Name.Equals(TEXT("none"), ESearchCase::IgnoreCase)) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandRegistry::RegisterSimpleAction: invalid Name ('%s'), ignoring."), + *Name); + return false; + } + + FVoiceCommandAction Action; + Action.Name = Name; + Action.Description = Description; + Action.ParamSchema = ParamSchema; + Action.Examples = Examples; + Action.Handler = Handler; + RegisterAction(Action); + return true; +} + +void UVoiceCommandRegistry::UnregisterAction(const FString& ActionName) +{ + const int32 Removed = Actions.Remove(ActionName); + if (Removed > 0) + { + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandRegistry: Unregistered action '%s'."), *ActionName); + } +} + +bool UVoiceCommandRegistry::OverrideActionHandler(const FString& ActionName, const FVoiceActionHandler& NewHandler) +{ + FVoiceCommandAction* Existing = Actions.Find(ActionName); + if (Existing == nullptr) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandRegistry::OverrideActionHandler: action '%s' not found."), + *ActionName); + return false; + } + + Existing->Handler = NewHandler; + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandRegistry: Overrode handler for '%s'."), *ActionName); + return true; +} + +TArray UVoiceCommandRegistry::GetRegisteredActions() const +{ + TArray Out; + Out.Reserve(Actions.Num()); + for (const TPair& Pair : Actions) { Out.Add(Pair.Value); } + return Out; +} + +bool UVoiceCommandRegistry::HasAction(const FString& ActionName) const { return Actions.Contains(ActionName); } + +const FVoiceCommandAction* UVoiceCommandRegistry::FindAction(const FString& ActionName) const +{ return Actions.Find(ActionName); } + +bool UVoiceCommandRegistry::DispatchAction(const FString& ActionName, const FString& ParamsJson) +{ + FVoiceCommandAction* Entry = Actions.Find(ActionName); + if (Entry == nullptr) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandRegistry::DispatchAction: no action registered for '%s'."), + *ActionName); + return false; + } + + // Handler delegate takes priority over ExecutorObject when bound. + if (Entry->Handler.IsBound()) + { + const bool bResult = Entry->Handler.Execute(ActionName, ParamsJson); + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandRegistry::DispatchAction: '%s' handled by delegate (result=%s)."), + *ActionName, + bResult ? TEXT("true") : TEXT("false")); + return bResult; + } + + // Fall back to interface executor; use Execute_ wrapper so Blueprint implementations also fire. + if (UObject* ExecObj = Entry->ExecutorObject.GetObject()) + { + const bool bResult = IVoiceCommandExecutor::Execute_ExecuteVoiceAction(ExecObj, ActionName, ParamsJson); + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandRegistry::DispatchAction: '%s' handled by ExecutorObject (result=%s)."), + *ActionName, + bResult ? TEXT("true") : TEXT("false")); + return bResult; + } + + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandRegistry::DispatchAction: '%s' has no Handler or ExecutorObject bound."), + *ActionName); + return false; +} + +FString UVoiceCommandRegistry::BuildSystemPrompt() const +{ + FString Prompt; + Prompt.Reserve(1024); + + Prompt += TEXT("You are a voice command parser for an Unreal Engine application. "); + Prompt += TEXT("Your task is to extract a single action and its parameters from the user's text.\n\n"); + Prompt += TEXT("Available actions:\n"); + + int32 Index = 1; + for (const TPair& Pair : Actions) + { + const FVoiceCommandAction& Action = Pair.Value; + + Prompt += FString::Printf(TEXT("%d. %s"), Index++, *Action.Name); + if (!Action.Description.IsEmpty()) { Prompt += FString::Printf(TEXT(" - %s"), *Action.Description); } + Prompt += TEXT("\n"); + + if (!Action.ParamSchema.IsEmpty()) { Prompt += FString::Printf(TEXT(" params: %s\n"), *Action.ParamSchema); } + + if (Action.Examples.Num() > 0) + { + Prompt += TEXT(" Examples:"); + for (const FString& Example : Action.Examples) { Prompt += FString::Printf(TEXT(" \"%s\""), *Example); } + Prompt += TEXT("\n"); + } + + Prompt += TEXT("\n"); + } + + // Protocol constant: 'none' is always last and always present, even if nobody registered anything. + Prompt += FString::Printf(TEXT("%d. none - unrecognized command\n"), Index); + Prompt += TEXT(" params: {}\n\n"); + + Prompt += TEXT("Output format: {\"action\": \"\", \"params\": {}}\n"); + Prompt += TEXT("Example: {\"action\": \"move\", \"params\": {\"direction\": \"left\", \"distance\": 30}}\n"); + Prompt += TEXT("Respond ONLY with a valid JSON object. No prose."); + + return Prompt; +} + +UVoiceCommandRegistry* UVoiceCommandRegistry::Get(const UObject* WorldContextObject) +{ + if (WorldContextObject == nullptr) { return nullptr; } + + UGameInstance* GameInstance = UGameplayStatics::GetGameInstance(WorldContextObject); + if (GameInstance == nullptr) { return nullptr; } + + return GameInstance->GetSubsystem(); +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandSubsystem.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandSubsystem.cpp new file mode 100644 index 0000000..ae0a827 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommandSubsystem.cpp @@ -0,0 +1,608 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "VoiceCommandSubsystem.h" + +#include "BasemashVoiceCommanderLog.h" +#include "Dom/JsonObject.h" +#include "Engine/GameInstance.h" +#include "HttpModule.h" +#include "Interfaces/IHttpResponse.h" +#include "Serialization/JsonReader.h" +#include "Serialization/JsonSerializer.h" +#include "VoiceCommandRegistry.h" +#include "VoiceCommanderSettings.h" + +void UVoiceCommandSubsystem::Initialize(FSubsystemCollectionBase& Collection) +{ + Super::Initialize(Collection); + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: Initialized.")); +} + +void UVoiceCommandSubsystem::Deinitialize() +{ + // Cancel any in-flight HTTP requests we dispatched so their BindUObject callbacks + // don't fire on a destroyed subsystem (PIE shutdown, seamless travel, etc.). + int32 CancelledCount = 0; + for (TWeakPtr& WeakReq : InFlightRequests) + { + if (TSharedPtr Pinned = WeakReq.Pin()) + { + Pinned->CancelRequest(); + ++CancelledCount; + } + } + InFlightRequests.Reset(); + + if (CancelledCount > 0) + { + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandSubsystem: Cancelled %d in-flight request(s) on Deinitialize."), + CancelledCount); + } + + Super::Deinitialize(); +} + +void UVoiceCommandSubsystem::SetApiKey(const FString& Key) +{ + RuntimeApiKey = Key; + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandSubsystem: Runtime API key %s (%d chars)."), + Key.IsEmpty() ? TEXT("cleared") : TEXT("set"), + Key.Len()); +} + +FString UVoiceCommandSubsystem::GetEffectiveApiKey(const FString& PerCallOverride) const +{ + if (!PerCallOverride.IsEmpty()) { return PerCallOverride; } + + if (!RuntimeApiKey.IsEmpty()) { return RuntimeApiKey; } + + if (const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get()) + { + if (!Settings->GroqApiKey.IsEmpty()) { return Settings->GroqApiKey; } + } + + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandSubsystem: No API key available (per-call/runtime/settings all empty).")); + return FString(); +} + +void UVoiceCommandSubsystem::SendAudioToGroq(const TArray& WavAudioData, + const FString& PerCallApiKey, + const FVoiceRequestCallbacks& Callbacks) +{ + // Check bIsProcessing BEFORE touching CurrentCallbacks — otherwise a concurrent + // second request clobbers the in-flight request's delegates. Fire the busy-path + // error through the caller's own callback struct (local temp), not the shared member. + if (bIsProcessing) + { + const FString BusyMsg = TEXT("Already processing a request."); + if (Callbacks.Context.IsValid() && Callbacks.OnError.IsBound()) { Callbacks.OnError.Broadcast(BusyMsg); } + OnAnyError.Broadcast(BusyMsg); + // Do NOT reset CurrentCallbacks: we'd clobber the in-flight request's state. + return; + } + + CurrentCallbacks = Callbacks; + + const FString ApiKey = GetEffectiveApiKey(PerCallApiKey); + if (ApiKey.IsEmpty()) + { + FireError(TEXT("Groq API key not configured.")); + FinishRequest(); + return; + } + + const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get(); + if (Settings == nullptr) + { + FireError(TEXT("VoiceCommander settings not available.")); + FinishRequest(); + return; + } + + bIsProcessing = true; + + CurrentAudioData = WavAudioData; + CurrentTranscription.Empty(); + RequestStartTime = FDateTime::Now(); + + FHttpRequestRef Request = FHttpModule::Get().CreateRequest(); + Request->SetURL(TEXT("https://api.groq.com/openai/v1/audio/transcriptions")); + Request->SetVerb(TEXT("POST")); + Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *ApiKey)); + + const FString Boundary = FString::Printf(TEXT("----UnrealBoundary%d"), FMath::RandRange(100000, 999999)); + Request->SetHeader(TEXT("Content-Type"), FString::Printf(TEXT("multipart/form-data; boundary=%s"), *Boundary)); + + TArray Payload; + + auto AppendString = [&Payload](const FString& Str) + { + FTCHARToUTF8 Converter(*Str); + Payload.Append(reinterpret_cast(Converter.Get()), Converter.Length()); + }; + + AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary)); + AppendString(TEXT("Content-Disposition: form-data; name=\"file\"; filename=\"recording.wav\"\r\n")); + AppendString(TEXT("Content-Type: audio/wav\r\n\r\n")); + Payload.Append(WavAudioData); + AppendString(TEXT("\r\n")); + + AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary)); + AppendString(TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n")); + AppendString(FString::Printf(TEXT("%s\r\n"), *Settings->SttModel)); + + if (!Settings->SttLanguage.IsEmpty()) + { + AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary)); + AppendString(TEXT("Content-Disposition: form-data; name=\"language\"\r\n\r\n")); + AppendString(FString::Printf(TEXT("%s\r\n"), *Settings->SttLanguage)); + } + + AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary)); + AppendString(TEXT("Content-Disposition: form-data; name=\"response_format\"\r\n\r\n")); + AppendString(TEXT("json\r\n")); + + AppendString(FString::Printf(TEXT("--%s--\r\n"), *Boundary)); + + Request->SetContent(Payload); + Request->OnProcessRequestComplete().BindUObject(this, &UVoiceCommandSubsystem::OnSTTResponse); + Request->ProcessRequest(); + InFlightRequests.Add(TWeakPtr(Request)); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandSubsystem: Sending %d bytes of audio to Groq (model=%s, lang=%s)."), + WavAudioData.Num(), + *Settings->SttModel, + *Settings->SttLanguage); +} + +void UVoiceCommandSubsystem::OnSTTResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess) +{ + if (!bSuccess || !Res.IsValid()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("STT request failed - no response.")); + FireError(TEXT("STT request failed - no response.")); + FinishRequest(); + return; + } + + const int32 Code = Res->GetResponseCode(); + const FString Body = Res->GetContentAsString(); + + if (Code != 200) + { + const FString Err = FString::Printf(TEXT("STT error %d: %s"), Code, *Body); + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), Err); + FireError(Err); + FinishRequest(); + return; + } + + TSharedPtr JsonObject; + TSharedRef> Reader = TJsonReaderFactory<>::Create(Body); + if (!FJsonSerializer::Deserialize(Reader, JsonObject) || !JsonObject.IsValid()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("STT response JSON parse failed.")); + FireError(TEXT("STT response JSON parse failed.")); + FinishRequest(); + return; + } + + FString TranscribedText; + JsonObject->TryGetStringField(TEXT("text"), TranscribedText); + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: Transcription = \"%s\""), *TranscribedText); + + if (TranscribedText.IsEmpty()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("STT returned empty transcription.")); + FireError(TEXT("STT returned empty transcription.")); + FinishRequest(); + return; + } + + CurrentTranscription = TranscribedText; + STTCompleteTime = FDateTime::Now(); + + FireTranscription(TranscribedText); + + // Chain STT -> LLM; carry same callbacks so per-request fan-out stays consistent. + // Pass empty PerCall key because GetEffectiveApiKey already resolved successfully + // (and re-resolving via runtime/settings gives the same answer). + // bChainedFromSTT lets ParseCommandWithLLM skip the bIsProcessing guard and the + // timing re-stamp without the empty-string sentinel the old code relied on. + FVoiceRequestCallbacks ChainCallbacks = CurrentCallbacks; + bChainedFromSTT = true; + ParseCommandWithLLM(TranscribedText, FString(), ChainCallbacks); +} + +void UVoiceCommandSubsystem::ParseCommandWithLLM(const FString& TranscribedText, + const FString& PerCallApiKey, + const FVoiceRequestCallbacks& Callbacks) +{ + // When chained from STT the audio path already owns CurrentCallbacks + bIsProcessing; + // otherwise check busy first (C1) and only then adopt the caller's callbacks. + if (!bChainedFromSTT) + { + if (bIsProcessing) + { + const FString BusyMsg = TEXT("Already processing a request."); + if (Callbacks.Context.IsValid() && Callbacks.OnError.IsBound()) { Callbacks.OnError.Broadcast(BusyMsg); } + OnAnyError.Broadcast(BusyMsg); + return; + } + + CurrentCallbacks = Callbacks; + } + + const FString ApiKey = GetEffectiveApiKey(PerCallApiKey); + if (ApiKey.IsEmpty()) + { + FireError(TEXT("Groq API key not configured.")); + bChainedFromSTT = false; + FinishRequest(); + return; + } + + const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get(); + if (Settings == nullptr) + { + FireError(TEXT("VoiceCommander settings not available.")); + bChainedFromSTT = false; + FinishRequest(); + return; + } + + // Fresh text-only call: start timing + mark busy. Chained-from-STT path already did both. + if (!bChainedFromSTT) + { + bIsProcessing = true; + RequestStartTime = FDateTime::Now(); + STTCompleteTime = RequestStartTime; + CurrentTranscription = TranscribedText; + } + + FHttpRequestRef Request = FHttpModule::Get().CreateRequest(); + Request->SetURL(TEXT("https://api.groq.com/openai/v1/chat/completions")); + Request->SetVerb(TEXT("POST")); + Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *ApiKey)); + Request->SetHeader(TEXT("Content-Type"), TEXT("application/json")); + + const FString SystemPrompt = BuildSystemPrompt(); + + // Build the chat-completions body with FJsonObject so control chars / quotes / newlines + // in prompts or transcription all get escaped correctly by the writer (M6). + const TSharedRef Root = MakeShared(); + Root->SetStringField(TEXT("model"), Settings->LlmModel); + Root->SetNumberField(TEXT("temperature"), 0.0); + + const TSharedRef SystemMsg = MakeShared(); + SystemMsg->SetStringField(TEXT("role"), TEXT("system")); + SystemMsg->SetStringField(TEXT("content"), SystemPrompt); + + const TSharedRef UserMsg = MakeShared(); + UserMsg->SetStringField(TEXT("role"), TEXT("user")); + UserMsg->SetStringField(TEXT("content"), TranscribedText); + + TArray> Messages; + Messages.Add(MakeShared(SystemMsg)); + Messages.Add(MakeShared(UserMsg)); + Root->SetArrayField(TEXT("messages"), Messages); + + FString JsonBody; + TSharedRef> Writer = TJsonWriterFactory<>::Create(&JsonBody); + FJsonSerializer::Serialize(Root, Writer); + + Request->SetContentAsString(JsonBody); + Request->OnProcessRequestComplete().BindUObject(this, &UVoiceCommandSubsystem::OnLLMResponse); + Request->ProcessRequest(); + InFlightRequests.Add(TWeakPtr(Request)); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandSubsystem: Sending to LLM (model=%s): \"%s\""), + *Settings->LlmModel, + *TranscribedText); + + // Chain flag is single-use — reset now that dispatch is underway so a later stand-alone + // text-only call runs through the full guard + timing path. + bChainedFromSTT = false; +} + +void UVoiceCommandSubsystem::OnLLMResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess) +{ + if (!bSuccess || !Res.IsValid()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM request failed - no response.")); + FireError(TEXT("LLM request failed - no response.")); + FinishRequest(); + return; + } + + const int32 Code = Res->GetResponseCode(); + const FString Body = Res->GetContentAsString(); + + if (Code != 200) + { + const FString Err = FString::Printf(TEXT("LLM error %d: %s"), Code, *Body); + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), Err); + FireError(Err); + FinishRequest(); + return; + } + + TSharedPtr JsonObject; + TSharedRef> Reader = TJsonReaderFactory<>::Create(Body); + if (!FJsonSerializer::Deserialize(Reader, JsonObject) || !JsonObject.IsValid()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM response JSON parse failed.")); + FireError(TEXT("LLM response JSON parse failed.")); + FinishRequest(); + return; + } + + const TArray>* Choices = nullptr; + if (!JsonObject->TryGetArrayField(TEXT("choices"), Choices) || Choices->Num() == 0) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM response has no choices.")); + FireError(TEXT("LLM response has no choices.")); + FinishRequest(); + return; + } + + const TSharedPtr Choice = (*Choices)[0]->AsObject(); + if (!Choice.IsValid()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM choice is not an object.")); + FireError(TEXT("LLM choice is not an object.")); + FinishRequest(); + return; + } + + // Use Try* variants: GetObjectField crashes if the field is absent or wrong type (M5). + // UE 5.7 TryGetObjectField signature is (FStringView, const TSharedPtr*&) + // — returns pointer to the internal shared ptr, not a fresh one. + const TSharedPtr* MessagePtr = nullptr; + if (!Choice->TryGetObjectField(TEXT("message"), MessagePtr) || !MessagePtr || !MessagePtr->IsValid()) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM message missing.")); + FireError(TEXT("LLM message missing.")); + FinishRequest(); + return; + } + const TSharedPtr& Message = *MessagePtr; + + FString Content; + if (!Message->TryGetStringField(TEXT("content"), Content)) + { + SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM message content missing.")); + FireError(TEXT("LLM message content missing.")); + FinishRequest(); + return; + } + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: LLM response = %s"), *Content); + + TSharedPtr CommandJson; + TSharedRef> CmdReader = TJsonReaderFactory<>::Create(Content); + if (!FJsonSerializer::Deserialize(CmdReader, CommandJson) || !CommandJson.IsValid()) + { + // Truncate + flatten so a multi-line prose response doesn't flood the error log (m9). + const FString Snippet = Content.Left(200).Replace(TEXT("\n"), TEXT(" ")).Replace(TEXT("\r"), TEXT("")); + const FString Err = FString::Printf(TEXT("Failed to parse LLM command JSON: %s"), *Snippet); + SendLogToPanel(TEXT(""), TEXT("{}"), Content, Err); + FireError(Err); + FinishRequest(); + return; + } + + FString Action; + CommandJson->TryGetStringField(TEXT("action"), Action); + + FString ParamsJson; + const TSharedPtr* ParamsPtr = nullptr; + if (CommandJson->TryGetObjectField(TEXT("params"), ParamsPtr) && ParamsPtr && (*ParamsPtr).IsValid()) + { + TSharedRef> Writer = TJsonWriterFactory<>::Create(&ParamsJson); + FJsonSerializer::Serialize((*ParamsPtr).ToSharedRef(), Writer); + } + else { ParamsJson = TEXT("{}"); } + + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: Action='%s' Params=%s"), *Action, *ParamsJson); + + // LLM signals "didn't understand" with action="none". Fire the event so BP UI can react, + // but skip registry dispatch to avoid a spurious "no action registered for 'none'" warning (m3). + if (Action.Equals(TEXT("none"), ESearchCase::IgnoreCase)) + { + SendLogToPanel(Action, ParamsJson, Content); + FireCommandParsed(Action, ParamsJson); + FinishRequest(); + return; + } + + SendLogToPanel(Action, ParamsJson, Content); + + // Dispatch to registry BEFORE firing event: keeps execution ordering deterministic. + // Dispatch result is logged but must not gate the event, listeners may want to observe anyway. + if (UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this)) + { + const bool bDispatched = Registry->DispatchAction(Action, ParamsJson); + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandSubsystem: Dispatch result for '%s' = %s"), + *Action, + bDispatched ? TEXT("handled") : TEXT("unhandled")); + } + else + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandSubsystem: No registry available to dispatch '%s'."), + *Action); + } + + FireCommandParsed(Action, ParamsJson); + FinishRequest(); +} + +FString UVoiceCommandSubsystem::BuildSystemPrompt() const +{ + FString Base; + if (const UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this)) + { + Base = Registry->BuildSystemPrompt(); + if (Registry->GetRegisteredActions().Num() == 0) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandSubsystem: Registry has no actions; sending base prompt only.")); + } + } + else + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandSubsystem: Registry unavailable; sending minimal prompt.")); + } + + const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get(); + if (Settings && !Settings->SttLanguage.IsEmpty()) + { + return FString::Printf(TEXT("User speaks language code: %s.\n\n%s"), *Settings->SttLanguage, *Base); + } + + return Base; +} + +void UVoiceCommandSubsystem::SendLogToPanel(const FString& Action, + const FString& ParamsJson, + const FString& LLMRaw, + const FString& Error) +{ + const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get(); + if (Settings == nullptr || !Settings->bLogToPanel || Settings->LogPanelUrl.IsEmpty()) { return; } + + const FDateTime Now = FDateTime::Now(); + int64 SttMs = (STTCompleteTime - RequestStartTime).GetTotalMilliseconds(); + int64 LlmMs = (Now - STTCompleteTime).GetTotalMilliseconds(); + const int64 TotalMs = (Now - RequestStartTime).GetTotalMilliseconds(); + + if (SttMs < 0) SttMs = 0; + if (LlmMs < 0) LlmMs = 0; + + FHttpRequestRef Request = FHttpModule::Get().CreateRequest(); + Request->SetURL(Settings->LogPanelUrl); + Request->SetVerb(TEXT("POST")); + // If the log panel isn't running, don't stall editor shutdown on the default HTTP retry (n3). + Request->SetTimeout(5.0f); + + const FString Boundary = FString::Printf(TEXT("----LogBoundary%d"), FMath::RandRange(100000, 999999)); + Request->SetHeader(TEXT("Content-Type"), FString::Printf(TEXT("multipart/form-data; boundary=%s"), *Boundary)); + + TArray Payload; + + auto AppendString = [&Payload](const FString& Str) + { + FTCHARToUTF8 Converter(*Str); + Payload.Append(reinterpret_cast(Converter.Get()), Converter.Length()); + }; + + auto AddField = [&](const FString& Name, const FString& Value) + { + AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary)); + AppendString(FString::Printf(TEXT("Content-Disposition: form-data; name=\"%s\"\r\n\r\n"), *Name)); + AppendString(Value + TEXT("\r\n")); + }; + + AddField(TEXT("timestamp"), Now.ToIso8601()); + AddField(TEXT("transcription"), CurrentTranscription); + AddField(TEXT("action"), Action); + AddField(TEXT("params"), ParamsJson); + AddField(TEXT("llm_raw"), LLMRaw); + AddField(TEXT("stt_duration_ms"), FString::FromInt(SttMs)); + AddField(TEXT("llm_duration_ms"), FString::FromInt(LlmMs)); + AddField(TEXT("total_duration_ms"), FString::FromInt(TotalMs)); + AddField(TEXT("error"), Error); + + if (CurrentAudioData.Num() > 0) + { + AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary)); + AppendString(TEXT("Content-Disposition: form-data; name=\"audio\"; filename=\"recording.wav\"\r\n")); + AppendString(TEXT("Content-Type: audio/wav\r\n\r\n")); + Payload.Append(CurrentAudioData); + AppendString(TEXT("\r\n")); + } + + AppendString(FString::Printf(TEXT("--%s--\r\n"), *Boundary)); + + Request->SetContent(Payload); + Request->OnProcessRequestComplete().BindUObject(this, &UVoiceCommandSubsystem::OnLogPanelResponse); + Request->ProcessRequest(); + InFlightRequests.Add(TWeakPtr(Request)); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceCommandSubsystem: Log sent to panel (action=%s, stt=%lldms, llm=%lldms)."), + *Action, + SttMs, + LlmMs); +} + +void UVoiceCommandSubsystem::OnLogPanelResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess) +{ + if (!bSuccess || !Res.IsValid() || Res->GetResponseCode() != 200) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceCommandSubsystem: Log panel POST failed (panel may not be running).")); + } +} + +void UVoiceCommandSubsystem::FireTranscription(const FString& Text) +{ + // Per-request fan-out: only if Context UObject is still alive so we don't + // call into a destroyed component. + if (CurrentCallbacks.Context.IsValid() && CurrentCallbacks.OnTranscription.IsBound()) + { + CurrentCallbacks.OnTranscription.Broadcast(Text); + } + + OnAnyTranscriptionReceived.Broadcast(Text); +} + +void UVoiceCommandSubsystem::FireCommandParsed(const FString& Action, const FString& ParamsJson) +{ + if (CurrentCallbacks.Context.IsValid() && CurrentCallbacks.OnCommandParsed.IsBound()) + { + CurrentCallbacks.OnCommandParsed.Broadcast(Action, ParamsJson); + } + + OnAnyCommandParsed.Broadcast(Action, ParamsJson); +} + +void UVoiceCommandSubsystem::FireError(const FString& ErrorMessage) +{ + if (CurrentCallbacks.Context.IsValid() && CurrentCallbacks.OnError.IsBound()) + { + CurrentCallbacks.OnError.Broadcast(ErrorMessage); + } + + OnAnyError.Broadcast(ErrorMessage); +} + +void UVoiceCommandSubsystem::FinishRequest() +{ + bIsProcessing = false; + CurrentCallbacks = FVoiceRequestCallbacks(); + CurrentAudioData.Empty(); + CurrentTranscription.Empty(); + // Drop completed/expired HTTP handles. Any log-panel request that outlives FinishRequest + // will expire naturally; cancelling it here is fine (it already did its job for this request). + // On Deinitialize we iterate what's left and cancel, which is the protection for M1. + InFlightRequests.Reset(); +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommanderSettings.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommanderSettings.cpp new file mode 100644 index 0000000..42c4f58 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceCommanderSettings.cpp @@ -0,0 +1,23 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "VoiceCommanderSettings.h" + +const UVoiceCommanderSettings* UVoiceCommanderSettings::Get() { return GetDefault(); } + +FString UVoiceCommanderSettings::GetGroqApiKey() { return GetDefault()->GroqApiKey; } + +FString UVoiceCommanderSettings::GetSttModel() { return GetDefault()->SttModel; } + +FString UVoiceCommanderSettings::GetLlmModel() { return GetDefault()->LlmModel; } + +FString UVoiceCommanderSettings::GetSttLanguage() { return GetDefault()->SttLanguage; } + +FString UVoiceCommanderSettings::GetLogPanelUrl() { return GetDefault()->LogPanelUrl; } + +bool UVoiceCommanderSettings::GetLogToPanel() { return GetDefault()->bLogToPanel; } + +FName UVoiceCommanderSettings::GetCategoryName() const +{ + // Lands the settings under Project Settings -> Plugins -> Voice Commander. + return TEXT("Plugins"); +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceObjectEditorComponent.cpp b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceObjectEditorComponent.cpp new file mode 100644 index 0000000..00cb85d --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Private/VoiceObjectEditorComponent.cpp @@ -0,0 +1,961 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#include "VoiceObjectEditorComponent.h" + +#include "BasemashVoiceCommanderLog.h" +#include "Camera/CameraComponent.h" +#include "CollisionQueryParams.h" +#include "Components/MeshComponent.h" +#include "Components/PostProcessComponent.h" +#include "Components/PrimitiveComponent.h" +#include "Components/SceneComponent.h" +#include "Dom/JsonObject.h" +#include "Engine/EngineTypes.h" +#include "Engine/World.h" +#include "GameFramework/Actor.h" +#include "GameFramework/Pawn.h" +#include "GameFramework/PlayerController.h" +#include "Kismet/GameplayStatics.h" +#include "Materials/MaterialInterface.h" +#include "Serialization/JsonReader.h" +#include "Serialization/JsonSerializer.h" +#include "VoiceCaptureComponent.h" +#include "VoiceCommandAction.h" +#include "VoiceCommandRegistry.h" +#include "VoiceCommanderSettings.h" + +UVoiceObjectEditorComponent::UVoiceObjectEditorComponent() +{ + PrimaryComponentTick.bCanEverTick = true; + PrimaryComponentTick.bStartWithTickEnabled = true; +} + +// ============================================================================ +// Lifecycle +// ============================================================================ + +void UVoiceObjectEditorComponent::BeginPlay() +{ + Super::BeginPlay(); + + AActor* Owner = GetOwner(); + if (Owner == nullptr) + { + UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent::BeginPlay: no owning actor.")); + return; + } + + // --- Outline post process --- + if (bCreateOutlineComponent) + { + UPostProcessComponent* PP = NewObject(Owner, TEXT("VoiceEditorOutlinePostProcess")); + if (PP != nullptr) + { + PP->bUnbound = true; + if (USceneComponent* OwnerRoot = Owner->GetRootComponent()) + { + PP->SetupAttachment(OwnerRoot); + } + PP->RegisterComponent(); + + if (!OutlineMaterial.IsNull()) + { + if (UMaterialInterface* Loaded = OutlineMaterial.LoadSynchronous()) + { + PP->Settings.WeightedBlendables.Array.Add(FWeightedBlendable(1.f, Loaded)); + } + else + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: OutlineMaterial failed to load (%s)."), + *OutlineMaterial.ToSoftObjectPath().ToString()); + } + } + + OutlinePostProcess = PP; + } + } + + // NOTE: CaptureComponent is not used internally here — Registry::DispatchAction is the canonical + // path into this component. We deliberately do NOT auto-find it; the field is reserved for + // BP/subclass queries. Binding to OnCommandParsed would also cause double-dispatch (snap+lerp = instant). + + // --- Registry builtins --- + const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get(); + UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this); + + if (Registry == nullptr) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: VoiceCommandRegistry unavailable; skipping builtin registration. " + "Voice actions will not dispatch until registry is available.")); + return; + } + + // TScriptInterface(this): the templated ctor sees that our class + // inherits IVoiceCommandExecutor natively (is_base_of check), so it both calls SetObject(this) + // AND SetInterface(this) — storing the UObject pointer and the raw interface pointer + // (offset differs due to multiple inheritance). The registry's DispatchAction calls + // IVoiceCommandExecutor::Execute_ExecuteVoiceAction(Object, ...) which works off the UObject. + TScriptInterface ExecutorIface(this); + + auto RegisterIfEnabled = [&](bool bEnabled, + const TCHAR* Name, + const TCHAR* Description, + const TCHAR* Schema, + const TArray& Examples) + { + if (!bEnabled) return; + + FVoiceCommandAction Action; + Action.Name = Name; + Action.Description = Description; + Action.ParamSchema = Schema; + Action.Examples = Examples; + Action.ExecutorObject = ExecutorIface; + + Registry->RegisterAction(Action); + RegisteredActionNames.Add(Action.Name); + }; + + const bool bMove = !Settings || Settings->bEnableBuiltinMove; + const bool bRotate = !Settings || Settings->bEnableBuiltinRotate; + const bool bApplyMaterial = !Settings || Settings->bEnableBuiltinApplyMaterial; + const bool bUndo = !Settings || Settings->bEnableBuiltinUndo; + const bool bUndoAll = !Settings || Settings->bEnableBuiltinUndoAll; + const bool bSetTime = !Settings || Settings->bEnableBuiltinSetTime; + + RegisterIfEnabled( + bMove, + TEXT("move"), + TEXT("Move the selected object (the cue ball) along the table plane. direction is one of up, down, left, right. " + "Phrases like \"move the cue ball left\" or just \"left\" mean the same thing. " + "distance is in centimeters; if the user does not say a number, use 50. Never moves vertically."), + TEXT("{\"direction\": \"up|down|left|right\", \"distance\": }"), + {TEXT("move the cue ball left"), TEXT("cue ball right"), TEXT("move up 100"), TEXT("down 30")}); + + RegisterIfEnabled(bRotate, + TEXT("rotate"), + TEXT("Rotate the selected object around the given axis. axis defaults to yaw; " + "if the user does not say an angle, use 90."), + TEXT("{\"axis\": \"yaw|pitch|roll\", \"angle\": }"), + {TEXT("rotate"), TEXT("rotate 45 degrees"), TEXT("turn 90"), TEXT("rotate yaw 90")}); + + RegisterIfEnabled(bApplyMaterial, + TEXT("apply_material"), + TEXT("Apply a material to the selected object."), + TEXT("{\"material\": \"\"}"), + {TEXT("apply brick"), TEXT("set concrete"), TEXT("apply wood")}); + + RegisterIfEnabled(bUndo, + TEXT("undo"), + TEXT("Undo the last action."), + TEXT("{}"), + {TEXT("undo"), TEXT("undo last"), TEXT("go back"), TEXT("revert")}); + + RegisterIfEnabled(bUndoAll, + TEXT("undo_all"), + TEXT("Reset the selected object to its original state."), + TEXT("{}"), + {TEXT("reset"), TEXT("undo all"), TEXT("original"), TEXT("restore")}); + + RegisterIfEnabled(bSetTime, + TEXT("set_time"), + TEXT("Set the time of day (0-24)."), + TEXT("{\"hours\": <0-24>}"), + {TEXT("set time to 14"), TEXT("set time 8"), TEXT("noon")}); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Registered %d builtin actions."), + RegisteredActionNames.Num()); + + // BP-only convenience: auto-select a designated actor (e.g. the cue ball) so voice commands target + // it immediately without a raycast. Pairs with bLockSelection to lock voice control onto it. + if (!InitialSelection.IsNull()) + { + if (AActor* Target = InitialSelection.LoadSynchronous()) + { + SetSelectedActor(Target); + } + else + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: InitialSelection set but failed to load (%s)."), + *InitialSelection.ToSoftObjectPath().ToString()); + } + } +} + +void UVoiceObjectEditorComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) +{ + if (UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this)) + { + // UVoiceCommandRegistry does not expose FindAction in 5.7.3 — fall back to GetRegisteredActions(). + // The list is small (single-digit builtins), so O(n^2) is fine. Only unregister when the registry + // entry still points at us: prevents stomping a sibling UVoiceObjectEditorComponent that re-registered + // the same name (M9) and handles the override-handler case (m4) where another script replaced us + // as executor — in that case we leave the entry alone. + const TArray All = Registry->GetRegisteredActions(); + for (const FString& ActionName : RegisteredActionNames) + { + for (const FVoiceCommandAction& A : All) + { + if (A.Name == ActionName && A.ExecutorObject.GetObject() == this) + { + Registry->UnregisterAction(ActionName); + break; + } + } + } + } + RegisteredActionNames.Reset(); + + if (OutlinePostProcess) + { + OutlinePostProcess->DestroyComponent(); + OutlinePostProcess = nullptr; + } + + FinishAnimationImmediately(); + UndoStack.Reset(); + OriginalStates.Reset(); + + Super::EndPlay(EndPlayReason); +} + +void UVoiceObjectEditorComponent::TickComponent(float DeltaTime, + ELevelTick TickType, + FActorComponentTickFunction* ThisTickFunction) +{ + Super::TickComponent(DeltaTime, TickType, ThisTickFunction); + + if (!IsAnimating() || !IsValid(AnimatingActor)) return; + + AnimElapsed += DeltaTime; + const float Alpha = (AnimDuration > SMALL_NUMBER) ? FMath::Clamp(AnimElapsed / AnimDuration, 0.f, 1.f) : 1.f; + + const float SmoothAlpha = FMath::SmoothStep(0.f, 1.f, Alpha); + + if (bAnimatingLocation) + { + const FVector NewLoc = FMath::Lerp(AnimStartLocation, AnimTargetLocation, SmoothAlpha); + AnimatingActor->SetActorLocation(NewLoc); + } + + if (bAnimatingRotation) + { + const FQuat StartQuat = AnimStartRotation.Quaternion(); + const FQuat TargetQuat = AnimTargetRotation.Quaternion(); + const FQuat NewQuat = FQuat::Slerp(StartQuat, TargetQuat, SmoothAlpha); + AnimatingActor->SetActorRotation(NewQuat.Rotator()); + } + + if (Alpha >= 1.f) + { + if (bAnimatingLocation) AnimatingActor->SetActorLocation(AnimTargetLocation); + if (bAnimatingRotation) AnimatingActor->SetActorRotation(AnimTargetRotation); + + bAnimatingLocation = false; + bAnimatingRotation = false; + AnimatingActor = nullptr; + AnimElapsed = 0.f; + } +} + +// ============================================================================ +// Voice dispatch +// ============================================================================ + +bool UVoiceObjectEditorComponent::ExecuteVoiceAction_Implementation(const FString& ActionName, + const FString& ParamsJson) +{ + // Undo variants don't require selection or params. + if (ActionName == TEXT("undo")) + { + ExecuteUndo(); + return true; + } + if (ActionName == TEXT("undo_all")) + { + ExecuteUndoAll(); + return true; + } + + // Parse params once. + TSharedPtr Params; + if (!ParamsJson.IsEmpty()) + { + TSharedRef> Reader = TJsonReaderFactory<>::Create(ParamsJson); + FJsonSerializer::Deserialize(Reader, Params); + } + + if (ActionName == TEXT("move")) + { + FString Direction = TEXT("up"); + double Distance = 50.0; + if (Params.IsValid()) + { + Params->TryGetStringField(TEXT("direction"), Direction); + Params->TryGetNumberField(TEXT("distance"), Distance); + } + // LLM emits distance 0 for a bare "move left" with no spoken number; fall back to a usable step. + if (Distance <= 0.0) { Distance = 50.0; } + ExecuteMove(Direction, static_cast(Distance)); + return true; + } + + if (ActionName == TEXT("rotate")) + { + FString Axis = TEXT("yaw"); + double Angle = 90.0; + if (Params.IsValid()) + { + Params->TryGetStringField(TEXT("axis"), Axis); + Params->TryGetNumberField(TEXT("angle"), Angle); + } + // LLM emits angle 0 for a bare "rotate" with no spoken number; fall back to a quarter turn. + if (FMath::IsNearlyZero(Angle)) { Angle = 90.0; } + ExecuteRotate(Axis, static_cast(Angle)); + return true; + } + + if (ActionName == TEXT("apply_material")) + { + FString MaterialName; + if (Params.IsValid()) + { + Params->TryGetStringField(TEXT("material"), MaterialName); + } + ExecuteApplyMaterial(MaterialName); + return true; + } + + if (ActionName == TEXT("set_time")) + { + double Hours = 12.0; + if (Params.IsValid()) + { + Params->TryGetNumberField(TEXT("hours"), Hours); + } + ExecuteSetTime(static_cast(Hours)); + return true; + } + + return false; +} + +// ============================================================================ +// Selection +// ============================================================================ + +bool UVoiceObjectEditorComponent::GetSelectionCameraRay(FVector& OutStart, FVector& OutEnd) const +{ + // 1) explicit camera + if (SelectionCameraComponent) + { + OutStart = SelectionCameraComponent->GetComponentLocation(); + OutEnd = OutStart + SelectionCameraComponent->GetForwardVector() * SelectionTraceDistance; + return true; + } + + // 2) auto-find UCameraComponent on owner + if (AActor* Owner = GetOwner()) + { + if (UCameraComponent* Cam = Owner->FindComponentByClass()) + { + OutStart = Cam->GetComponentLocation(); + OutEnd = OutStart + Cam->GetForwardVector() * SelectionTraceDistance; + return true; + } + } + + // 3) fall back to PlayerController view + if (APlayerController* PC = UGameplayStatics::GetPlayerController(this, 0)) + { + FVector Loc; + FRotator Rot; + PC->GetPlayerViewPoint(Loc, Rot); + OutStart = Loc; + OutEnd = Loc + Rot.Vector() * SelectionTraceDistance; + return true; + } + + return false; +} + +void UVoiceObjectEditorComponent::DoSelect() +{ + // Locked to InitialSelection (or a BP SetSelectedActor call): ignore raycast selection entirely. + if (bLockSelection) + { + return; + } + + FVector Start, End; + if (!GetSelectionCameraRay(Start, End)) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent::DoSelect: no camera source available.")); + return; + } + + UWorld* World = GetWorld(); + if (World == nullptr) return; + + FHitResult Hit; + FCollisionQueryParams Params(SCENE_QUERY_STAT(VoiceEditorSelect), false); + if (AActor* Owner = GetOwner()) Params.AddIgnoredActor(Owner); + + if (World->LineTraceSingleByChannel(Hit, Start, End, SelectionTraceChannel.GetValue(), Params)) + { + SetSelectedActorInternal(Hit.GetActor(), bEnforceMovable); + } + else + { + SetSelectedActorInternal(nullptr, bEnforceMovable); + } +} + +void UVoiceObjectEditorComponent::SetSelectedActor(AActor* NewActor) +{ + SetSelectedActorInternal(NewActor, bEnforceMovable); +} + +void UVoiceObjectEditorComponent::ClearSelection() { SetSelectedActorInternal(nullptr, bEnforceMovable); } + +void UVoiceObjectEditorComponent::SetSelectedActorInternal(AActor* NewActor, bool bRequireMovable) +{ + // No-op if same actor — avoids redundant highlight toggle and spurious OnSelectionChanged broadcast. + if (NewActor == SelectedActor) + { + return; + } + + if (NewActor != nullptr && bRequireMovable) + { + bool bHasMovable = false; + TArray SceneComps; + NewActor->GetComponents(SceneComps); + for (USceneComponent* Comp : SceneComps) + { + if (Comp && Comp->Mobility == EComponentMobility::Movable) + { + bHasMovable = true; + break; + } + } + if (!bHasMovable) + { + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: %s is not movable, skipping."), + *NewActor->GetName()); + return; + } + } + + AActor* PreviousActor = SelectedActor; + + // Disable highlight on previous selection (if any). + if (PreviousActor) + { + TArray PrevPrims; + PreviousActor->GetComponents(PrevPrims); + for (UPrimitiveComponent* Comp : PrevPrims) + { + if (Comp != nullptr) Comp->SetRenderCustomDepth(false); + } + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Deselected %s"), + *PreviousActor->GetName()); + } + + SelectedActor = NewActor; + + // Enable highlight on new selection. Outline rendering depends on a post-process material + // reading CustomDepth; without OutlineMaterial bound the stencil is set but invisible. + if (NewActor) + { + TArray PrimitiveComponents; + NewActor->GetComponents(PrimitiveComponents); + for (UPrimitiveComponent* Comp : PrimitiveComponents) + { + if (Comp == nullptr) continue; + Comp->SetRenderCustomDepth(true); + Comp->SetCustomDepthStencilValue(1); + } + UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceObjectEditorComponent: Selected %s"), *NewActor->GetName()); + } + + OnSelectionChanged.Broadcast(PreviousActor, NewActor); +} + +// ============================================================================ +// Action execution +// ============================================================================ + +static void DisableOutlineOn(AActor* Actor) +{ + if (Actor == nullptr) return; + TArray Comps; + Actor->GetComponents(Comps); + for (UPrimitiveComponent* Comp : Comps) + { + if (Comp != nullptr) Comp->SetRenderCustomDepth(false); + } +} + +void UVoiceObjectEditorComponent::ExecuteMove(const FString& Direction, float Distance) +{ + if (!SelectedActor) + { + UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent::ExecuteMove: no selected actor.")); + return; + } + + USceneComponent* RootComp = SelectedActor->GetRootComponent(); + if (RootComp && RootComp->Mobility != EComponentMobility::Movable) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: Cannot move %s - actor is not movable!"), + *SelectedActor->GetName()); + return; + } + + // Commit any in-flight animation first so the undo snapshot captures the final + // transform of the prior command, not its pre-lerp start (m1). + FinishAnimationImmediately(); + + DisableOutlineOn(SelectedActor); + SaveOriginalStateIfNeeded(SelectedActor); + SaveUndoState(SelectedActor, EUndoActionType::Move); + + // Determine camera-relative direction: prefer owner pawn's ControlRotation + // so LLM-emitted "left/right/forward/backward" are viewport-relative. + FRotator CameraRot = FRotator::ZeroRotator; + if (APawn* OwnerPawn = Cast(GetOwner())) + { + CameraRot = OwnerPawn->GetControlRotation(); + } + else if (AActor* Owner = GetOwner()) + { + CameraRot = Owner->GetActorRotation(); + } + + const FRotator YawOnly(0.f, CameraRot.Yaw, 0.f); + const FVector CamForward = FRotationMatrix(YawOnly).GetUnitAxis(EAxis::X); + const FVector CamRight = FRotationMatrix(YawOnly).GetUnitAxis(EAxis::Y); + + // Top-down billiard view: every direction stays in the table (XY) plane. + // "up"/"down" map to the camera-forward axis (screen up/down), never world Z, + // so the ball slides across the table instead of lifting off it. + FVector CamDir = FVector::ZeroVector; + if (Direction == TEXT("up")) + CamDir = CamForward; + else if (Direction == TEXT("down")) + CamDir = -CamForward; + else if (Direction == TEXT("right")) + CamDir = CamRight; + else if (Direction == TEXT("left")) + CamDir = -CamRight; + else + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: Unknown direction '%s', defaulting to up."), + *Direction); + CamDir = CamForward; + } + + // Snap to nearest world axis so the ball slides along the X/Y grid. + const FVector WorldAxes[] = {FVector(1, 0, 0), FVector(-1, 0, 0), FVector(0, 1, 0), FVector(0, -1, 0)}; + float BestDot = -2.f; + FVector MoveDir = FVector::ZeroVector; + for (const FVector& Axis : WorldAxes) + { + const float Dot = FVector::DotProduct(CamDir, Axis); + if (Dot > BestDot) + { + BestDot = Dot; + MoveDir = Axis; + } + } + + const FVector TargetLocation = SelectedActor->GetActorLocation() + MoveDir * Distance; + StartSmoothMove(SelectedActor, TargetLocation); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Moving %s %s by %.1f cm (smooth)."), + *SelectedActor->GetName(), + *Direction, + Distance); +} + +void UVoiceObjectEditorComponent::ExecuteRotate(const FString& Axis, float Angle) +{ + if (!SelectedActor) + { + UE_LOG( + LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent::ExecuteRotate: no selected actor.")); + return; + } + + USceneComponent* RootComp = SelectedActor->GetRootComponent(); + if (RootComp && RootComp->Mobility != EComponentMobility::Movable) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: Cannot rotate %s - actor is not movable!"), + *SelectedActor->GetName()); + return; + } + + // Commit any in-flight animation first so the undo snapshot captures the final + // transform of the prior command, not its pre-lerp start (m1). + FinishAnimationImmediately(); + + DisableOutlineOn(SelectedActor); + SaveOriginalStateIfNeeded(SelectedActor); + SaveUndoState(SelectedActor, EUndoActionType::Rotate); + + FRotator DeltaRotation = FRotator::ZeroRotator; + if (Axis == TEXT("yaw")) + DeltaRotation.Yaw = Angle; + else if (Axis == TEXT("pitch")) + DeltaRotation.Pitch = Angle; + else if (Axis == TEXT("roll")) + DeltaRotation.Roll = Angle; + else + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: Unknown axis '%s', defaulting to yaw."), + *Axis); + DeltaRotation.Yaw = Angle; + } + + const FRotator TargetRotation = SelectedActor->GetActorRotation() + DeltaRotation; + StartSmoothRotate(SelectedActor, TargetRotation); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Rotating %s on %s by %.1f degrees (smooth)."), + *SelectedActor->GetName(), + *Axis, + Angle); +} + +void UVoiceObjectEditorComponent::ExecuteApplyMaterial(const FString& MaterialName) +{ + if (!SelectedActor) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent::ExecuteApplyMaterial: no selected actor.")); + return; + } + + TSoftObjectPtr* MatPtr = MaterialMap.Find(MaterialName); + if (!MatPtr) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: Unknown material '%s' (not in MaterialMap)."), + *MaterialName); + return; + } + + UMaterialInterface* Material = MatPtr->LoadSynchronous(); + if (!Material) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: Material '%s' failed to load (%s)."), + *MaterialName, + *MatPtr->ToSoftObjectPath().ToString()); + return; + } + + // Apply-material doesn't animate, but a prior Move/Rotate might still be mid-lerp. + // Commit it so the undo snapshot (below) captures the final transform (m1). + FinishAnimationImmediately(); + + DisableOutlineOn(SelectedActor); + SaveOriginalStateIfNeeded(SelectedActor); + SaveUndoState(SelectedActor, EUndoActionType::Material); + + TArray MeshComponents; + SelectedActor->GetComponents(MeshComponents); + for (UMeshComponent* MeshComp : MeshComponents) + { + if (MeshComp == nullptr) continue; + const int32 NumMaterials = MeshComp->GetNumMaterials(); + for (int32 i = 0; i < NumMaterials; i++) + { + MeshComp->SetMaterial(i, Material); + } + } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Applied material '%s' to %s."), + *MaterialName, + *SelectedActor->GetName()); +} + +void UVoiceObjectEditorComponent::ExecuteSetTime(float Hours) +{ + float TimeOfDay = FMath::Fmod(Hours, 24.0f); + if (TimeOfDay < 0.f) TimeOfDay += 24.0f; + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Time set to %.2f (mod 24). Wire a Blueprint/subclass to apply to your sky " + "system."), + TimeOfDay); +} + +// ============================================================================ +// Undo +// ============================================================================ + +void UVoiceObjectEditorComponent::ExecuteUndo() +{ + if (UndoStack.Num() == 0) + { + UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent: Undo stack is empty!")); + return; + } + + FinishAnimationImmediately(); + + FUndoEntry Entry = UndoStack.Pop(); + + AActor* Target = Entry.Actor.Get(); + if (!IsValid(Target)) + { + UE_LOG( + LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent: Undo target actor no longer valid.")); + return; + } + + switch (Entry.ActionType) + { + case EUndoActionType::Move: StartSmoothMove(Target, Entry.PrevLocation); break; + case EUndoActionType::Rotate: StartSmoothRotate(Target, Entry.PrevRotation); break; + case EUndoActionType::Material: RestoreMaterials(Target, Entry.PrevMaterials); break; + } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Undo executed on %s. %d steps remaining."), + *Target->GetName(), + UndoStack.Num()); +} + +void UVoiceObjectEditorComponent::ExecuteUndoAll() +{ + if (!SelectedActor) + { + UE_LOG( + LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent: No object selected for undo_all!")); + return; + } + + const TWeakObjectPtr Key(SelectedActor); + const FOriginalState* State = OriginalStates.Find(Key); + if (!State) + { + UE_LOG(LogBasemashVoiceCommander, + Warning, + TEXT("VoiceObjectEditorComponent: No original state saved for %s!"), + *SelectedActor->GetName()); + return; + } + + FinishAnimationImmediately(); + + StartSmoothMoveAndRotate(SelectedActor, State->Location, State->Rotation); + RestoreMaterials(SelectedActor, State->Materials); + + // Drop any undo entries for this actor — it's back to origin, so they'd be confusing. + AActor* SelectedRaw = SelectedActor; + UndoStack.RemoveAll([SelectedRaw](const FUndoEntry& Entry) { return Entry.Actor.Get() == SelectedRaw; }); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Reset %s to original state."), + *SelectedActor->GetName()); +} + +// ============================================================================ +// Undo helpers +// ============================================================================ + +TArray> UVoiceObjectEditorComponent::CaptureMaterials(AActor* Actor) const +{ + TArray> Result; + if (Actor == nullptr) return Result; + + TArray MeshComponents; + Actor->GetComponents(MeshComponents); + + for (UMeshComponent* MeshComp : MeshComponents) + { + if (MeshComp == nullptr) + { + Result.AddDefaulted(); + continue; + } + TArray CompMaterials; + const int32 NumMats = MeshComp->GetNumMaterials(); + for (int32 i = 0; i < NumMats; i++) + { + CompMaterials.Add(MeshComp->GetMaterial(i)); + } + Result.Add(MoveTemp(CompMaterials)); + } + return Result; +} + +void UVoiceObjectEditorComponent::RestoreMaterials(AActor* Actor, const TArray>& Materials) +{ + if (Actor == nullptr) return; + + TArray MeshComponents; + Actor->GetComponents(MeshComponents); + + const int32 NumComps = FMath::Min(MeshComponents.Num(), Materials.Num()); + for (int32 CompIdx = 0; CompIdx < NumComps; CompIdx++) + { + UMeshComponent* MeshComp = MeshComponents[CompIdx]; + if (MeshComp == nullptr) continue; + const TArray& CompMats = Materials[CompIdx]; + const int32 NumMats = FMath::Min(MeshComp->GetNumMaterials(), CompMats.Num()); + for (int32 MatIdx = 0; MatIdx < NumMats; MatIdx++) + { + if (CompMats[MatIdx]) + { + MeshComp->SetMaterial(MatIdx, CompMats[MatIdx]); + } + } + } +} + +void UVoiceObjectEditorComponent::SaveOriginalStateIfNeeded(AActor* Actor) +{ + if (Actor == nullptr) return; + + const TWeakObjectPtr Key(Actor); + if (OriginalStates.Contains(Key)) return; + + FOriginalState State; + State.Location = Actor->GetActorLocation(); + State.Rotation = Actor->GetActorRotation(); + State.Materials = CaptureMaterials(Actor); + OriginalStates.Add(Key, MoveTemp(State)); + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Saved original state for %s."), + *Actor->GetName()); +} + +void UVoiceObjectEditorComponent::SaveUndoState(AActor* Actor, EUndoActionType ActionType) +{ + if (Actor == nullptr) return; + + FUndoEntry Entry; + Entry.Actor = Actor; + Entry.PrevLocation = Actor->GetActorLocation(); + Entry.PrevRotation = Actor->GetActorRotation(); + Entry.PrevMaterials = CaptureMaterials(Actor); + Entry.ActionType = ActionType; + + UndoStack.Add(MoveTemp(Entry)); + + if (MaxUndoSteps > 0 && UndoStack.Num() > MaxUndoSteps) + { + UndoStack.RemoveAt(0, UndoStack.Num() - MaxUndoSteps); + } + + UE_LOG(LogBasemashVoiceCommander, + Log, + TEXT("VoiceObjectEditorComponent: Undo state saved (%d steps in stack)."), + UndoStack.Num()); +} + +// ============================================================================ +// Smooth animation +// ============================================================================ + +void UVoiceObjectEditorComponent::FinishAnimationImmediately() +{ + if (!IsAnimating()) return; + + if (IsValid(AnimatingActor)) + { + if (bAnimatingLocation) AnimatingActor->SetActorLocation(AnimTargetLocation); + if (bAnimatingRotation) AnimatingActor->SetActorRotation(AnimTargetRotation); + } + + bAnimatingLocation = false; + bAnimatingRotation = false; + AnimatingActor = nullptr; + AnimElapsed = 0.f; +} + +void UVoiceObjectEditorComponent::StartSmoothMove(AActor* Actor, const FVector& TargetLocation) +{ + FinishAnimationImmediately(); + if (!IsValid(Actor)) return; + + AnimatingActor = Actor; + AnimStartLocation = Actor->GetActorLocation(); + AnimTargetLocation = TargetLocation; + AnimStartRotation = Actor->GetActorRotation(); + AnimTargetRotation = Actor->GetActorRotation(); + AnimElapsed = 0.f; + bAnimatingLocation = true; + bAnimatingRotation = false; +} + +void UVoiceObjectEditorComponent::StartSmoothRotate(AActor* Actor, const FRotator& TargetRotation) +{ + FinishAnimationImmediately(); + if (!IsValid(Actor)) return; + + AnimatingActor = Actor; + AnimStartLocation = Actor->GetActorLocation(); + AnimTargetLocation = Actor->GetActorLocation(); + AnimStartRotation = Actor->GetActorRotation(); + AnimTargetRotation = TargetRotation; + AnimElapsed = 0.f; + bAnimatingLocation = false; + bAnimatingRotation = true; +} + +void UVoiceObjectEditorComponent::StartSmoothMoveAndRotate(AActor* Actor, + const FVector& TargetLocation, + const FRotator& TargetRotation) +{ + FinishAnimationImmediately(); + if (!IsValid(Actor)) return; + + AnimatingActor = Actor; + AnimStartLocation = Actor->GetActorLocation(); + AnimTargetLocation = TargetLocation; + AnimStartRotation = Actor->GetActorRotation(); + AnimTargetRotation = TargetRotation; + AnimElapsed = 0.f; + bAnimatingLocation = true; + bAnimatingRotation = true; +} diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/BasemashVoiceCommanderLog.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/BasemashVoiceCommanderLog.h new file mode 100644 index 0000000..010a61c --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/BasemashVoiceCommanderLog.h @@ -0,0 +1,8 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Logging/LogMacros.h" + +BASEMASHVOICECOMMANDER_API DECLARE_LOG_CATEGORY_EXTERN(LogBasemashVoiceCommander, Log, All); diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/PSVoiceAudioConsumer.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/PSVoiceAudioConsumer.h new file mode 100644 index 0000000..f0031bf --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/PSVoiceAudioConsumer.h @@ -0,0 +1,61 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "HAL/ThreadSafeBool.h" +#include "IPixelStreaming2AudioConsumer.h" +#include "UObject/NoExportTypes.h" + +#include "PSVoiceAudioConsumer.generated.h" + +/** + * Receives raw PCM audio from a Pixel Streaming 2 browser peer (microphone). + * Buffers audio while recording, then hands it off for voice command processing. + */ +UCLASS() +class BASEMASHVOICECOMMANDER_API UPSVoiceAudioConsumer : public UObject, public IPixelStreaming2AudioConsumer +{ + GENERATED_BODY() + +public: + //~ Begin IPixelStreaming2AudioConsumer interface + virtual void ConsumeRawPCM(const int16_t* AudioData, int InSampleRate, size_t NChannels, size_t NFrames) override; + virtual void OnAudioConsumerAdded() override; + virtual void OnAudioConsumerRemoved() override; + //~ End IPixelStreaming2AudioConsumer interface + + /** Start buffering incoming audio data. */ + void StartBuffering(); + + /** Stop buffering and retrieve the captured audio. Empties internal buffer. */ + void StopBuffering(TArray& OutPCMData, int32& OutSampleRate, int32& OutNumChannels); + + [[nodiscard]] bool IsConnected() const { return bConnected; } + + /** Reset first-data flag so we can verify audio flow after switching to a new peer. */ + void ResetFirstDataFlag() { bGotFirstData = false; } + + /** + * Sync fence used by the owning component during teardown. Taking + releasing the + * audio critical section ensures any in-flight ConsumeRawPCM has drained before + * the consumer/component is destroyed. Safe to call from the game thread after + * the sink's RemoveAudioConsumer(). + */ + void Fence() { FScopeLock Lock(&CriticalSection); } + + /** Fired when this consumer is removed from a sink (peer disconnected). Must be Unbind()'d only on game thread. */ + FSimpleDelegate OnRemovedDelegate; + +private: + TArray Buffer; + bool bIsBuffering = false; + FCriticalSection CriticalSection; + int32 SampleRate = 0; + int32 NumChannels = 0; + + // Touched by the audio thread (ConsumeRawPCM), the PS2 worker thread (OnAudioConsumer*), + // and the game thread (IsConnected / ResetFirstDataFlag) — must be atomic. + FThreadSafeBool bConnected = false; + FThreadSafeBool bGotFirstData = false; +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCaptureComponent.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCaptureComponent.h new file mode 100644 index 0000000..c3214c2 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCaptureComponent.h @@ -0,0 +1,163 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "AudioCaptureCore.h" +#include "Components/ActorComponent.h" +#include "CoreMinimal.h" +#include "HAL/ThreadSafeBool.h" + +#include "VoiceCaptureComponent.generated.h" + +class UPSVoiceAudioConsumer; +class IPixelStreaming2AudioSink; +class UVoiceCommandSubsystem; + +/** + * Audio source selection for UVoiceCaptureComponent. + * LocalMic - Capture from the default OS microphone via Audio::FAudioCapture. + * PixelStreaming - Capture from a remote browser peer microphone via PS2 audio consumer. + */ +UENUM(BlueprintType) +enum class EVoiceAudioSource : uint8 +{ + LocalMic UMETA(DisplayName = "Local Microphone"), + PixelStreaming UMETA(DisplayName = "Pixel Streaming (Browser Mic)") +}; + +/** Broadcast after successful transcription. Fired only on the component that initiated the request. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnVoiceCaptureTranscription, const FString&, Text); +/** Broadcast after LLM has parsed a command. Fired only on the component that initiated the request. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams( + FOnVoiceCaptureCommand, const FString&, Action, const FString&, ParamsJson); +/** Broadcast on any error during the voice pipeline. Fired only on the component that initiated the request. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnVoiceCaptureError, const FString&, ErrorMessage); + +/** + * UVoiceCaptureComponent + * + * Drop on any actor to give it voice-capture abilities. Handles: + * - Local mic or Pixel Streaming browser mic audio capture + * - Start/stop recording lifecycle with per-component event isolation + * - Stereo-to-mono mixdown, silence detection, WAV conversion + * - Round-trip to UVoiceCommandSubsystem (Groq STT + LLM) + * + * Transcription, parsed-command, and error events are delivered ONLY to the component + * that initiated the request (via FVoiceRequestCallbacks) — multiple components can + * coexist without cross-talk. + */ +UCLASS(ClassGroup = ("VoiceCommander"), meta = (BlueprintSpawnableComponent, DisplayName = "Voice Capture")) +class BASEMASHVOICECOMMANDER_API UVoiceCaptureComponent : public UActorComponent +{ + GENERATED_BODY() + +public: + UVoiceCaptureComponent(); + + /** Which audio source to use at runtime. Change before BeginPlay for initial setup. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander") + EVoiceAudioSource AudioSource = EVoiceAudioSource::LocalMic; + + /** + * Runtime-only per-component API key. Set via SetApiKey() or Blueprint at runtime. + * NOT serialized to asset — use Project Settings for a persistent default. + * If non-empty, takes priority over Subsystem runtime key and Settings default. + */ + UPROPERTY(Transient, BlueprintReadWrite, Category = "Voice Commander", meta = (PasswordField = true)) + FString ApiKeyOverride; + + /** Blueprint helper to set the per-component API key at runtime. */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void SetApiKey(const FString& Key) { ApiKeyOverride = Key; } + + /** Silence threshold (max absolute float sample value). Below this, request is aborted with an error event. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Audio") + float SilenceThreshold = 0.001f; + + /** If true, saves debug_recording.wav to ProjectSavedDir on every send. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Debug") + bool bSaveDebugRecording = false; + + /** Begin capturing audio into the internal buffer. No-op if already recording. */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void StartRecording(); + + /** Stop capturing, perform silence check + WAV encode, dispatch to the subsystem. */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void StopRecordingAndSend(); + + /** True between StartRecording and StopRecordingAndSend. */ + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander") + bool IsRecording() const { return bIsRecording; } + + /** + * True if the configured audio source is ready to capture. + * - LocalMic: underlying Audio::FAudioCapture stream is open. + * - PixelStreaming: PS2 consumer is registered on an audio sink. + */ + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander") + bool IsAudioSourceReady() const; + + /** Events fired after Groq round-trip — scoped to THIS component only. */ + UPROPERTY(BlueprintAssignable, Category = "Voice Commander") + FOnVoiceCaptureTranscription OnTranscription; + + UPROPERTY(BlueprintAssignable, Category = "Voice Commander") + FOnVoiceCaptureCommand OnCommandParsed; + + UPROPERTY(BlueprintAssignable, Category = "Voice Commander") + FOnVoiceCaptureError OnError; + +protected: + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; + + // --- Local mic state --- + Audio::FAudioCapture AudioCapture; + int32 CaptureSampleRate = 0; + int32 CaptureNumChannels = 0; + TArray RecordedPCMData; + FCriticalSection RecordingCriticalSection; + + // --- Pixel Streaming state --- + UPROPERTY() + UPSVoiceAudioConsumer* PSAudioConsumer = nullptr; + + TWeakPtr PSAudioSink; + + // Game-thread dominant, but read from the stale OnRemovedDelegate lambda path too. + FThreadSafeBool bPSAudioRegistered = false; + FTimerHandle PSRetryTimerHandle; + FString CurrentAudioPlayerId; + + /** Handle for OnAudioTrackOpenNative so we can cleanly unsubscribe in EndPlay. */ + FDelegateHandle PSAudioTrackOpenHandle; + + // --- Recording state --- + // Written on the game thread (Start/Stop), read on the audio thread (LocalMic capture callback). + // FThreadSafeBool keeps the cross-thread read/write formally race-free without expanding the critical section. + FThreadSafeBool bIsRecording = false; + + // --- Lifecycle --- + void InitLocalMic(); + void CleanupLocalMic(); + void InitPixelStreamingAudio(); + void PSRetryRegistration(); + void CleanupPixelStreamingAudio(); + void SwitchAudioToPlayer(const FString& StreamerId, const FString& PlayerId); + + // --- Utility --- + TArray ConvertPCMToWav(const TArray& PCMData, int32 SampleRate, int32 NumChannels) const; + + // --- Subsystem callback glue --- + // Subsystem invokes these via FVoiceRequestCallbacks when this component is the request origin. + // Each one simply re-broadcasts the corresponding multicast so listeners on this component only see events for their own requests. + UFUNCTION() + void HandleTranscription(const FString& Text); + + UFUNCTION() + void HandleCommandParsed(const FString& Action, const FString& ParamsJson); + + UFUNCTION() + void HandleError(const FString& ErrorMessage); +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandAction.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandAction.h new file mode 100644 index 0000000..20d353d --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandAction.h @@ -0,0 +1,43 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ScriptInterface.h" +#include "VoiceCommandExecutor.h" +#include "VoiceCommandAction.generated.h" + +DECLARE_DYNAMIC_DELEGATE_RetVal_TwoParams( + bool, FVoiceActionHandler, const FString&, ActionName, const FString&, ParamsJson); + +USTRUCT(BlueprintType) +struct BASEMASHVOICECOMMANDER_API FVoiceCommandAction +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander") + FString Name; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander") + FString Description; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander") + FString ParamSchema; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander") + TArray Examples; + + // Prefer Handler if bound; otherwise ExecutorObject. + UPROPERTY( + EditAnywhere, + BlueprintReadWrite, + Category = "Voice Commander", + meta = (ToolTip = "Blueprint-bindable delegate path. Prefer this for one-off BP event graphs. For C++ executors that also need Blueprint override support, set ExecutorObject (IVoiceCommandExecutor interface) instead.")) + FVoiceActionHandler Handler; + + UPROPERTY(EditAnywhere, + BlueprintReadWrite, + Category = "Voice Commander", + meta = (ToolTip = "Native/C++ executor path. Set this when your action is implemented in a UObject that implements IVoiceCommandExecutor (supports Blueprint override via BlueprintNativeEvent). Handler takes priority when both are set.")) + TScriptInterface ExecutorObject; +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandExecutor.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandExecutor.h new file mode 100644 index 0000000..a4d1d14 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandExecutor.h @@ -0,0 +1,28 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/Interface.h" + +#include "VoiceCommandExecutor.generated.h" + +UINTERFACE(MinimalAPI, Blueprintable) +class UVoiceCommandExecutor : public UInterface +{ + GENERATED_BODY() +}; + +class BASEMASHVOICECOMMANDER_API IVoiceCommandExecutor +{ + GENERATED_BODY() + +public: + // Return true if this executor handled the action. Blueprint classes can implement this too. + UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "Voice Commander") + bool ExecuteVoiceAction(const FString& ActionName, const FString& ParamsJson); + virtual bool ExecuteVoiceAction_Implementation(const FString& ActionName, const FString& ParamsJson) + { + return false; + } +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandRegistry.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandRegistry.h new file mode 100644 index 0000000..0dd74a4 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandRegistry.h @@ -0,0 +1,68 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Subsystems/GameInstanceSubsystem.h" +#include "VoiceCommandAction.h" + +#include "VoiceCommandRegistry.generated.h" + +UCLASS(BlueprintType) +class BASEMASHVOICECOMMANDER_API UVoiceCommandRegistry : public UGameInstanceSubsystem +{ + GENERATED_BODY() + +public: + virtual void Initialize(FSubsystemCollectionBase& Collection) override; + virtual void Deinitialize() override; + + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry") + void RegisterAction(const FVoiceCommandAction& Action); + + /** Convenience overload: register an action from Blueprint with a single call. + * Wraps the full FVoiceCommandAction — use this when you don't need an + * IVoiceCommandExecutor ExecutorObject, just a Blueprint event handler. + * Returns true if registered (false if Name is empty or "none"). */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry") + bool RegisterSimpleAction(const FString& Name, + const FString& Description, + const FString& ParamSchema, + const TArray& Examples, + const FVoiceActionHandler& Handler); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry") + void UnregisterAction(const FString& ActionName); + + // Replace only the Handler of an existing action, keep Name/Description/Examples/Schema. + // If action doesn't exist, does nothing. Returns true on success. + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry") + bool OverrideActionHandler(const FString& ActionName, const FVoiceActionHandler& NewHandler); + + [[nodiscard]] UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry") + TArray GetRegisteredActions() const; + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Registry") + bool HasAction(const FString& ActionName) const; + + /** Returns pointer to the registered FVoiceCommandAction for the given name, or null. + * The returned pointer is valid until the next Register/Unregister call. */ + [[nodiscard]] const FVoiceCommandAction* FindAction(const FString& ActionName) const; + + // Dispatches to Handler delegate first, then falls back to ExecutorObject. + // Returns true if either handler ran and returned true. + [[nodiscard]] bool DispatchAction(const FString& ActionName, const FString& ParamsJson); + + // Builds the LLM system prompt describing all registered actions. + // Format matches the existing VoiceCommandSubsystem contract + // (LLM returns {"action": "...", "params": {...}} JSON only). + [[nodiscard]] FString BuildSystemPrompt() const; + + // Convenience helper to get the registry from any UObject + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Registry", meta = (WorldContext = "WorldContextObject")) + static UVoiceCommandRegistry* Get(const UObject* WorldContextObject); + +private: + UPROPERTY() + TMap Actions; +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandSubsystem.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandSubsystem.h new file mode 100644 index 0000000..3d60d05 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommandSubsystem.h @@ -0,0 +1,114 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Interfaces/IHttpRequest.h" +#include "Subsystems/GameInstanceSubsystem.h" + +#include "VoiceCommandSubsystem.generated.h" + +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnVoiceCommandParsed, const FString&, Action, const FString&, ParamsJson); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTranscriptionReceived, const FString&, TranscribedText); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnVoiceCommandError, const FString&, ErrorMessage); + +/** + * Per-request callback payload. Components binding these only receive events + * from requests they initiated; other listeners see the global multicasts. + */ +USTRUCT(BlueprintType) +struct BASEMASHVOICECOMMANDER_API FVoiceRequestCallbacks +{ + GENERATED_BODY() + + // Weak ref guards against the caller UObject being destroyed mid-flight. + UPROPERTY() + TWeakObjectPtr Context; + + UPROPERTY() + FOnTranscriptionReceived OnTranscription; + + UPROPERTY() + FOnVoiceCommandParsed OnCommandParsed; + + UPROPERTY() + FOnVoiceCommandError OnError; +}; + +UCLASS(BlueprintType) +class BASEMASHVOICECOMMANDER_API UVoiceCommandSubsystem : public UGameInstanceSubsystem +{ + GENERATED_BODY() + +public: + virtual void Initialize(FSubsystemCollectionBase& Collection) override; + virtual void Deinitialize() override; + + /** Runtime override for the Groq API key. Takes priority over Project Settings default. + * NOTE: Does not persist across GameInstance restarts (PIE stop/start, seamless travel). + * Set again after restart, or rely on the Settings default. */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void SetApiKey(const FString& Key); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander") + FString GetEffectiveApiKey(const FString& PerCallOverride) const; + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void SendAudioToGroq(const TArray& WavAudioData, + const FString& PerCallApiKey, + const FVoiceRequestCallbacks& Callbacks); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ParseCommandWithLLM(const FString& TranscribedText, + const FString& PerCallApiKey, + const FVoiceRequestCallbacks& Callbacks); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander") + bool IsProcessing() const { return bIsProcessing; } + + UPROPERTY(BlueprintAssignable, Category = "Voice Commander") + FOnVoiceCommandParsed OnAnyCommandParsed; + + UPROPERTY(BlueprintAssignable, Category = "Voice Commander") + FOnTranscriptionReceived OnAnyTranscriptionReceived; + + UPROPERTY(BlueprintAssignable, Category = "Voice Commander") + FOnVoiceCommandError OnAnyError; + +private: + FString RuntimeApiKey; + + bool bIsProcessing = false; + + FVoiceRequestCallbacks CurrentCallbacks; + TArray CurrentAudioData; + FString CurrentTranscription; + FDateTime RequestStartTime; + FDateTime STTCompleteTime; + + // True while OnSTTResponse is chaining into ParseCommandWithLLM. Lets the LLM path + // skip the bIsProcessing guard and preserve timing set by the audio path. + bool bChainedFromSTT = false; + + // Weak refs to every in-flight HTTP request we dispatched. Cleared on FinishRequest; + // iterated in Deinitialize to cancel outstanding work before teardown so BindUObject + // callbacks never fire on a destroyed subsystem. + TArray> InFlightRequests; + + void OnSTTResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess); + void OnLLMResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess); + + FString BuildSystemPrompt() const; + + void SendLogToPanel(const FString& Action, + const FString& ParamsJson, + const FString& LLMRaw, + const FString& Error = TEXT("")); + void OnLogPanelResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess); + + void FireTranscription(const FString& Text); + void FireCommandParsed(const FString& Action, const FString& ParamsJson); + void FireError(const FString& ErrorMessage); + + void FinishRequest(); +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommanderSettings.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommanderSettings.h new file mode 100644 index 0000000..7aaf4c4 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceCommanderSettings.h @@ -0,0 +1,104 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Engine/DeveloperSettings.h" +#include "UObject/Object.h" +#include "VoiceCommanderSettings.generated.h" + +/** + * Project-level settings for the Basemash Voice Commander plugin. + * Surfaces under Project Settings -> Plugins -> Voice Commander. + * Values are persisted to DefaultGame.ini (config=Game, defaultconfig). + * + * NOTE: Blueprint users should treat Settings as read-only. Do NOT write to the CDO + * from Blueprint (e.g. via Get() -> SetMember) — this can silently persist to + * DefaultGame.ini on editor save. To change the API key at runtime, go through + * UVoiceCommandSubsystem::SetApiKey() instead. + */ +UCLASS(config = Game, defaultconfig, BlueprintType, meta = (DisplayName = "Voice Commander")) +class BASEMASHVOICECOMMANDER_API UVoiceCommanderSettings : public UDeveloperSettings +{ + GENERATED_BODY() + +public: + // --- API --- + + // Fallback Groq API key used when no per-user/runtime override is set. + UPROPERTY( + config, + EditAnywhere, + Category = "API", + meta = (DisplayName = "Groq API Key (Default)", + PasswordField = true, + ToolTip = "Development default. Persists to DefaultGame.ini and SHIPS IN PACKAGED BUILDS. For production, leave empty here and set at runtime via UVoiceCommandSubsystem::SetApiKey() from an environment variable, OS keychain, or backend proxy.")) + FString GroqApiKey; + + UPROPERTY(config, EditAnywhere, Category = "API", meta = (DisplayName = "STT Model")) + FString SttModel = TEXT("whisper-large-v3-turbo"); + + UPROPERTY(config, EditAnywhere, Category = "API", meta = (DisplayName = "LLM Model")) + FString LlmModel = TEXT("llama-3.3-70b-versatile"); + + UPROPERTY(config, EditAnywhere, Category = "API", meta = (DisplayName = "STT Language")) + FString SttLanguage = TEXT("en"); + + // --- Logging --- + + UPROPERTY(config, EditAnywhere, Category = "Logging", meta = (DisplayName = "Log Panel URL")) + FString LogPanelUrl = TEXT("http://localhost:3001/api/log"); + + UPROPERTY(config, EditAnywhere, Category = "Logging", meta = (DisplayName = "Log To Panel")) + bool bLogToPanel = true; + + // --- Builtin Actions --- + + UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Move")) + bool bEnableBuiltinMove = true; + + UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Rotate")) + bool bEnableBuiltinRotate = true; + + UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Apply Material")) + bool bEnableBuiltinApplyMaterial = true; + + UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Undo")) + bool bEnableBuiltinUndo = true; + + UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Undo All")) + bool bEnableBuiltinUndoAll = true; + + UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Set Time")) + bool bEnableBuiltinSetTime = true; + + /** Convenience accessor for C++ and Blueprint callers. Returns the CDO. + * Blueprint users: the const is stripped at the BP layer — do NOT write + * through this pointer. Use SetApiKey on the subsystem for runtime key + * changes, and use the GetX() helpers below for read access. */ + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static const UVoiceCommanderSettings* Get(); + + // --- Safe per-property BP getters (return by copy, cannot mutate the CDO) --- + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static FString GetGroqApiKey(); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static FString GetSttModel(); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static FString GetLlmModel(); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static FString GetSttLanguage(); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static FString GetLogPanelUrl(); + + [[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings") + static bool GetLogToPanel(); + + // UDeveloperSettings interface + virtual FName GetCategoryName() const override; +}; diff --git a/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceObjectEditorComponent.h b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceObjectEditorComponent.h new file mode 100644 index 0000000..25e1103 --- /dev/null +++ b/Plugins/BasemashVoiceCommander/Source/BasemashVoiceCommander/Public/VoiceObjectEditorComponent.h @@ -0,0 +1,229 @@ +// Copyright (c) Basemash. All Rights Reserved. + +#pragma once + +#include "Components/ActorComponent.h" +#include "CoreMinimal.h" +#include "Engine/EngineTypes.h" +#include "UObject/SoftObjectPtr.h" +#include "VoiceCommandExecutor.h" + +#include "VoiceObjectEditorComponent.generated.h" + +class UCameraComponent; +class UPostProcessComponent; +class UMaterialInterface; +class UVoiceCaptureComponent; + +// Tip akcije za undo sistem - koji tip operacije je izveden +UENUM() +enum class EUndoActionType : uint8 +{ + Move, + Rotate, + Material +}; + +// Undo stack entry. Raw TArray> can't be USTRUCT/UPROPERTY-reflected, +// so we leave these as plain C++ structs (no USTRUCT), matching the original code. +struct FUndoEntry +{ + TWeakObjectPtr Actor; + FVector PrevLocation = FVector::ZeroVector; + FRotator PrevRotation = FRotator::ZeroRotator; + TArray> PrevMaterials; + EUndoActionType ActionType = EUndoActionType::Move; +}; + +// Originalno stanje objekta - cuva se pri prvom kontaktu, za "vrati u originalno" +struct FOriginalState +{ + FVector Location = FVector::ZeroVector; + FRotator Rotation = FRotator::ZeroRotator; + TArray> Materials; +}; + +/** Fired whenever the selected actor changes (raycast hit, BP-driven SetSelectedActor, or ClearSelection). + * PreviousActor / NewActor may be null. Use this to sync HUD/UI/SFX to selection state. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnSelectionChanged, AActor*, PreviousActor, AActor*, NewActor); + +/** + * UVoiceObjectEditorComponent + * + * Opinionated sample executor that implements the "object editor" voice actions + * (move / rotate / apply_material / set_time / undo / undo_all) on top of the + * BasemashVoiceCommander pipeline. + * + * Drop onto any pawn-ish actor alongside a UVoiceCaptureComponent to get + * voice-driven manipulation of any movable actor picked by the selection raycast. + * + * Registers itself as an IVoiceCommandExecutor with UVoiceCommandRegistry for + * every enabled built-in action from UVoiceCommanderSettings. Registry dispatch + * is the only path into this component — does not bind to + * UVoiceCaptureComponent::OnCommandParsed (dual-binding caused double-dispatch + * on move/rotate). + */ +UCLASS(ClassGroup = ("VoiceCommander"), meta = (BlueprintSpawnableComponent, DisplayName = "Voice Object Editor")) +class BASEMASHVOICECOMMANDER_API UVoiceObjectEditorComponent : public UActorComponent, public IVoiceCommandExecutor +{ + GENERATED_BODY() + +public: + UVoiceObjectEditorComponent(); + + // --- Wiring --- + + /** Optional reference to the capture component on the same actor. Not used internally by the editor component; reserved for BP/subclass queries (e.g., IsAudioSourceReady(), StartRecording from BP). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander") + UVoiceCaptureComponent* CaptureComponent = nullptr; + + /** Optional explicit camera for selection raycast. If null: auto-find UCameraComponent on owner, else fall back to PlayerController view. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection") + UCameraComponent* SelectionCameraComponent = nullptr; + + // --- Selection config --- + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection") + float SelectionTraceDistance = 10000.f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection") + TEnumAsByte SelectionTraceChannel = ECC_Visibility; + + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Voice Commander|Selection") + AActor* SelectedActor = nullptr; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection") + bool bEnforceMovable = false; + + /** Optional. If set, this actor is auto-selected at BeginPlay (e.g. the cue ball). Ideal for + * Blueprint-only projects with no character: drop this component's actor in the level and point + * this at the object voice should control. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection") + TSoftObjectPtr InitialSelection; + + /** If true, DoSelect() (the camera raycast) is disabled so the selection cannot change. + * Combine with InitialSelection to lock voice control onto a single object. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection") + bool bLockSelection = false; + + /** Fires when SelectedActor changes (DoSelect raycast, SetSelectedActor BP call, or ClearSelection). + * Both arguments may be null (e.g., NewActor is null when selection is cleared). */ + UPROPERTY(BlueprintAssignable, Category = "Voice Commander|Selection") + FOnSelectionChanged OnSelectionChanged; + + // --- Outline post process --- + + /** If true, a UPostProcessComponent will be created on the owner at BeginPlay and driven by OutlineMaterial. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Outline") + bool bCreateOutlineComponent = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Outline") + TSoftObjectPtr OutlineMaterial; + + // --- Material map (replaces hardcoded path list) --- + + /** Map from voice-recognized material name (e.g. "Mat_Cigla") to material asset reference. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Materials") + TMap> MaterialMap; + + // --- Animation config --- + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Animation") + float AnimDuration = 0.8f; + + // --- Undo config --- + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Undo") + int32 MaxUndoSteps = 20; + + // --- Blueprint-callable actions --- + + /** Camera-raycast selection: line-trace from SelectionCameraComponent (or auto-found camera) forward + * by SelectionTraceDistance, and select the first hit actor with at least one Movable scene component. + * Wraps SetSelectedActorInternal with bEnforceMovable=true. */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Selection") + void DoSelect(); + + /** Programmatic selection setter — bypass raycast and select ANY actor (Movable or not). + * Use this for click-to-select, tag/class lookup, UI list pick, custom proximity rules, etc. + * Pass nullptr to clear selection (equivalent to ClearSelection). Fires OnSelectionChanged + * unless the actor is already selected. */ + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Selection") + void SetSelectedActor(AActor* NewActor); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander|Selection") + void ClearSelection(); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ExecuteMove(const FString& Direction, float Distance); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ExecuteRotate(const FString& Axis, float Angle); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ExecuteApplyMaterial(const FString& MaterialName); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ExecuteSetTime(float Hours); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ExecuteUndo(); + + UFUNCTION(BlueprintCallable, Category = "Voice Commander") + void ExecuteUndoAll(); + + // --- IVoiceCommandExecutor --- + virtual bool ExecuteVoiceAction_Implementation(const FString& ActionName, const FString& ParamsJson) override; + +protected: + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; + virtual void + TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override; + + // --- Undo + state tracking --- + TArray UndoStack; + TMap, FOriginalState> OriginalStates; + + void SaveUndoState(AActor* Actor, EUndoActionType ActionType); + void SaveOriginalStateIfNeeded(AActor* Actor); + [[nodiscard]] TArray> CaptureMaterials(AActor* Actor) const; + void RestoreMaterials(AActor* Actor, const TArray>& Materials); + + // --- Smooth animation --- + + bool bAnimatingLocation = false; + bool bAnimatingRotation = false; + inline bool IsAnimating() const { return bAnimatingRotation || bAnimatingLocation; } + FVector AnimStartLocation = FVector::ZeroVector; + FVector AnimTargetLocation = FVector::ZeroVector; + FRotator AnimStartRotation = FRotator::ZeroRotator; + FRotator AnimTargetRotation = FRotator::ZeroRotator; + float AnimElapsed = 0.f; + + UPROPERTY() + AActor* AnimatingActor = nullptr; + + void StartSmoothMove(AActor* Actor, const FVector& TargetLocation); + void StartSmoothRotate(AActor* Actor, const FRotator& TargetRotation); + void StartSmoothMoveAndRotate(AActor* Actor, const FVector& TargetLocation, const FRotator& TargetRotation); + void FinishAnimationImmediately(); + + // --- Outline PP (created at runtime if bCreateOutlineComponent) --- + UPROPERTY() + UPostProcessComponent* OutlinePostProcess = nullptr; + + // --- Selection helpers --- + [[nodiscard]] bool GetSelectionCameraRay(FVector& OutStart, FVector& OutEnd) const; + + /** Single source of truth for selection state changes. Both DoSelect and SetSelectedActor + * funnel through this. Handles Movable enforcement (only when bEnforceMovable=true, + * set by DoSelect's raycast path), CustomDepth highlight toggle, and OnSelectionChanged + * broadcast. No-op if NewActor matches current SelectedActor. */ + void SetSelectedActorInternal(AActor* NewActor, bool bRequireMovable); + + // --- Registration bookkeeping --- + // Names of actions we registered so we can cleanly unregister on EndPlay + // without stepping on actions registered by other executors. + TArray RegisteredActionNames; +}; diff --git a/Samples/PixelStreaming2/WebServers/get_ps_servers.bat b/Samples/PixelStreaming2/WebServers/get_ps_servers.bat new file mode 100644 index 0000000..6c0874e --- /dev/null +++ b/Samples/PixelStreaming2/WebServers/get_ps_servers.bat @@ -0,0 +1,215 @@ +@Rem Copyright Epic Games, Inc. All Rights Reserved. + +@echo off + +@Rem Set script location as working directory for commands. +pushd "%~dp0" + +@Rem Turned on delayed expansion so we have variables evaluated in nested scope +setlocal EnableDelayedExpansion + +@Rem Unset all our variables as these persist between cmd sessions if cmd not closed. +SET "PSInfraOrg=" +SET "PSInfraRepo=" +SET "PSInfraTagOrBranch=" +SET "ReleaseVersion=" +SET "ReleaseUrl=" +SET "IsTag=" +SET "RefType=" +SET "Url=" +SET "DownloadVersion=" +SET "FlagPassed=" + +:arg_loop_start +SET ARG=%1 +if DEFINED ARG ( + if "%ARG%"=="/h" ( + goto print_help + ) + if "%ARG%"=="/v" ( + SET UEVersion=%2 + SET FlagPassed=1 + SHIFT + ) + if "%ARG%"=="/b" ( + SET PSInfraTagOrBranch=%2 + SET IsTag=0 + SET FlagPassed=1 + SHIFT + ) + if "%ARG%"=="/t" ( + SET PSInfraTagOrBranch=%2 + SET IsTag=1 + SET FlagPassed=1 + SHIFT + ) + if "%ARG%"=="/r" ( + SET "ReleaseVersion=%2" + SET "ReleaseUrl=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/!ReleaseVersion!/!ReleaseVersion!.zip" + SET IsTag=0 + SET FlagPassed=1 + SHIFT + ) + SHIFT + goto arg_loop_start +) + +@Rem Name and version of ps-infra that we are downloading +SET PSInfraOrg=EpicGamesExt +SET PSInfraRepo=PixelStreamingInfrastructure + +@Rem If a UE version is supplied set the right branch or tag to fetch for that version of UE +if DEFINED UEVersion ( + if "%UEVersion%"=="4.26" ( + SET PSInfraTagOrBranch=UE4.26 + SET IsTag=0 + ) + if "%UEVersion%"=="4.27" ( + SET PSInfraTagOrBranch=UE4.27 + SET IsTag=0 + ) + if "%UEVersion%"=="5.0" ( + SET PSInfraTagOrBranch=UE5.0 + SET IsTag=0 + ) + if "%UEVersion%"=="5.1" ( + SET PSInfraTagOrBranch=UE5.1 + SET IsTag=0 + ) + if "%UEVersion%"=="5.2" ( + SET PSInfraTagOrBranch=UE5.2 + SET IsTag=0 + ) + if "%UEVersion%"=="5.3" ( + SET PSInfraTagOrBranch=UE5.3 + SET IsTag=0 + ) + if "%UEVersion%"=="5.4" ( + SET PSInfraTagOrBranch=UE5.4 + SET IsTag=0 + ) + if "%UEVersion%"=="5.5" ( + SET PSInfraTagOrBranch=UE5.5 + SET IsTag=0 + ) + if "%UEVersion%"=="5.6" ( + SET PSInfraTagOrBranch=UE5.6 + SET IsTag=0 + ) + if "%UEVersion%"=="5.7" ( + SET PSInfraTagOrBranch=UE5.7 + SET IsTag=0 + ) +) + +@Rem If no arguments select a specific version, fetch the appropriate default +if NOT DEFINED PSInfraTagOrBranch ( + SET PSInfraTagOrBranch=UE5.7 + SET IsTag=0 +) +echo Tag or branch: !PSInfraTagOrBranch! + +@Rem Whether the named reference is a tag or a branch affects the Url we fetch it on +if %IsTag%==1 ( + SET RefType=tags +) else ( + SET RefType=heads +) + +@Rem We have a branch, no user-specified release, then check repo for the presence of a RELEASE_VERSION file in the current branch +if %IsTag%==0 ( + if NOT DEFINED ReleaseUrl ( + @Rem We don't want to auto-set the release version if the user passed an explicit flag. + if NOT DEFINED FlagPassed ( + FOR /F "tokens=* USEBACKQ" %%F IN (`curl -s -f -L https://raw.githubusercontent.com/EpicGamesExt/PixelStreamingInfrastructure/%PSInfraTagOrBranch%/RELEASE_VERSION`) DO ( + SET "ReleaseVersion=!PSInfraTagOrBranch!-%%F" + SET "ReleaseUrl=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/!ReleaseVersion!/!ReleaseVersion!.zip" + ) + ) + ) +) + +@Rem Set our DownloadVersion here as we use this to check the contents of our DOWNLOAD_VERSION file shortly. +SET "DownloadVersion=%PSInfraTagOrBranch%" +if DEFINED ReleaseVersion ( + SET "DownloadVersion=!ReleaseVersion!" + echo Release: !ReleaseVersion! +) + +@Rem Check for the existence of a DOWNLOAD_VERSION file and if found, check its contents against our %DownloadVersion% +if exist DOWNLOAD_VERSION ( + + @Rem Read DOWNLOAD_VERSION file into variable + FOR /F "delims=" %%F IN ( DOWNLOAD_VERSION ) DO ( + SET "PreviousDownloadVersion=%%F" + @Rem Remove whitespace + SET "PreviousDownloadVersion=!PreviousDownloadVersion: =!" + ) + + if !DownloadVersion! == !PreviousDownloadVersion! ( + echo Downloaded version ^(!DownloadVersion!^) of PS infra matches release version ^(!PreviousDownloadVersion!^)...skipping install. + goto :EOF + ) else ( + echo There is a newer released version ^(!DownloadVersion!^) - had ^(!PreviousDownloadVersion!^), downloading... + ) +) else ( + echo DOWNLOAD_VERSION file not found...beginning ps-infra download. +) + +@Rem By default set the download url to the .zip of the branch +SET Url=https://github.com/%PSInfraOrg%/%PSInfraRepo%/archive/refs/%RefType%/%PSInfraTagOrBranch%.zip + +@Rem If we have a ReleaseUrl then set it to our download url +if DEFINED ReleaseUrl ( + SET Url=!ReleaseUrl! +) + +@Rem Download ps-infra and follow redirects. +echo Attempting downloading Pixel Streaming infrastructure from: !Url! +curl -L !Url! > ps-infra.zip + +@Rem Unarchive the .zip +tar -xmf ps-infra.zip || echo bad archive, contents: && type ps-infra.zip && exit 0 + +@Rem Remove old infra +if exist Frontend\ ( rmdir /s /q Frontend ) +if exist Matchmaker\ ( rmdir /s /q Matchmaker ) +if exist SignallingWebserver\ ( rmdir /s /q SignallingWebserver ) +if exist SFU\ ( rmdir /s /q SFU ) + +@Rem Rename the extracted, versioned, directory +for /d %%i in ("PixelStreamingInfrastructure-*") do ( + for /d %%j in ("%%i/*") do ( + echo "%%i\%%j" + move "%%i\%%j" . + ) + for %%j in ("%%i/*") do ( + echo "%%i\%%j" + move "%%i\%%j" . + ) + + echo "%%i" + rmdir /s /q "%%i" +) + +@Rem Delete the downloaded zip +del ps-infra.zip + +@Rem Create a DOWNLOAD_VERSION file, which we use as a comparison file to check if we should auto upgrade when these scripts are run again +echo %DownloadVersion%> DOWNLOAD_VERSION +goto :EOF + +:print_help +echo. +echo Tool for fetching PixelStreaming Infrastructure. If no flags are set specifying a version to fetch, +echo the recommended version will be chosen as a default. +echo. +echo Usage: +echo %~n0%~x0 [^/h] [^/v ^] [^/b ^] [^/t ^] [^/r ^] +echo Where: +echo /v Specify a version of Unreal Engine to download the recommended release for +echo /b Specify a specific branch for the tool to download from repo +echo /t Specify a specific tag for the tool to download from repo +echo /r Specify a specific release url path e.g. https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/${RELEASE_HERE}.zip +echo /h Display this help message +goto :EOF diff --git a/Samples/PixelStreaming2/WebServers/get_ps_servers.sh b/Samples/PixelStreaming2/WebServers/get_ps_servers.sh new file mode 100644 index 0000000..8f5f828 --- /dev/null +++ b/Samples/PixelStreaming2/WebServers/get_ps_servers.sh @@ -0,0 +1,199 @@ +#!/bin/bash +# Copyright Epic Games, Inc. All Rights Reserved. + +BASH_LOCATION="$(cd -P -- "$(dirname -- "$0")" && pwd -P)" + +pushd "${BASH_LOCATION}" > /dev/null + +print_help() { + echo " + Tool for fetching PixelStreaming Infrastructure. If no flags are set specifying a version to fetch, + the recommended version will be chosen as a default. + + Usage: + ${0} [-h] [-v ] [-b ] [-t ] + Where: + -v Specify a version of Unreal Engine to download the recommended + release for + -b Specify a specific branch for the tool to download from repo + -t Specify a specific tag for the tool to download from repo + -r Specify a specific release url path e.g. https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download//.zip + -h Display this help message +" + exit 1 +} + +# Set all default variables (e.g. # Name and version of ps-infra that we are downloading) +PSInfraOrg=EpicGamesExt +PSInfraRepo=PixelStreamingInfrastructure +PSInfraTagOrBranch=UE5.7 +RefType=heads +IsTag=0 +ReleaseUrlBase=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download +# Unset any variables that don't have defaults that we use that may have persisted between bash terminals. +unset Url +unset DownloadVersion +unset FlagPassed +unset ReleaseVersion +unset ReleaseUrl + +while (($#)) ; do + case "$1" in + -h ) print_help;; + -v ) UEVersion="$2"; FlagPassed=1; shift 2;; + -b ) PSInfraTagOrBranch="$2"; FlagPassed=1; IsTag=0; shift 2;; + -t ) PSInfraTagOrBranch="$2"; FlagPassed=1; IsTag=1; shift 2;; + -r ) ReleaseVersion="$2"; FlagPassed=1; IsTag=0; ReleaseUrl=$ReleaseUrlBase/$ReleaseVersion/$ReleaseVersion.tar.gz; shift 2;; + * ) echo "Unknown command: $1"; shift;; + esac + done + +# If a UE version is supplied set the right branch or tag to fetch for that version of UE +if [ ! -z "$UEVersion" ] +then + if [ "$UEVersion" = "4.26" ] + then + PSInfraTagOrBranch=UE4.26 + IsTag=0 + fi + if [ "$UEVersion" = "4.27" ] + then + PSInfraTagOrBranch=UE4.27 + IsTag=0 + fi + if [ "$UEVersion" = "5.0" ] + then + PSInfraTagOrBranch=UE5.0 + IsTag=0 + fi + if [ "$UEVersion" = "5.1" ] + then + PSInfraTagOrBranch=UE5.1 + IsTag=0 + fi + if [ "$UEVersion" = "5.2" ] + then + PSInfraTagOrBranch=UE5.2 + IsTag=0 + fi + if [ "$UEVersion" = "5.3" ] + then + PSInfraTagOrBranch=UE5.3 + IsTag=0 + fi + if [ "$UEVersion" = "5.4" ] + then + PSInfraTagOrBranch=UE5.4 + IsTag=0 + fi + if [ "$UEVersion" = "5.5" ] + then + PSInfraTagOrBranch=UE5.5 + IsTag=0 + fi + if [ "$UEVersion" = "5.6" ] + then + PSInfraTagOrBranch=UE5.6 + IsTag=0 + fi + if [ "$UEVersion" = "5.7" ] + then + PSInfraTagOrBranch=UE5.7 + IsTag=0 + fi +fi + +# If no arguments select a specific version, fetch the appropriate default +if [ -z "$PSInfraTagOrBranch" ] +then + PSInfraTagOrBranch=UE5.7 + IsTag=0 +fi +echo "Tag or branch: $PSInfraTagOrBranch" + +# Whether the named reference is a tag or a branch affects the Url we fetch it on +if [ "$IsTag" -eq 1 ] +then + RefType=tags +else + RefType=heads +fi + +# We have a branch, no user-specified release, then check repo for the presence of a RELEASE_VERSION file in the current branch. +if [ "$IsTag" -eq 0 ] && [ -z "$ReleaseUrl" ] && [ -z "$FlagPassed" ] +then + RelUrl=https://raw.githubusercontent.com/EpicGamesExt/PixelStreamingInfrastructure/$PSInfraTagOrBranch/RELEASE_VERSION + if curl --output /dev/null --silent -r 0-0 --fail "$RelUrl"; then + ReleaseVersion="$PSInfraTagOrBranch-$(curl -L -s $RelUrl)" + ReleaseUrl=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/$ReleaseVersion/$ReleaseVersion.tar.gz + echo "Valid RELEASE_VERSION file found in Github repo at $RelUrl" + else + echo "RELEASE_VERSION file does not exist at: $RelUrl" + fi +else + echo "Skipping downloading RELEASE_VERSION file." +fi + +#Set our DownloadVersion here as we use this to check the contents of our DOWNLOAD_VERSION file shortly. +DownloadVersion="$PSInfraTagOrBranch" +if [ ! -z "$ReleaseVersion" ] +then + DownloadVersion="$ReleaseVersion" + echo "Release: $ReleaseVersion" +fi + +#Rem Check for the existence of a DOWNLOAD_VERSION file and if found, check its contents against our $DownloadVersion +if test -f DOWNLOAD_VERSION; +then + PREVIOUS_DOWNLOAD_VERSION=$(cat DOWNLOAD_VERSION) + if [ "$DownloadVersion" = "$PREVIOUS_DOWNLOAD_VERSION" ] + then + echo "Downloaded version ($DownloadVersion) of PS infra matches release version ($PREVIOUS_DOWNLOAD_VERSION)...skipping install." + exit 0 + else + echo "There is a newer released version ($DownloadVersion) - had ($PREVIOUS_DOWNLOAD_VERSION), downloading..." + #Remove old infra + rm -rf Frontend + rm -rf Matchmaker + rm -rf SignallingWebServer + rm -rf SFU + fi +else + echo "DOWNLOAD_VERSION file not found..." +fi + +# Pre-download - Set the download url to the .zip of the branch +Url=https://github.com/$PSInfraOrg/$PSInfraRepo/archive/refs/$RefType/$PSInfraTagOrBranch.tar.gz + +#Check if ReleaseUrl is valid by CURLing with fail fast and if success then set it to our download url +if [ ! -z "$ReleaseUrl" ] +then + echo "Checking if release url is valid, url: $ReleaseUrl" + if curl --output /dev/null --silent -r 0-0 --fail "$ReleaseUrl"; then + echo "Valid release url at: $ReleaseUrl" + Url=$ReleaseUrl + else + echo "Invalid Github release url: $ReleaseUrl" + exit 1 + fi +fi + +# Download - download ps-infra and follow redirects. +echo "Beginning ps-infra download from: $Url" +curl -L $Url > ps-infra.tar.gz + +# Unarchive the .tar +tar -xmf ps-infra.tar.gz || $(echo "bad archive, contents:" && head --lines=20 ps-infra.tar.gz && exit 0) + +# Move the server folders into the current directory (WebServers) and delete the original directory +mv PixelStreamingInfrastructure-*/* . +# Copy any files and folders beginning with dot (ignored by * glob) and discard errors regarding to not being able to move "." and ".." +mv PixelStreamingInfrastructure-*/.* . 2>/dev/null || : +rm -rf PixelStreamingInfrastructure-* + +# Delete the downloaded tar +rm ps-infra.tar.gz + +#Create a DOWNLOAD_VERSION file, which we use as a comparison file to check if we should auto upgrade when these scripts are run again +echo "$DownloadVersion" >| DOWNLOAD_VERSION +exit 0 diff --git a/Source/PSC_SharkTable.Target.cs b/Source/PSC_SharkTable.Target.cs new file mode 100644 index 0000000..f19b642 --- /dev/null +++ b/Source/PSC_SharkTable.Target.cs @@ -0,0 +1,12 @@ +using UnrealBuildTool; + +public class PSC_SharkTableTarget : TargetRules +{ + public PSC_SharkTableTarget(TargetInfo Target) : base(Target) + { + DefaultBuildSettings = BuildSettingsVersion.Latest; + IncludeOrderVersion = EngineIncludeOrderVersion.Latest; + Type = TargetType.Game; + ExtraModuleNames.Add("PSC_SharkTable"); + } +} diff --git a/Source/PSC_SharkTable/PSC_SharkTable.Build.cs b/Source/PSC_SharkTable/PSC_SharkTable.Build.cs new file mode 100644 index 0000000..550d99f --- /dev/null +++ b/Source/PSC_SharkTable/PSC_SharkTable.Build.cs @@ -0,0 +1,10 @@ +using UnrealBuildTool; + +public class PSC_SharkTable : ModuleRules +{ + public PSC_SharkTable(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; + PrivateDependencyModuleNames.Add("Core"); + } +} diff --git a/Source/PSC_SharkTable/PSC_SharkTable.cpp b/Source/PSC_SharkTable/PSC_SharkTable.cpp new file mode 100644 index 0000000..6d49aa9 --- /dev/null +++ b/Source/PSC_SharkTable/PSC_SharkTable.cpp @@ -0,0 +1,4 @@ +#include "CoreTypes.h" +#include "Modules/ModuleManager.h" + +IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, PSC_SharkTable, "PSC_SharkTable"); diff --git a/Source/PSC_SharkTableEditor.Target.cs b/Source/PSC_SharkTableEditor.Target.cs new file mode 100644 index 0000000..359a383 --- /dev/null +++ b/Source/PSC_SharkTableEditor.Target.cs @@ -0,0 +1,12 @@ +using UnrealBuildTool; + +public class PSC_SharkTableEditorTarget : TargetRules +{ + public PSC_SharkTableEditorTarget(TargetInfo Target) : base(Target) + { + DefaultBuildSettings = BuildSettingsVersion.Latest; + IncludeOrderVersion = EngineIncludeOrderVersion.Latest; + Type = TargetType.Editor; + ExtraModuleNames.Add("PSC_SharkTable"); + } +} diff --git a/UE5_BP_to_CPP_Packaging_Guide.md b/UE5_BP_to_CPP_Packaging_Guide.md new file mode 100644 index 0000000..92ca2e0 --- /dev/null +++ b/UE5_BP_to_CPP_Packaging_Guide.md @@ -0,0 +1,242 @@ +# UE5 Blueprint-to-C++ Project Conversion & Packaging Fix Guide + +## Problem + +When you copy an Unreal Engine 5 Blueprint-only project to a new machine, "Package Project" does nothing -- no error in the UI, no output. The log shows: + +``` +LogProjectPackagingSettings: UProjectPackagingSettings FindBestTargetInfo for '' resulted in null FTargetInfo*. +Listing targets that were searched: + End of target list. +``` + +This happens because: +1. The project has no `Source/` directory (BP-only projects auto-generate source in `Intermediate/Source/` during packaging) +2. Stale configs from the original machine override local settings +3. The editor can't discover build targets without compiled `.target` metadata files + +## Step-by-Step Fix + +### 1. Add Modules to .uproject + +The `.uproject` file MUST declare a Modules section. Without it, the editor won't look for `.target` files in `Binaries/` and the target list will always be empty. + +Add this before the `"Plugins"` array: + +```json +{ + "FileVersion": 3, + "EngineAssociation": "5.7", + "Modules": [ + { + "Name": "YourProjectName", + "Type": "Runtime", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + ... + ] +} +``` + +> **Critical:** Without this section, even if `.target` files exist in `Binaries/`, the editor's `FindBestTargetInfo` will return an empty target list. + +### 2. Create Source/ Directory + +Create the following structure at the project root: + +``` +Source/ + YourProjectName.Target.cs (Game target) + YourProjectNameEditor.Target.cs (Editor target) + YourProjectName/ + YourProjectName.Build.cs (Module build rules) + YourProjectName.cpp (Module implementation) +``` + +**YourProjectName.Target.cs:** +```csharp +using UnrealBuildTool; + +public class YourProjectNameTarget : TargetRules +{ + public YourProjectNameTarget(TargetInfo Target) : base(Target) + { + DefaultBuildSettings = BuildSettingsVersion.Latest; + IncludeOrderVersion = EngineIncludeOrderVersion.Latest; + Type = TargetType.Game; + ExtraModuleNames.Add("YourProjectName"); + } +} +``` + +**YourProjectNameEditor.Target.cs:** +```csharp +using UnrealBuildTool; + +public class YourProjectNameEditorTarget : TargetRules +{ + public YourProjectNameEditorTarget(TargetInfo Target) : base(Target) + { + DefaultBuildSettings = BuildSettingsVersion.Latest; + IncludeOrderVersion = EngineIncludeOrderVersion.Latest; + Type = TargetType.Editor; + ExtraModuleNames.Add("YourProjectName"); + } +} +``` + +**YourProjectName/YourProjectName.Build.cs:** +```csharp +using UnrealBuildTool; + +public class YourProjectName : ModuleRules +{ + public YourProjectName(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; + PrivateDependencyModuleNames.Add("Core"); + } +} +``` + +**YourProjectName/YourProjectName.cpp:** +```cpp +#include "CoreTypes.h" +#include "Modules/ModuleManager.h" + +IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, YourProjectName, "YourProjectName"); +``` + +### 3. Build Both Targets from Command Line + +Close the editor, then build from terminal: + +```batch +:: Build Editor target (needed to open the project) +"C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat" YourProjectNameEditor Win64 Development "C:\Path\To\YourProject.uproject" -WaitMutex + +:: Build Game target (needed for packaging -- THIS IS THE ONE PEOPLE FORGET) +"C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat" YourProjectName Win64 Shipping "C:\Path\To\YourProject.uproject" -WaitMutex +``` + +> **Critical:** You must build BOTH targets. The Editor build creates `Simulore_TirsovaEditor.target`, and the Game build creates `Simulore_Tirsova-Win64-Shipping.target`. The packaging system only looks for Game-type targets. If you only build the Editor target, the target list will still be empty. + +### 4. Clean Stale Data from Copied Project + +When a project is copied from another machine, these directories contain stale/incompatible data: + +| Directory | Safe to delete | Regenerated by | +|---|---|---| +| `Intermediate/` | Yes | Editor on startup | +| `Binaries/` | Yes | Build step above | +| `DerivedDataCache/` | Yes | Editor on startup | +| `Saved/StagedBuilds/` | Yes | Packaging process | +| `Saved/Cooked/` | Yes | Cooking process | +| `Saved/Crashes/` | Yes | Not regenerated (just old logs) | +| `Saved/Temp/` | Yes | Editor on startup | + +**Never delete:** `Config/`, `Content/`, `Source/`, `.uproject`, `Saved/Config/` (fix it instead) + +### 5. Fix Hardcoded Paths (THE SILENT KILLER) + +Copied projects contain paths from the original machine in multiple locations. Check ALL of these: + +#### 5a. Config/DefaultGame.ini + +```ini +[/Script/UnrealEd.ProjectPackagingSettings] +BuildTarget=YourProjectName +StagingDirectory=(Path="") ; <-- was hardcoded to other user's Desktop +``` + +#### 5b. Saved/Config/WindowsEditor/Game.ini (HIGHEST PRIORITY -- overrides DefaultGame.ini!) + +This is the one that actually matters for packaging. The editor reads this file, not DefaultGame.ini: + +```ini +[/Script/DeveloperToolSettings.PlatformsMenuSettings] +StagingDirectory=(Path="") ; <-- fix this +CookBuildTarget=YourProjectName ; <-- was empty +PackageBuildTarget=YourProjectName ; <-- was empty (ROOT CAUSE of "no targets") +``` + +> **This was the hardest bug to find.** `DefaultGame.ini` has `BuildTarget=YourProjectName` but the editor actually reads `PackageBuildTarget` from `Saved/Config/WindowsEditor/Game.ini`, which has higher priority. If this is empty, the editor passes an empty string to `FindBestTargetInfo`. + +#### 5c. Saved/Config/WindowsEditor/EditorPerProjectUserSettings.ini + +Search for and fix any paths referencing other machines: +```ini +SwarmIntermediateFolder= ; <-- was C:/Users/OtherUser/... +LastExecutedLaunchDevice= ; <-- was Windows@OTHER-PC +LastExecutedLaunchName= ; <-- was OTHER-PC +``` + +#### Quick way to find all stale paths: +```bash +grep -rn "C:/Users/" Config/ Saved/Config/ --include="*.ini" | grep -v "prime" +``` +Replace `prime` with the current username. + +## UE5 Config Priority System + +Understanding this prevents hours of debugging: + +| Priority | Source | Hex | +|---|---|---| +| 1 (lowest) | Constructor defaults | 0x00 | +| 2 | Scalability (BaseScalability.ini tiers) | 0x01 | +| 3 | GameUserSettings (in-game quality menu) | 0x02 | +| 4 | ProjectSettings (`[/Script/Engine.RendererSettings]`) | 0x03 | +| 5 | **`[SystemSettings]` and `[ConsoleVariables]` in DefaultEngine.ini** | 0x04 | +| 6 | Device Profiles | 0x05 | +| 7 | Standalone ConsoleVariables.ini (NOT loaded in Shipping!) | 0x06 | +| 8 | Command line | 0x07 | +| 9 | C++ code Set() | 0x08 | +| 10 (highest) | In-game console (~) | 0x09 | + +### Where to put rendering overrides: + +- **`[/Script/Engine.RendererSettings]`** -- Project-level feature toggles (Lumen on/off, RayTracing on/off, VSM on/off). These affect shader compilation. +- **`[SystemSettings]`** -- Runtime quality overrides that must survive scalability and device profile changes. Use this for forced quality settings in packaged builds. +- **`[ConsoleVariables]`** -- Same priority as SystemSettings. Good for non-rendering cvars (shader pipeline cache, etc). +- **`DefaultGameUserSettings.ini` `[ScalabilityGroups]`** -- Default scalability levels for first-time users. Gets overridden by hardware auto-detect on first run. + +### Scalability override gotcha: + +If you set `sg.ShadowQuality=3` in `[ScalabilityGroups]` AND `r.Shadow.MaxResolution=512` in `[SystemSettings]`, the scalability system will apply its shadow preset values first, then `[SystemSettings]` overrides specific cvars. Your individual settings win because they have higher priority. + +**However:** Device Profiles (priority 0x05) can override `[SystemSettings]` (priority 0x04). If you see cvars being overridden, check `Saved/Config/WindowsEditor/` for device profile configs. The WindowsEditor device profile commonly overrides shadow and scalability settings. + +## Packaging from Command Line (Bypass Editor UI) + +If the editor UI still refuses to package, use RunUAT directly: + +```batch +"C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\RunUAT.bat" BuildCookRun ^ + -project="C:\Path\To\YourProject.uproject" ^ + -platform=Win64 ^ + -clientconfig=Shipping ^ + -cook -build -stage -package -pak ^ + -prereqs -compressed ^ + -target=YourProjectName ^ + -unrealexe="C:\Program Files\Epic Games\UE_5.7\Engine\Binaries\Win64\UnrealEditor-Cmd.exe" +``` + +This bypasses the editor's target discovery entirely and invokes UnrealBuildTool + cooker directly. + +## Checklist Summary + +When transferring a BP-only UE5 project to a new machine: + +- [ ] Add `"Modules"` section to `.uproject` +- [ ] Create `Source/` with Target.cs, Build.cs, and .cpp files +- [ ] Build Editor target from command line +- [ ] Build Game target (Shipping) from command line +- [ ] Delete `Intermediate/`, `Binaries/` (before building), `DerivedDataCache/`, `Saved/StagedBuilds/`, `Saved/Cooked/` +- [ ] Fix `Config/DefaultGame.ini` -- `BuildTarget`, `StagingDirectory` +- [ ] Fix `Saved/Config/WindowsEditor/Game.ini` -- `PackageBuildTarget`, `CookBuildTarget`, `StagingDirectory` +- [ ] Fix `Saved/Config/WindowsEditor/EditorPerProjectUserSettings.ini` -- remove other machine's paths +- [ ] Grep all .ini files for hardcoded paths from other machines +- [ ] Open editor, verify Platforms > Windows > Package Project works