adittion of power calculation for shot distance and power UI

This commit is contained in:
DjordjeIlic 2026-06-08 16:59:05 +02:00
parent a15d714566
commit 3b960967c4
54 changed files with 4164 additions and 26 deletions

19
.vsconfig Normal file
View file

@ -0,0 +1,19 @@
{
"version": "1.0",
"components": [
"Component.Unreal.Debugger",
"Component.Unreal.Ide",
"Microsoft.Net.Component.4.6.2.TargetingPack",
"Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL",
"Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64",
"Microsoft.VisualStudio.Component.VC.14.44.17.14.ATL",
"Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64",
"Microsoft.VisualStudio.Component.VC.Llvm.Clang",
"Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
"Microsoft.VisualStudio.Component.Windows11SDK.22621",
"Microsoft.VisualStudio.Workload.CoreEditor",
"Microsoft.VisualStudio.Workload.ManagedDesktop",
"Microsoft.VisualStudio.Workload.NativeDesktop",
"Microsoft.VisualStudio.Workload.NativeGame"
]
}

View file

@ -1,5 +1,9 @@
[/Script/UnrealEd.ProjectPackagingSettings]
BuildTarget=PSC_SharkTable
StagingDirectory=(Path="")
[/Script/CommonUI.CommonUISettings]
CommonButtonAcceptKeyHandling=TriggerClick
@ -11,3 +15,6 @@ EditorUseRemoteSignallingServer=True
ConnectionURL="ws://10.10.100.204:8888"
LogStats=True
[/Script/BasemashVoiceCommander.VoiceCommanderSettings]
GroqApiKey=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

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
Content/Blueprints/BP_PowerSelection.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/Blueprints/BP_SpinUI.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/Blueprints/BP_WhiteTravel.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/Levels/MainLevel.umap (Stored with Git LFS)

Binary file not shown.

BIN
Content/Materials/Black.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/Materials/MP_NDI_FEED.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/Materials/MP_NDI_FEED_Video.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/Materials/M_NDIStream.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/Materials/NewMediaPlayer.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/Materials/NewMediaPlayer_Video.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/Materials/NewNDIMediaSource.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/Structs/CF_WhiteTravelAnglePower.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/Structs/CF_WhiteTravelPower.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/Pictures/CueBal.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/Pictures/Curve_SpinMapping.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/Pictures/Red_circle_frame_transparent_svg.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/Pictures/minus.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/Pictures/plus.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/WBP_Automatic.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/UI/WBP_LineLabel.uasset (Stored with Git LFS)

Binary file not shown.

BIN
Content/UI/WBP_PowerSelection.uasset (Stored with Git LFS) Normal file

Binary file not shown.

BIN
Content/UI/WBP_Spin.uasset (Stored with Git LFS) Normal file

Binary file not shown.

View file

@ -3,6 +3,13 @@
"EngineAssociation": "5.7",
"Category": "",
"Description": "",
"Modules": [
{
"Name": "PSC_SharkTable",
"Type": "Runtime",
"LoadingPhase": "Default"
}
],
"Plugins": [
{
"Name": "ModelingToolsEditorMode",

View file

@ -0,0 +1,35 @@
{
"FileVersion": 3,
"Version": 2,
"VersionName": "0.1.1",
"FriendlyName": "Basemash Voice Commander",
"Description": "Voice-controlled object manipulation plugin. Local mic or Pixel Streaming 2 browser mic as audio source. Groq STT + LLM command parsing. Fully modular via Blueprint-registered actions.",
"Category": "Audio",
"CreatedBy": "Basemash",
"CreatedByURL": "",
"DocsURL": "",
"MarketplaceURL": "",
"SupportURL": "",
"CanContainContent": true,
"IsBetaVersion": true,
"IsExperimentalVersion": false,
"Installed": false,
"Modules": [
{
"Name": "BasemashVoiceCommander",
"Type": "Runtime",
"LoadingPhase": "Default"
}
],
"Plugins": [
{
"Name": "AudioCapture",
"Enabled": true
},
{
"Name": "PixelStreaming2",
"Enabled": true,
"Optional": true
}
]
}

View file

@ -0,0 +1,8 @@
[FilterPlugin]
; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and
; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively.
;
; Examples:
; /README.txt
; /Extras/...
; /Binaries/ThirdParty/*.dll

View file

@ -0,0 +1,33 @@
// Copyright (c) Basemash. All Rights Reserved.
using UnrealBuildTool;
public class BasemashVoiceCommander : ModuleRules
{
public BasemashVoiceCommander(ReadOnlyTargetRules Target) : base(Target)
{
PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
PrecompileForTargets = PrecompileTargetsType.Any;
PublicDependencyModuleNames.AddRange(new string[]
{
"Core",
"CoreUObject",
"Engine",
"HTTP",
"Json",
"JsonUtilities",
"AudioCapture",
"AudioCaptureCore",
"AudioMixer",
"PixelStreaming2",
"PixelStreaming2Core",
"DeveloperSettings"
});
PrivateDependencyModuleNames.AddRange(new string[]
{
"Projects"
});
}
}

View file

@ -0,0 +1,22 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "BasemashVoiceCommanderLog.h"
#include "Modules/ModuleManager.h"
DEFINE_LOG_CATEGORY(LogBasemashVoiceCommander);
class FBasemashVoiceCommanderModule : public IModuleInterface
{
public:
virtual void StartupModule() override
{
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("BasemashVoiceCommander module started."));
}
virtual void ShutdownModule() override
{
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("BasemashVoiceCommander module shut down."));
}
};
IMPLEMENT_MODULE(FBasemashVoiceCommanderModule, BasemashVoiceCommander)

View file

@ -0,0 +1,92 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "PSVoiceAudioConsumer.h"
#include "Async/Async.h"
#include "BasemashVoiceCommanderLog.h"
#include "Engine/Engine.h"
void UPSVoiceAudioConsumer::ConsumeRawPCM(const int16_t* AudioData, int InSampleRate, size_t NChannels, size_t NFrames)
{
FScopeLock Lock(&CriticalSection);
// First-data flag lets us verify audio actually flows from the new peer after switch.
// Kept inside the lock so the check+set is atomic versus ResetFirstDataFlag on the game thread.
const bool bFireFirstData = !bGotFirstData;
if (bFireFirstData)
{
bGotFirstData = true;
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("PSVoiceAudioConsumer: First audio data received! %d Hz, %d ch, %d frames"),
InSampleRate,
static_cast<int32>(NChannels),
static_cast<int32>(NFrames));
AsyncTask(ENamedThreads::GameThread,
[]()
{
if (GEngine != nullptr)
GEngine->AddOnScreenDebugMessage(80, 10.f, FColor::Green, TEXT("PS2: Audio data flowing!"));
});
}
if (!bIsBuffering)
{
return;
}
SampleRate = InSampleRate;
NumChannels = static_cast<int32>(NChannels);
// Convert signed 16-bit PCM to normalized float [-1, 1] for downstream voice processing.
const int32 TotalSamples = static_cast<int32>(NFrames * NChannels);
Buffer.Reserve(Buffer.Num() + TotalSamples);
for (int32 i = 0; i < TotalSamples; i++)
{
Buffer.Add(static_cast<float>(AudioData[i]) / 32767.0f);
}
}
void UPSVoiceAudioConsumer::OnAudioConsumerAdded()
{
bConnected = true;
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("PSVoiceAudioConsumer: Registered with Pixel Streaming audio sink."));
}
void UPSVoiceAudioConsumer::OnAudioConsumerRemoved()
{
// This runs on a PS2 worker thread. Reset flags immediately so downstream observers
// reading bConnected via FThreadSafeBool see the correct state right away.
bConnected = false;
bGotFirstData = false;
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("PSVoiceAudioConsumer: Removed from Pixel Streaming audio sink."));
// FSimpleDelegate is not thread-safe — the game-thread owner may Unbind() concurrently.
// Hop to the game thread before firing so Unbind/ExecuteIfBound can never race.
TWeakObjectPtr<UPSVoiceAudioConsumer> WeakThis(this);
AsyncTask(ENamedThreads::GameThread,
[WeakThis]()
{
if (UPSVoiceAudioConsumer* StrongThis = WeakThis.Get())
{
StrongThis->OnRemovedDelegate.ExecuteIfBound();
}
});
}
void UPSVoiceAudioConsumer::StartBuffering()
{
FScopeLock Lock(&CriticalSection);
Buffer.Empty();
bIsBuffering = true;
}
void UPSVoiceAudioConsumer::StopBuffering(TArray<float>& OutPCMData, int32& OutSampleRate, int32& OutNumChannels)
{
FScopeLock Lock(&CriticalSection);
bIsBuffering = false;
OutPCMData = MoveTemp(Buffer);
OutSampleRate = SampleRate > 0 ? SampleRate : 48000; // Default to 48kHz (PS2 WebRTC native rate).
OutNumChannels = NumChannels > 0 ? NumChannels : 1;
Buffer.Empty();
}

View file

@ -0,0 +1,569 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "VoiceCaptureComponent.h"
#include "Async/Async.h"
#include "BasemashVoiceCommanderLog.h"
#include "Engine/GameInstance.h"
#include "Engine/World.h"
#include "GameFramework/Actor.h"
#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "PSVoiceAudioConsumer.h"
#include "TimerManager.h"
#include "VoiceCommandSubsystem.h"
// Pixel Streaming 2
#include "IPixelStreaming2AudioSink.h"
#include "IPixelStreaming2Module.h"
#include "IPixelStreaming2Streamer.h"
#include "PixelStreaming2Delegates.h"
#include "Templates/PointerVariants.h"
UVoiceCaptureComponent::UVoiceCaptureComponent() { PrimaryComponentTick.bCanEverTick = false; }
void UVoiceCaptureComponent::BeginPlay()
{
Super::BeginPlay();
if (AudioSource == EVoiceAudioSource::PixelStreaming) { InitPixelStreamingAudio(); }
else { InitLocalMic(); }
}
void UVoiceCaptureComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
// Always clear the PS2 retry timer even if we went down the LocalMic branch —
// AudioSource may have been toggled at runtime.
if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); }
// Unsubscribe from OnAudioTrackOpenNative if we subscribed.
if (PSAudioTrackOpenHandle.IsValid())
{
if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get())
{
Delegates->OnAudioTrackOpenNative.Remove(PSAudioTrackOpenHandle);
}
PSAudioTrackOpenHandle.Reset();
}
if (AudioSource == EVoiceAudioSource::PixelStreaming) { CleanupPixelStreamingAudio(); }
else { CleanupLocalMic(); }
Super::EndPlay(EndPlayReason);
}
bool UVoiceCaptureComponent::IsAudioSourceReady() const
{
if (AudioSource == EVoiceAudioSource::PixelStreaming) { return bPSAudioRegistered; }
return AudioCapture.IsStreamOpen();
}
// ============================================================================
// Local Mic
// ============================================================================
void UVoiceCaptureComponent::InitLocalMic()
{
Audio::FCaptureDeviceInfo DeviceInfo;
if (AudioCapture.GetCaptureDeviceInfo(DeviceInfo))
{
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: Mic device: %s, SampleRate: %d, Channels: %d"),
*DeviceInfo.DeviceName,
DeviceInfo.PreferredSampleRate,
DeviceInfo.InputChannels);
}
Audio::FAudioCaptureDeviceParams Params;
// Capture callback runs on the audio thread — must be cheap and thread-safe.
// Capture a weak UObject pointer; a raw `this` would UAF if the component is GC'd
// before the audio stream fully drains on shutdown.
TWeakObjectPtr<UVoiceCaptureComponent> WeakThis(this);
Audio::FOnAudioCaptureFunction OnCapture = [WeakThis](const void* InBuffer,
int32 NumFrames,
int32 NumChannels,
int32 SampleRate,
double /*StreamTime*/,
bool /*bOverflow*/)
{
UVoiceCaptureComponent* StrongThis = WeakThis.Get();
if (StrongThis == nullptr) { return; }
const float* InAudio = static_cast<const float*>(InBuffer);
FScopeLock Lock(&StrongThis->RecordingCriticalSection);
if (StrongThis->bIsRecording)
{
StrongThis->CaptureSampleRate = SampleRate;
StrongThis->CaptureNumChannels = NumChannels;
StrongThis->RecordedPCMData.Append(InAudio, NumFrames * NumChannels);
}
};
if (AudioCapture.OpenAudioCaptureStream(Params, OnCapture, 1024))
{
if (AudioCapture.StartStream())
{
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Audio capture stream started."));
}
else
{
UE_LOG(
LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: Failed to start audio capture stream."));
}
}
else
{
UE_LOG(LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: Failed to open audio capture stream."));
}
}
void UVoiceCaptureComponent::CleanupLocalMic()
{
if (AudioCapture.IsStreamOpen())
{
AudioCapture.StopStream();
AudioCapture.CloseStream();
}
}
// ============================================================================
// Pixel Streaming Audio
// ============================================================================
void UVoiceCaptureComponent::InitPixelStreamingAudio()
{
if (!IPixelStreaming2Module::IsAvailable())
{
UE_LOG(LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: PixelStreaming2 module not available!"));
return;
}
PSAudioConsumer = NewObject<UPSVoiceAudioConsumer>(this);
// When the consumer is removed (peer disconnected), restart polling for a new peer.
// Use a weak pointer to the component so we don't fire on a destroyed component.
// NOTE: OnRemovedDelegate is now guaranteed to be invoked on the game thread by
// UPSVoiceAudioConsumer::OnAudioConsumerRemoved (see M2). The explicit AsyncTask
// here is defensive belt-and-braces; it is a no-op when already on the game thread.
TWeakObjectPtr<UVoiceCaptureComponent> WeakThis(this);
PSAudioConsumer->OnRemovedDelegate.BindLambda(
[WeakThis]()
{
AsyncTask(ENamedThreads::GameThread,
[WeakThis]()
{
UVoiceCaptureComponent* StrongThis = WeakThis.Get();
if (StrongThis == nullptr) { return; }
// m2: guard against stale "removed" notifications for a sink we already
// switched away from. If we've already re-attached to a new sink
// (PSAudioSink.Pin() is non-null and we're registered), this "removed"
// event is for the old peer — skip the reset + re-arm so we don't
// stomp the newly-established registration.
if (StrongThis->bPSAudioRegistered && StrongThis->PSAudioSink.Pin().IsValid())
{
UE_LOG(LogBasemashVoiceCommander,
Verbose,
TEXT("VoiceCaptureComponent: Ignoring stale PS2 remove event; already "
"re-registered to a new sink."));
return;
}
StrongThis->bPSAudioRegistered = false;
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCaptureComponent: PS2 audio sink lost, restarting poll..."));
if (UWorld* World = StrongThis->GetWorld())
{
World->GetTimerManager().SetTimer(StrongThis->PSRetryTimerHandle,
StrongThis,
&UVoiceCaptureComponent::PSRetryRegistration,
2.0f,
true,
0.0f);
}
});
});
// When a new browser peer opens an audio track, switch the consumer over to it.
if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get())
{
PSAudioTrackOpenHandle = Delegates->OnAudioTrackOpenNative.AddLambda(
[WeakThis](FString StreamerId, FString PlayerId, bool bIsRemote)
{
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: PS2 AudioTrack opened. streamer=%s player=%s remote=%d"),
*StreamerId,
*PlayerId,
bIsRemote ? 1 : 0);
if (!bIsRemote)
{
return; // Only browser (remote) mics are of interest.
}
AsyncTask(ENamedThreads::GameThread,
[WeakThis, StreamerId, PlayerId]()
{
if (UVoiceCaptureComponent* StrongThis = WeakThis.Get())
{
StrongThis->SwitchAudioToPlayer(StreamerId, PlayerId);
}
});
});
}
// Kick off a 2-second polling cycle that looks for any connected peer with an audio sink.
if (UWorld* World = GetWorld())
{
World->GetTimerManager().SetTimer(
PSRetryTimerHandle, this, &UVoiceCaptureComponent::PSRetryRegistration, 2.0f, true, 0.0f);
}
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: PS2 audio init - polling for browser mic..."));
}
void UVoiceCaptureComponent::PSRetryRegistration()
{
// PS2 module is optional on some platforms; Get() asserts when !IsAvailable().
// If PS2 went away (or never existed), bail and shut the retry timer down for good.
if (!IPixelStreaming2Module::IsAvailable())
{
if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); }
return;
}
if (bPSAudioRegistered)
{
if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); }
return;
}
IPixelStreaming2Module& PSModule = IPixelStreaming2Module::Get();
TArray<FString> StreamerIds = PSModule.GetStreamerIds();
for (const FString& StreamerId : StreamerIds)
{
TSharedPtr<IPixelStreaming2Streamer> Streamer = PSModule.FindStreamer(StreamerId);
if (Streamer == nullptr) { continue; }
TArray<FString> Players = Streamer->GetConnectedPlayers();
for (const FString& PlayerId : Players)
{
TWeakPtr<IPixelStreaming2AudioSink> SinkWeak = Streamer->GetPeerAudioSink(PlayerId);
TSharedPtr<IPixelStreaming2AudioSink> Sink = SinkWeak.Pin();
if (Sink != nullptr)
{
SwitchAudioToPlayer(StreamerId, PlayerId);
return;
}
}
// Fallback: some backends expose an "unlistened" sink if no specific peer is ready.
{
TWeakPtr<IPixelStreaming2AudioSink> SinkWeak = Streamer->GetUnlistenedAudioSink();
TSharedPtr<IPixelStreaming2AudioSink> Sink = SinkWeak.Pin();
if (Sink != nullptr)
{
Sink->AddAudioConsumer(TWeakPtrVariant<IPixelStreaming2AudioConsumer>(PSAudioConsumer));
PSAudioSink = SinkWeak;
bPSAudioRegistered = true;
if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); }
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: PS2 registered via unlistened sink (streamer: %s)"),
*StreamerId);
return;
}
}
}
}
void UVoiceCaptureComponent::CleanupPixelStreamingAudio()
{
if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); }
if (PSAudioConsumer != nullptr && bPSAudioRegistered)
{
if (TSharedPtr<IPixelStreaming2AudioSink> Sink = PSAudioSink.Pin())
{
Sink->RemoveAudioConsumer(TWeakPtrVariant<IPixelStreaming2AudioConsumer>(PSAudioConsumer));
}
// M4: RemoveAudioConsumer does not guarantee ConsumeRawPCM on the audio thread
// has finished. Briefly take + release the consumer's critical section to ensure
// any in-flight callback has drained before we tear the component down.
PSAudioConsumer->Fence();
bPSAudioRegistered = false;
}
// Break the consumer's lambda binding so no AsyncTask queued from it references this component.
// Safe to Unbind() here because UPSVoiceAudioConsumer::OnAudioConsumerRemoved now hops to the
// game thread before calling ExecuteIfBound (M2), so Unbind and ExecuteIfBound are serialized.
if (PSAudioConsumer != nullptr) { PSAudioConsumer->OnRemovedDelegate.Unbind(); }
}
void UVoiceCaptureComponent::SwitchAudioToPlayer(const FString& StreamerId, const FString& PlayerId)
{
if (PSAudioConsumer == nullptr) { return; }
// Already attached to this peer — nothing to do.
if (bPSAudioRegistered && CurrentAudioPlayerId == PlayerId) { return; }
IPixelStreaming2Module& PSModule = IPixelStreaming2Module::Get();
TSharedPtr<IPixelStreaming2Streamer> Streamer = PSModule.FindStreamer(StreamerId);
if (Streamer == nullptr) { return; }
TWeakPtr<IPixelStreaming2AudioSink> NewSinkWeak = Streamer->GetPeerAudioSink(PlayerId);
TSharedPtr<IPixelStreaming2AudioSink> NewSink = NewSinkWeak.Pin();
if (NewSink == nullptr) { return; }
// Detach from the old sink before attaching to the new one.
if (bPSAudioRegistered)
{
if (TSharedPtr<IPixelStreaming2AudioSink> OldSink = PSAudioSink.Pin())
{
OldSink->RemoveAudioConsumer(TWeakPtrVariant<IPixelStreaming2AudioConsumer>(PSAudioConsumer));
}
// M4: Ensure any in-flight ConsumeRawPCM has drained before attaching to the new sink.
PSAudioConsumer->Fence();
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: PS2 unregistered audio from player %s"),
*CurrentAudioPlayerId);
}
NewSink->AddAudioConsumer(TWeakPtrVariant<IPixelStreaming2AudioConsumer>(PSAudioConsumer));
PSAudioSink = NewSinkWeak;
bPSAudioRegistered = true;
CurrentAudioPlayerId = PlayerId;
// Reset first-data flag so we can verify the new peer actually streams audio.
PSAudioConsumer->ResetFirstDataFlag();
if (UWorld* World = GetWorld()) { World->GetTimerManager().ClearTimer(PSRetryTimerHandle); }
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: PS2 switched mic to player %s (streamer: %s)"),
*PlayerId,
*StreamerId);
}
// ============================================================================
// Recording lifecycle
// ============================================================================
void UVoiceCaptureComponent::StartRecording()
{
if (bIsRecording) { return; }
bIsRecording = true;
if (AudioSource == EVoiceAudioSource::PixelStreaming && PSAudioConsumer != nullptr) { PSAudioConsumer->StartBuffering(); }
else
{
FScopeLock Lock(&RecordingCriticalSection);
RecordedPCMData.Empty();
}
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: Recording started. Source: %s"),
AudioSource == EVoiceAudioSource::PixelStreaming ? TEXT("PixelStreaming") : TEXT("LocalMic"));
}
void UVoiceCaptureComponent::StopRecordingAndSend()
{
if (!bIsRecording) { return; }
bIsRecording = false;
TArray<float> PCMData;
int32 SampleRate = 0;
int32 NumChannels = 0;
if (AudioSource == EVoiceAudioSource::PixelStreaming && PSAudioConsumer != nullptr)
{
PSAudioConsumer->StopBuffering(PCMData, SampleRate, NumChannels);
}
else
{
FScopeLock Lock(&RecordingCriticalSection);
PCMData = MoveTemp(RecordedPCMData);
SampleRate = CaptureSampleRate > 0 ? CaptureSampleRate : 48000;
NumChannels = CaptureNumChannels > 0 ? CaptureNumChannels : 1;
RecordedPCMData.Empty();
}
UE_LOG(
LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Recording stopped. Samples: %d"), PCMData.Num());
if (PCMData.Num() == 0)
{
UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceCaptureComponent: No audio data recorded."));
OnError.Broadcast(TEXT("No audio recorded"));
return;
}
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCaptureComponent: Audio format: %d Hz, %d channels"),
SampleRate,
NumChannels);
// Stereo → mono downmix (simple average). Preserves mono buffers as-is.
TArray<float> MonoData;
if (NumChannels == 2)
{
const int32 NumFrames = PCMData.Num() / 2;
MonoData.SetNum(NumFrames);
for (int32 i = 0; i < NumFrames; i++)
{
MonoData[i] = (PCMData[i * 2] + PCMData[i * 2 + 1]) * 0.5f;
}
}
else { MonoData = MoveTemp(PCMData); }
// Silence detection: if the loudest sample is below the threshold, abort the request.
float MaxAmplitude = 0.f;
for (float Sample : MonoData)
{
MaxAmplitude = FMath::Max(MaxAmplitude, FMath::Abs(Sample));
}
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Max amplitude: %f"), MaxAmplitude);
if (MaxAmplitude < SilenceThreshold)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCaptureComponent: Audio is silence (max %.6f < threshold %.6f)."),
MaxAmplitude,
SilenceThreshold);
OnError.Broadcast(TEXT("Audio was silence"));
return;
}
TArray<uint8> WavData = ConvertPCMToWav(MonoData, SampleRate, 1);
if (bSaveDebugRecording)
{
const FString DebugPath = FPaths::ProjectSavedDir() / TEXT("debug_recording.wav");
FFileHelper::SaveArrayToFile(WavData, *DebugPath);
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCaptureComponent: Saved debug WAV to %s"), *DebugPath);
}
// Resolve the subsystem via the owning actor's game instance.
UVoiceCommandSubsystem* Subsystem = nullptr;
if (AActor* OwnerActor = GetOwner())
{
if (UWorld* World = OwnerActor->GetWorld())
{
if (UGameInstance* GI = World->GetGameInstance())
{
Subsystem = GI->GetSubsystem<UVoiceCommandSubsystem>();
}
}
}
if (Subsystem == nullptr)
{
UE_LOG(LogBasemashVoiceCommander, Error, TEXT("VoiceCaptureComponent: UVoiceCommandSubsystem not found."));
OnError.Broadcast(TEXT("VoiceCommandSubsystem unavailable"));
return;
}
// Per-request callbacks — the subsystem invokes these only for this request,
// so events reach ONLY the component that originated the call.
FVoiceRequestCallbacks Callbacks;
Callbacks.Context = this;
Callbacks.OnTranscription.AddDynamic(this, &UVoiceCaptureComponent::HandleTranscription);
Callbacks.OnCommandParsed.AddDynamic(this, &UVoiceCaptureComponent::HandleCommandParsed);
Callbacks.OnError.AddDynamic(this, &UVoiceCaptureComponent::HandleError);
Subsystem->SendAudioToGroq(WavData, ApiKeyOverride, Callbacks);
}
// ============================================================================
// Subsystem → component event fan-out
// ============================================================================
void UVoiceCaptureComponent::HandleTranscription(const FString& Text) { OnTranscription.Broadcast(Text); }
void UVoiceCaptureComponent::HandleCommandParsed(const FString& Action, const FString& ParamsJson)
{
OnCommandParsed.Broadcast(Action, ParamsJson);
}
void UVoiceCaptureComponent::HandleError(const FString& ErrorMessage) { OnError.Broadcast(ErrorMessage); }
// ============================================================================
// WAV encoding (float PCM → 16-bit PCM WAV, byte-for-byte compatible with original)
// ============================================================================
TArray<uint8>
UVoiceCaptureComponent::ConvertPCMToWav(const TArray<float>& PCMData, int32 SampleRate, int32 NumChannels) const
{
// Float [-1.0, 1.0] → int16
TArray<int16> Int16Data;
Int16Data.SetNum(PCMData.Num());
for (int32 i = 0; i < PCMData.Num(); i++)
{
const float Clamped = FMath::Clamp(PCMData[i], -1.0f, 1.0f);
Int16Data[i] = static_cast<int16>(Clamped * 32767.0f);
}
const int32 DataSize = Int16Data.Num() * sizeof(int16);
const int32 FileSize = 44 + DataSize; // standard 44-byte WAV header
TArray<uint8> WavData;
WavData.SetNum(FileSize);
uint8* Ptr = WavData.GetData();
// RIFF header
FMemory::Memcpy(Ptr, "RIFF", 4);
Ptr += 4;
int32 ChunkSize = FileSize - 8;
FMemory::Memcpy(Ptr, &ChunkSize, 4);
Ptr += 4;
FMemory::Memcpy(Ptr, "WAVE", 4);
Ptr += 4;
// fmt subchunk
FMemory::Memcpy(Ptr, "fmt ", 4);
Ptr += 4;
int32 SubChunk1Size = 16;
FMemory::Memcpy(Ptr, &SubChunk1Size, 4);
Ptr += 4;
int16 AudioFormat = 1; // PCM
FMemory::Memcpy(Ptr, &AudioFormat, 2);
Ptr += 2;
int16 Channels = static_cast<int16>(NumChannels);
FMemory::Memcpy(Ptr, &Channels, 2);
Ptr += 2;
FMemory::Memcpy(Ptr, &SampleRate, 4);
Ptr += 4;
int32 ByteRate = SampleRate * NumChannels * sizeof(int16);
FMemory::Memcpy(Ptr, &ByteRate, 4);
Ptr += 4;
int16 BlockAlign = static_cast<int16>(NumChannels * sizeof(int16));
FMemory::Memcpy(Ptr, &BlockAlign, 2);
Ptr += 2;
int16 BitsPerSample = 16;
FMemory::Memcpy(Ptr, &BitsPerSample, 2);
Ptr += 2;
// data subchunk
FMemory::Memcpy(Ptr, "data", 4);
Ptr += 4;
FMemory::Memcpy(Ptr, &DataSize, 4);
Ptr += 4;
FMemory::Memcpy(Ptr, Int16Data.GetData(), DataSize);
return WavData;
}

View file

@ -0,0 +1,209 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "VoiceCommandRegistry.h"
#include "BasemashVoiceCommanderLog.h"
#include "Engine/GameInstance.h"
#include "Kismet/GameplayStatics.h"
void UVoiceCommandRegistry::Initialize(FSubsystemCollectionBase& Collection)
{
Super::Initialize(Collection);
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandRegistry: Initialized."));
}
void UVoiceCommandRegistry::Deinitialize()
{
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandRegistry: Deinitialized (%d actions cleared)."),
Actions.Num());
Actions.Empty();
Super::Deinitialize();
}
void UVoiceCommandRegistry::RegisterAction(const FVoiceCommandAction& Action)
{
if (Action.Name.IsEmpty())
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandRegistry::RegisterAction: Action has empty Name, ignoring."));
return;
}
// Protocol-reserved name; 'none' is always appended by BuildSystemPrompt and must not be overridden.
if (Action.Name.Equals(TEXT("none"), ESearchCase::IgnoreCase))
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandRegistry::RegisterAction: 'none' is a reserved protocol action, ignoring."));
return;
}
const bool bReplacing = Actions.Contains(Action.Name);
Actions.Add(Action.Name, Action);
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandRegistry: %s action '%s'."),
bReplacing ? TEXT("Replaced") : TEXT("Registered"),
*Action.Name);
}
bool UVoiceCommandRegistry::RegisterSimpleAction(const FString& Name,
const FString& Description,
const FString& ParamSchema,
const TArray<FString>& Examples,
const FVoiceActionHandler& Handler)
{
if (Name.IsEmpty() || Name.Equals(TEXT("none"), ESearchCase::IgnoreCase))
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandRegistry::RegisterSimpleAction: invalid Name ('%s'), ignoring."),
*Name);
return false;
}
FVoiceCommandAction Action;
Action.Name = Name;
Action.Description = Description;
Action.ParamSchema = ParamSchema;
Action.Examples = Examples;
Action.Handler = Handler;
RegisterAction(Action);
return true;
}
void UVoiceCommandRegistry::UnregisterAction(const FString& ActionName)
{
const int32 Removed = Actions.Remove(ActionName);
if (Removed > 0)
{
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandRegistry: Unregistered action '%s'."), *ActionName);
}
}
bool UVoiceCommandRegistry::OverrideActionHandler(const FString& ActionName, const FVoiceActionHandler& NewHandler)
{
FVoiceCommandAction* Existing = Actions.Find(ActionName);
if (Existing == nullptr)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandRegistry::OverrideActionHandler: action '%s' not found."),
*ActionName);
return false;
}
Existing->Handler = NewHandler;
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandRegistry: Overrode handler for '%s'."), *ActionName);
return true;
}
TArray<FVoiceCommandAction> UVoiceCommandRegistry::GetRegisteredActions() const
{
TArray<FVoiceCommandAction> Out;
Out.Reserve(Actions.Num());
for (const TPair<FString, FVoiceCommandAction>& Pair : Actions) { Out.Add(Pair.Value); }
return Out;
}
bool UVoiceCommandRegistry::HasAction(const FString& ActionName) const { return Actions.Contains(ActionName); }
const FVoiceCommandAction* UVoiceCommandRegistry::FindAction(const FString& ActionName) const
{ return Actions.Find(ActionName); }
bool UVoiceCommandRegistry::DispatchAction(const FString& ActionName, const FString& ParamsJson)
{
FVoiceCommandAction* Entry = Actions.Find(ActionName);
if (Entry == nullptr)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandRegistry::DispatchAction: no action registered for '%s'."),
*ActionName);
return false;
}
// Handler delegate takes priority over ExecutorObject when bound.
if (Entry->Handler.IsBound())
{
const bool bResult = Entry->Handler.Execute(ActionName, ParamsJson);
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandRegistry::DispatchAction: '%s' handled by delegate (result=%s)."),
*ActionName,
bResult ? TEXT("true") : TEXT("false"));
return bResult;
}
// Fall back to interface executor; use Execute_ wrapper so Blueprint implementations also fire.
if (UObject* ExecObj = Entry->ExecutorObject.GetObject())
{
const bool bResult = IVoiceCommandExecutor::Execute_ExecuteVoiceAction(ExecObj, ActionName, ParamsJson);
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandRegistry::DispatchAction: '%s' handled by ExecutorObject (result=%s)."),
*ActionName,
bResult ? TEXT("true") : TEXT("false"));
return bResult;
}
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandRegistry::DispatchAction: '%s' has no Handler or ExecutorObject bound."),
*ActionName);
return false;
}
FString UVoiceCommandRegistry::BuildSystemPrompt() const
{
FString Prompt;
Prompt.Reserve(1024);
Prompt += TEXT("You are a voice command parser for an Unreal Engine application. ");
Prompt += TEXT("Your task is to extract a single action and its parameters from the user's text.\n\n");
Prompt += TEXT("Available actions:\n");
int32 Index = 1;
for (const TPair<FString, FVoiceCommandAction>& Pair : Actions)
{
const FVoiceCommandAction& Action = Pair.Value;
Prompt += FString::Printf(TEXT("%d. %s"), Index++, *Action.Name);
if (!Action.Description.IsEmpty()) { Prompt += FString::Printf(TEXT(" - %s"), *Action.Description); }
Prompt += TEXT("\n");
if (!Action.ParamSchema.IsEmpty()) { Prompt += FString::Printf(TEXT(" params: %s\n"), *Action.ParamSchema); }
if (Action.Examples.Num() > 0)
{
Prompt += TEXT(" Examples:");
for (const FString& Example : Action.Examples) { Prompt += FString::Printf(TEXT(" \"%s\""), *Example); }
Prompt += TEXT("\n");
}
Prompt += TEXT("\n");
}
// Protocol constant: 'none' is always last and always present, even if nobody registered anything.
Prompt += FString::Printf(TEXT("%d. none - unrecognized command\n"), Index);
Prompt += TEXT(" params: {}\n\n");
Prompt += TEXT("Output format: {\"action\": \"<name>\", \"params\": {<parameters>}}\n");
Prompt += TEXT("Example: {\"action\": \"move\", \"params\": {\"direction\": \"left\", \"distance\": 30}}\n");
Prompt += TEXT("Respond ONLY with a valid JSON object. No prose.");
return Prompt;
}
UVoiceCommandRegistry* UVoiceCommandRegistry::Get(const UObject* WorldContextObject)
{
if (WorldContextObject == nullptr) { return nullptr; }
UGameInstance* GameInstance = UGameplayStatics::GetGameInstance(WorldContextObject);
if (GameInstance == nullptr) { return nullptr; }
return GameInstance->GetSubsystem<UVoiceCommandRegistry>();
}

View file

@ -0,0 +1,608 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "VoiceCommandSubsystem.h"
#include "BasemashVoiceCommanderLog.h"
#include "Dom/JsonObject.h"
#include "Engine/GameInstance.h"
#include "HttpModule.h"
#include "Interfaces/IHttpResponse.h"
#include "Serialization/JsonReader.h"
#include "Serialization/JsonSerializer.h"
#include "VoiceCommandRegistry.h"
#include "VoiceCommanderSettings.h"
void UVoiceCommandSubsystem::Initialize(FSubsystemCollectionBase& Collection)
{
Super::Initialize(Collection);
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: Initialized."));
}
void UVoiceCommandSubsystem::Deinitialize()
{
// Cancel any in-flight HTTP requests we dispatched so their BindUObject callbacks
// don't fire on a destroyed subsystem (PIE shutdown, seamless travel, etc.).
int32 CancelledCount = 0;
for (TWeakPtr<IHttpRequest, ESPMode::ThreadSafe>& WeakReq : InFlightRequests)
{
if (TSharedPtr<IHttpRequest, ESPMode::ThreadSafe> Pinned = WeakReq.Pin())
{
Pinned->CancelRequest();
++CancelledCount;
}
}
InFlightRequests.Reset();
if (CancelledCount > 0)
{
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandSubsystem: Cancelled %d in-flight request(s) on Deinitialize."),
CancelledCount);
}
Super::Deinitialize();
}
void UVoiceCommandSubsystem::SetApiKey(const FString& Key)
{
RuntimeApiKey = Key;
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandSubsystem: Runtime API key %s (%d chars)."),
Key.IsEmpty() ? TEXT("cleared") : TEXT("set"),
Key.Len());
}
FString UVoiceCommandSubsystem::GetEffectiveApiKey(const FString& PerCallOverride) const
{
if (!PerCallOverride.IsEmpty()) { return PerCallOverride; }
if (!RuntimeApiKey.IsEmpty()) { return RuntimeApiKey; }
if (const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get())
{
if (!Settings->GroqApiKey.IsEmpty()) { return Settings->GroqApiKey; }
}
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandSubsystem: No API key available (per-call/runtime/settings all empty)."));
return FString();
}
void UVoiceCommandSubsystem::SendAudioToGroq(const TArray<uint8>& WavAudioData,
const FString& PerCallApiKey,
const FVoiceRequestCallbacks& Callbacks)
{
// Check bIsProcessing BEFORE touching CurrentCallbacks — otherwise a concurrent
// second request clobbers the in-flight request's delegates. Fire the busy-path
// error through the caller's own callback struct (local temp), not the shared member.
if (bIsProcessing)
{
const FString BusyMsg = TEXT("Already processing a request.");
if (Callbacks.Context.IsValid() && Callbacks.OnError.IsBound()) { Callbacks.OnError.Broadcast(BusyMsg); }
OnAnyError.Broadcast(BusyMsg);
// Do NOT reset CurrentCallbacks: we'd clobber the in-flight request's state.
return;
}
CurrentCallbacks = Callbacks;
const FString ApiKey = GetEffectiveApiKey(PerCallApiKey);
if (ApiKey.IsEmpty())
{
FireError(TEXT("Groq API key not configured."));
FinishRequest();
return;
}
const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get();
if (Settings == nullptr)
{
FireError(TEXT("VoiceCommander settings not available."));
FinishRequest();
return;
}
bIsProcessing = true;
CurrentAudioData = WavAudioData;
CurrentTranscription.Empty();
RequestStartTime = FDateTime::Now();
FHttpRequestRef Request = FHttpModule::Get().CreateRequest();
Request->SetURL(TEXT("https://api.groq.com/openai/v1/audio/transcriptions"));
Request->SetVerb(TEXT("POST"));
Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *ApiKey));
const FString Boundary = FString::Printf(TEXT("----UnrealBoundary%d"), FMath::RandRange(100000, 999999));
Request->SetHeader(TEXT("Content-Type"), FString::Printf(TEXT("multipart/form-data; boundary=%s"), *Boundary));
TArray<uint8> Payload;
auto AppendString = [&Payload](const FString& Str)
{
FTCHARToUTF8 Converter(*Str);
Payload.Append(reinterpret_cast<const uint8*>(Converter.Get()), Converter.Length());
};
AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary));
AppendString(TEXT("Content-Disposition: form-data; name=\"file\"; filename=\"recording.wav\"\r\n"));
AppendString(TEXT("Content-Type: audio/wav\r\n\r\n"));
Payload.Append(WavAudioData);
AppendString(TEXT("\r\n"));
AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary));
AppendString(TEXT("Content-Disposition: form-data; name=\"model\"\r\n\r\n"));
AppendString(FString::Printf(TEXT("%s\r\n"), *Settings->SttModel));
if (!Settings->SttLanguage.IsEmpty())
{
AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary));
AppendString(TEXT("Content-Disposition: form-data; name=\"language\"\r\n\r\n"));
AppendString(FString::Printf(TEXT("%s\r\n"), *Settings->SttLanguage));
}
AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary));
AppendString(TEXT("Content-Disposition: form-data; name=\"response_format\"\r\n\r\n"));
AppendString(TEXT("json\r\n"));
AppendString(FString::Printf(TEXT("--%s--\r\n"), *Boundary));
Request->SetContent(Payload);
Request->OnProcessRequestComplete().BindUObject(this, &UVoiceCommandSubsystem::OnSTTResponse);
Request->ProcessRequest();
InFlightRequests.Add(TWeakPtr<IHttpRequest, ESPMode::ThreadSafe>(Request));
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandSubsystem: Sending %d bytes of audio to Groq (model=%s, lang=%s)."),
WavAudioData.Num(),
*Settings->SttModel,
*Settings->SttLanguage);
}
void UVoiceCommandSubsystem::OnSTTResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess)
{
if (!bSuccess || !Res.IsValid())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("STT request failed - no response."));
FireError(TEXT("STT request failed - no response."));
FinishRequest();
return;
}
const int32 Code = Res->GetResponseCode();
const FString Body = Res->GetContentAsString();
if (Code != 200)
{
const FString Err = FString::Printf(TEXT("STT error %d: %s"), Code, *Body);
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), Err);
FireError(Err);
FinishRequest();
return;
}
TSharedPtr<FJsonObject> JsonObject;
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Body);
if (!FJsonSerializer::Deserialize(Reader, JsonObject) || !JsonObject.IsValid())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("STT response JSON parse failed."));
FireError(TEXT("STT response JSON parse failed."));
FinishRequest();
return;
}
FString TranscribedText;
JsonObject->TryGetStringField(TEXT("text"), TranscribedText);
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: Transcription = \"%s\""), *TranscribedText);
if (TranscribedText.IsEmpty())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("STT returned empty transcription."));
FireError(TEXT("STT returned empty transcription."));
FinishRequest();
return;
}
CurrentTranscription = TranscribedText;
STTCompleteTime = FDateTime::Now();
FireTranscription(TranscribedText);
// Chain STT -> LLM; carry same callbacks so per-request fan-out stays consistent.
// Pass empty PerCall key because GetEffectiveApiKey already resolved successfully
// (and re-resolving via runtime/settings gives the same answer).
// bChainedFromSTT lets ParseCommandWithLLM skip the bIsProcessing guard and the
// timing re-stamp without the empty-string sentinel the old code relied on.
FVoiceRequestCallbacks ChainCallbacks = CurrentCallbacks;
bChainedFromSTT = true;
ParseCommandWithLLM(TranscribedText, FString(), ChainCallbacks);
}
void UVoiceCommandSubsystem::ParseCommandWithLLM(const FString& TranscribedText,
const FString& PerCallApiKey,
const FVoiceRequestCallbacks& Callbacks)
{
// When chained from STT the audio path already owns CurrentCallbacks + bIsProcessing;
// otherwise check busy first (C1) and only then adopt the caller's callbacks.
if (!bChainedFromSTT)
{
if (bIsProcessing)
{
const FString BusyMsg = TEXT("Already processing a request.");
if (Callbacks.Context.IsValid() && Callbacks.OnError.IsBound()) { Callbacks.OnError.Broadcast(BusyMsg); }
OnAnyError.Broadcast(BusyMsg);
return;
}
CurrentCallbacks = Callbacks;
}
const FString ApiKey = GetEffectiveApiKey(PerCallApiKey);
if (ApiKey.IsEmpty())
{
FireError(TEXT("Groq API key not configured."));
bChainedFromSTT = false;
FinishRequest();
return;
}
const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get();
if (Settings == nullptr)
{
FireError(TEXT("VoiceCommander settings not available."));
bChainedFromSTT = false;
FinishRequest();
return;
}
// Fresh text-only call: start timing + mark busy. Chained-from-STT path already did both.
if (!bChainedFromSTT)
{
bIsProcessing = true;
RequestStartTime = FDateTime::Now();
STTCompleteTime = RequestStartTime;
CurrentTranscription = TranscribedText;
}
FHttpRequestRef Request = FHttpModule::Get().CreateRequest();
Request->SetURL(TEXT("https://api.groq.com/openai/v1/chat/completions"));
Request->SetVerb(TEXT("POST"));
Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *ApiKey));
Request->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
const FString SystemPrompt = BuildSystemPrompt();
// Build the chat-completions body with FJsonObject so control chars / quotes / newlines
// in prompts or transcription all get escaped correctly by the writer (M6).
const TSharedRef<FJsonObject> Root = MakeShared<FJsonObject>();
Root->SetStringField(TEXT("model"), Settings->LlmModel);
Root->SetNumberField(TEXT("temperature"), 0.0);
const TSharedRef<FJsonObject> SystemMsg = MakeShared<FJsonObject>();
SystemMsg->SetStringField(TEXT("role"), TEXT("system"));
SystemMsg->SetStringField(TEXT("content"), SystemPrompt);
const TSharedRef<FJsonObject> UserMsg = MakeShared<FJsonObject>();
UserMsg->SetStringField(TEXT("role"), TEXT("user"));
UserMsg->SetStringField(TEXT("content"), TranscribedText);
TArray<TSharedPtr<FJsonValue>> Messages;
Messages.Add(MakeShared<FJsonValueObject>(SystemMsg));
Messages.Add(MakeShared<FJsonValueObject>(UserMsg));
Root->SetArrayField(TEXT("messages"), Messages);
FString JsonBody;
TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&JsonBody);
FJsonSerializer::Serialize(Root, Writer);
Request->SetContentAsString(JsonBody);
Request->OnProcessRequestComplete().BindUObject(this, &UVoiceCommandSubsystem::OnLLMResponse);
Request->ProcessRequest();
InFlightRequests.Add(TWeakPtr<IHttpRequest, ESPMode::ThreadSafe>(Request));
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandSubsystem: Sending to LLM (model=%s): \"%s\""),
*Settings->LlmModel,
*TranscribedText);
// Chain flag is single-use — reset now that dispatch is underway so a later stand-alone
// text-only call runs through the full guard + timing path.
bChainedFromSTT = false;
}
void UVoiceCommandSubsystem::OnLLMResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess)
{
if (!bSuccess || !Res.IsValid())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM request failed - no response."));
FireError(TEXT("LLM request failed - no response."));
FinishRequest();
return;
}
const int32 Code = Res->GetResponseCode();
const FString Body = Res->GetContentAsString();
if (Code != 200)
{
const FString Err = FString::Printf(TEXT("LLM error %d: %s"), Code, *Body);
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), Err);
FireError(Err);
FinishRequest();
return;
}
TSharedPtr<FJsonObject> JsonObject;
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Body);
if (!FJsonSerializer::Deserialize(Reader, JsonObject) || !JsonObject.IsValid())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM response JSON parse failed."));
FireError(TEXT("LLM response JSON parse failed."));
FinishRequest();
return;
}
const TArray<TSharedPtr<FJsonValue>>* Choices = nullptr;
if (!JsonObject->TryGetArrayField(TEXT("choices"), Choices) || Choices->Num() == 0)
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM response has no choices."));
FireError(TEXT("LLM response has no choices."));
FinishRequest();
return;
}
const TSharedPtr<FJsonObject> Choice = (*Choices)[0]->AsObject();
if (!Choice.IsValid())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM choice is not an object."));
FireError(TEXT("LLM choice is not an object."));
FinishRequest();
return;
}
// Use Try* variants: GetObjectField crashes if the field is absent or wrong type (M5).
// UE 5.7 TryGetObjectField signature is (FStringView, const TSharedPtr<FJsonObject>*&)
// — returns pointer to the internal shared ptr, not a fresh one.
const TSharedPtr<FJsonObject>* MessagePtr = nullptr;
if (!Choice->TryGetObjectField(TEXT("message"), MessagePtr) || !MessagePtr || !MessagePtr->IsValid())
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM message missing."));
FireError(TEXT("LLM message missing."));
FinishRequest();
return;
}
const TSharedPtr<FJsonObject>& Message = *MessagePtr;
FString Content;
if (!Message->TryGetStringField(TEXT("content"), Content))
{
SendLogToPanel(TEXT(""), TEXT("{}"), TEXT(""), TEXT("LLM message content missing."));
FireError(TEXT("LLM message content missing."));
FinishRequest();
return;
}
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: LLM response = %s"), *Content);
TSharedPtr<FJsonObject> CommandJson;
TSharedRef<TJsonReader<>> CmdReader = TJsonReaderFactory<>::Create(Content);
if (!FJsonSerializer::Deserialize(CmdReader, CommandJson) || !CommandJson.IsValid())
{
// Truncate + flatten so a multi-line prose response doesn't flood the error log (m9).
const FString Snippet = Content.Left(200).Replace(TEXT("\n"), TEXT(" ")).Replace(TEXT("\r"), TEXT(""));
const FString Err = FString::Printf(TEXT("Failed to parse LLM command JSON: %s"), *Snippet);
SendLogToPanel(TEXT(""), TEXT("{}"), Content, Err);
FireError(Err);
FinishRequest();
return;
}
FString Action;
CommandJson->TryGetStringField(TEXT("action"), Action);
FString ParamsJson;
const TSharedPtr<FJsonObject>* ParamsPtr = nullptr;
if (CommandJson->TryGetObjectField(TEXT("params"), ParamsPtr) && ParamsPtr && (*ParamsPtr).IsValid())
{
TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&ParamsJson);
FJsonSerializer::Serialize((*ParamsPtr).ToSharedRef(), Writer);
}
else { ParamsJson = TEXT("{}"); }
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceCommandSubsystem: Action='%s' Params=%s"), *Action, *ParamsJson);
// LLM signals "didn't understand" with action="none". Fire the event so BP UI can react,
// but skip registry dispatch to avoid a spurious "no action registered for 'none'" warning (m3).
if (Action.Equals(TEXT("none"), ESearchCase::IgnoreCase))
{
SendLogToPanel(Action, ParamsJson, Content);
FireCommandParsed(Action, ParamsJson);
FinishRequest();
return;
}
SendLogToPanel(Action, ParamsJson, Content);
// Dispatch to registry BEFORE firing event: keeps execution ordering deterministic.
// Dispatch result is logged but must not gate the event, listeners may want to observe anyway.
if (UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this))
{
const bool bDispatched = Registry->DispatchAction(Action, ParamsJson);
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandSubsystem: Dispatch result for '%s' = %s"),
*Action,
bDispatched ? TEXT("handled") : TEXT("unhandled"));
}
else
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandSubsystem: No registry available to dispatch '%s'."),
*Action);
}
FireCommandParsed(Action, ParamsJson);
FinishRequest();
}
FString UVoiceCommandSubsystem::BuildSystemPrompt() const
{
FString Base;
if (const UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this))
{
Base = Registry->BuildSystemPrompt();
if (Registry->GetRegisteredActions().Num() == 0)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandSubsystem: Registry has no actions; sending base prompt only."));
}
}
else
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandSubsystem: Registry unavailable; sending minimal prompt."));
}
const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get();
if (Settings && !Settings->SttLanguage.IsEmpty())
{
return FString::Printf(TEXT("User speaks language code: %s.\n\n%s"), *Settings->SttLanguage, *Base);
}
return Base;
}
void UVoiceCommandSubsystem::SendLogToPanel(const FString& Action,
const FString& ParamsJson,
const FString& LLMRaw,
const FString& Error)
{
const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get();
if (Settings == nullptr || !Settings->bLogToPanel || Settings->LogPanelUrl.IsEmpty()) { return; }
const FDateTime Now = FDateTime::Now();
int64 SttMs = (STTCompleteTime - RequestStartTime).GetTotalMilliseconds();
int64 LlmMs = (Now - STTCompleteTime).GetTotalMilliseconds();
const int64 TotalMs = (Now - RequestStartTime).GetTotalMilliseconds();
if (SttMs < 0) SttMs = 0;
if (LlmMs < 0) LlmMs = 0;
FHttpRequestRef Request = FHttpModule::Get().CreateRequest();
Request->SetURL(Settings->LogPanelUrl);
Request->SetVerb(TEXT("POST"));
// If the log panel isn't running, don't stall editor shutdown on the default HTTP retry (n3).
Request->SetTimeout(5.0f);
const FString Boundary = FString::Printf(TEXT("----LogBoundary%d"), FMath::RandRange(100000, 999999));
Request->SetHeader(TEXT("Content-Type"), FString::Printf(TEXT("multipart/form-data; boundary=%s"), *Boundary));
TArray<uint8> Payload;
auto AppendString = [&Payload](const FString& Str)
{
FTCHARToUTF8 Converter(*Str);
Payload.Append(reinterpret_cast<const uint8*>(Converter.Get()), Converter.Length());
};
auto AddField = [&](const FString& Name, const FString& Value)
{
AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary));
AppendString(FString::Printf(TEXT("Content-Disposition: form-data; name=\"%s\"\r\n\r\n"), *Name));
AppendString(Value + TEXT("\r\n"));
};
AddField(TEXT("timestamp"), Now.ToIso8601());
AddField(TEXT("transcription"), CurrentTranscription);
AddField(TEXT("action"), Action);
AddField(TEXT("params"), ParamsJson);
AddField(TEXT("llm_raw"), LLMRaw);
AddField(TEXT("stt_duration_ms"), FString::FromInt(SttMs));
AddField(TEXT("llm_duration_ms"), FString::FromInt(LlmMs));
AddField(TEXT("total_duration_ms"), FString::FromInt(TotalMs));
AddField(TEXT("error"), Error);
if (CurrentAudioData.Num() > 0)
{
AppendString(FString::Printf(TEXT("--%s\r\n"), *Boundary));
AppendString(TEXT("Content-Disposition: form-data; name=\"audio\"; filename=\"recording.wav\"\r\n"));
AppendString(TEXT("Content-Type: audio/wav\r\n\r\n"));
Payload.Append(CurrentAudioData);
AppendString(TEXT("\r\n"));
}
AppendString(FString::Printf(TEXT("--%s--\r\n"), *Boundary));
Request->SetContent(Payload);
Request->OnProcessRequestComplete().BindUObject(this, &UVoiceCommandSubsystem::OnLogPanelResponse);
Request->ProcessRequest();
InFlightRequests.Add(TWeakPtr<IHttpRequest, ESPMode::ThreadSafe>(Request));
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceCommandSubsystem: Log sent to panel (action=%s, stt=%lldms, llm=%lldms)."),
*Action,
SttMs,
LlmMs);
}
void UVoiceCommandSubsystem::OnLogPanelResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess)
{
if (!bSuccess || !Res.IsValid() || Res->GetResponseCode() != 200)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceCommandSubsystem: Log panel POST failed (panel may not be running)."));
}
}
void UVoiceCommandSubsystem::FireTranscription(const FString& Text)
{
// Per-request fan-out: only if Context UObject is still alive so we don't
// call into a destroyed component.
if (CurrentCallbacks.Context.IsValid() && CurrentCallbacks.OnTranscription.IsBound())
{
CurrentCallbacks.OnTranscription.Broadcast(Text);
}
OnAnyTranscriptionReceived.Broadcast(Text);
}
void UVoiceCommandSubsystem::FireCommandParsed(const FString& Action, const FString& ParamsJson)
{
if (CurrentCallbacks.Context.IsValid() && CurrentCallbacks.OnCommandParsed.IsBound())
{
CurrentCallbacks.OnCommandParsed.Broadcast(Action, ParamsJson);
}
OnAnyCommandParsed.Broadcast(Action, ParamsJson);
}
void UVoiceCommandSubsystem::FireError(const FString& ErrorMessage)
{
if (CurrentCallbacks.Context.IsValid() && CurrentCallbacks.OnError.IsBound())
{
CurrentCallbacks.OnError.Broadcast(ErrorMessage);
}
OnAnyError.Broadcast(ErrorMessage);
}
void UVoiceCommandSubsystem::FinishRequest()
{
bIsProcessing = false;
CurrentCallbacks = FVoiceRequestCallbacks();
CurrentAudioData.Empty();
CurrentTranscription.Empty();
// Drop completed/expired HTTP handles. Any log-panel request that outlives FinishRequest
// will expire naturally; cancelling it here is fine (it already did its job for this request).
// On Deinitialize we iterate what's left and cancel, which is the protection for M1.
InFlightRequests.Reset();
}

View file

@ -0,0 +1,23 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "VoiceCommanderSettings.h"
const UVoiceCommanderSettings* UVoiceCommanderSettings::Get() { return GetDefault<UVoiceCommanderSettings>(); }
FString UVoiceCommanderSettings::GetGroqApiKey() { return GetDefault<UVoiceCommanderSettings>()->GroqApiKey; }
FString UVoiceCommanderSettings::GetSttModel() { return GetDefault<UVoiceCommanderSettings>()->SttModel; }
FString UVoiceCommanderSettings::GetLlmModel() { return GetDefault<UVoiceCommanderSettings>()->LlmModel; }
FString UVoiceCommanderSettings::GetSttLanguage() { return GetDefault<UVoiceCommanderSettings>()->SttLanguage; }
FString UVoiceCommanderSettings::GetLogPanelUrl() { return GetDefault<UVoiceCommanderSettings>()->LogPanelUrl; }
bool UVoiceCommanderSettings::GetLogToPanel() { return GetDefault<UVoiceCommanderSettings>()->bLogToPanel; }
FName UVoiceCommanderSettings::GetCategoryName() const
{
// Lands the settings under Project Settings -> Plugins -> Voice Commander.
return TEXT("Plugins");
}

View file

@ -0,0 +1,961 @@
// Copyright (c) Basemash. All Rights Reserved.
#include "VoiceObjectEditorComponent.h"
#include "BasemashVoiceCommanderLog.h"
#include "Camera/CameraComponent.h"
#include "CollisionQueryParams.h"
#include "Components/MeshComponent.h"
#include "Components/PostProcessComponent.h"
#include "Components/PrimitiveComponent.h"
#include "Components/SceneComponent.h"
#include "Dom/JsonObject.h"
#include "Engine/EngineTypes.h"
#include "Engine/World.h"
#include "GameFramework/Actor.h"
#include "GameFramework/Pawn.h"
#include "GameFramework/PlayerController.h"
#include "Kismet/GameplayStatics.h"
#include "Materials/MaterialInterface.h"
#include "Serialization/JsonReader.h"
#include "Serialization/JsonSerializer.h"
#include "VoiceCaptureComponent.h"
#include "VoiceCommandAction.h"
#include "VoiceCommandRegistry.h"
#include "VoiceCommanderSettings.h"
UVoiceObjectEditorComponent::UVoiceObjectEditorComponent()
{
PrimaryComponentTick.bCanEverTick = true;
PrimaryComponentTick.bStartWithTickEnabled = true;
}
// ============================================================================
// Lifecycle
// ============================================================================
void UVoiceObjectEditorComponent::BeginPlay()
{
Super::BeginPlay();
AActor* Owner = GetOwner();
if (Owner == nullptr)
{
UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent::BeginPlay: no owning actor."));
return;
}
// --- Outline post process ---
if (bCreateOutlineComponent)
{
UPostProcessComponent* PP = NewObject<UPostProcessComponent>(Owner, TEXT("VoiceEditorOutlinePostProcess"));
if (PP != nullptr)
{
PP->bUnbound = true;
if (USceneComponent* OwnerRoot = Owner->GetRootComponent())
{
PP->SetupAttachment(OwnerRoot);
}
PP->RegisterComponent();
if (!OutlineMaterial.IsNull())
{
if (UMaterialInterface* Loaded = OutlineMaterial.LoadSynchronous())
{
PP->Settings.WeightedBlendables.Array.Add(FWeightedBlendable(1.f, Loaded));
}
else
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: OutlineMaterial failed to load (%s)."),
*OutlineMaterial.ToSoftObjectPath().ToString());
}
}
OutlinePostProcess = PP;
}
}
// NOTE: CaptureComponent is not used internally here — Registry::DispatchAction is the canonical
// path into this component. We deliberately do NOT auto-find it; the field is reserved for
// BP/subclass queries. Binding to OnCommandParsed would also cause double-dispatch (snap+lerp = instant).
// --- Registry builtins ---
const UVoiceCommanderSettings* Settings = UVoiceCommanderSettings::Get();
UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this);
if (Registry == nullptr)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: VoiceCommandRegistry unavailable; skipping builtin registration. "
"Voice actions will not dispatch until registry is available."));
return;
}
// TScriptInterface<IVoiceCommandExecutor>(this): the templated ctor sees that our class
// inherits IVoiceCommandExecutor natively (is_base_of check), so it both calls SetObject(this)
// AND SetInterface(this) — storing the UObject pointer and the raw interface pointer
// (offset differs due to multiple inheritance). The registry's DispatchAction calls
// IVoiceCommandExecutor::Execute_ExecuteVoiceAction(Object, ...) which works off the UObject.
TScriptInterface<IVoiceCommandExecutor> ExecutorIface(this);
auto RegisterIfEnabled = [&](bool bEnabled,
const TCHAR* Name,
const TCHAR* Description,
const TCHAR* Schema,
const TArray<FString>& Examples)
{
if (!bEnabled) return;
FVoiceCommandAction Action;
Action.Name = Name;
Action.Description = Description;
Action.ParamSchema = Schema;
Action.Examples = Examples;
Action.ExecutorObject = ExecutorIface;
Registry->RegisterAction(Action);
RegisteredActionNames.Add(Action.Name);
};
const bool bMove = !Settings || Settings->bEnableBuiltinMove;
const bool bRotate = !Settings || Settings->bEnableBuiltinRotate;
const bool bApplyMaterial = !Settings || Settings->bEnableBuiltinApplyMaterial;
const bool bUndo = !Settings || Settings->bEnableBuiltinUndo;
const bool bUndoAll = !Settings || Settings->bEnableBuiltinUndoAll;
const bool bSetTime = !Settings || Settings->bEnableBuiltinSetTime;
RegisterIfEnabled(
bMove,
TEXT("move"),
TEXT("Move the selected object (the cue ball) along the table plane. direction is one of up, down, left, right. "
"Phrases like \"move the cue ball left\" or just \"left\" mean the same thing. "
"distance is in centimeters; if the user does not say a number, use 50. Never moves vertically."),
TEXT("{\"direction\": \"up|down|left|right\", \"distance\": <cm, default 50>}"),
{TEXT("move the cue ball left"), TEXT("cue ball right"), TEXT("move up 100"), TEXT("down 30")});
RegisterIfEnabled(bRotate,
TEXT("rotate"),
TEXT("Rotate the selected object around the given axis. axis defaults to yaw; "
"if the user does not say an angle, use 90."),
TEXT("{\"axis\": \"yaw|pitch|roll\", \"angle\": <degrees, default 90>}"),
{TEXT("rotate"), TEXT("rotate 45 degrees"), TEXT("turn 90"), TEXT("rotate yaw 90")});
RegisterIfEnabled(bApplyMaterial,
TEXT("apply_material"),
TEXT("Apply a material to the selected object."),
TEXT("{\"material\": \"<name>\"}"),
{TEXT("apply brick"), TEXT("set concrete"), TEXT("apply wood")});
RegisterIfEnabled(bUndo,
TEXT("undo"),
TEXT("Undo the last action."),
TEXT("{}"),
{TEXT("undo"), TEXT("undo last"), TEXT("go back"), TEXT("revert")});
RegisterIfEnabled(bUndoAll,
TEXT("undo_all"),
TEXT("Reset the selected object to its original state."),
TEXT("{}"),
{TEXT("reset"), TEXT("undo all"), TEXT("original"), TEXT("restore")});
RegisterIfEnabled(bSetTime,
TEXT("set_time"),
TEXT("Set the time of day (0-24)."),
TEXT("{\"hours\": <0-24>}"),
{TEXT("set time to 14"), TEXT("set time 8"), TEXT("noon")});
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Registered %d builtin actions."),
RegisteredActionNames.Num());
// BP-only convenience: auto-select a designated actor (e.g. the cue ball) so voice commands target
// it immediately without a raycast. Pairs with bLockSelection to lock voice control onto it.
if (!InitialSelection.IsNull())
{
if (AActor* Target = InitialSelection.LoadSynchronous())
{
SetSelectedActor(Target);
}
else
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: InitialSelection set but failed to load (%s)."),
*InitialSelection.ToSoftObjectPath().ToString());
}
}
}
void UVoiceObjectEditorComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
if (UVoiceCommandRegistry* Registry = UVoiceCommandRegistry::Get(this))
{
// UVoiceCommandRegistry does not expose FindAction in 5.7.3 — fall back to GetRegisteredActions().
// The list is small (single-digit builtins), so O(n^2) is fine. Only unregister when the registry
// entry still points at us: prevents stomping a sibling UVoiceObjectEditorComponent that re-registered
// the same name (M9) and handles the override-handler case (m4) where another script replaced us
// as executor — in that case we leave the entry alone.
const TArray<FVoiceCommandAction> All = Registry->GetRegisteredActions();
for (const FString& ActionName : RegisteredActionNames)
{
for (const FVoiceCommandAction& A : All)
{
if (A.Name == ActionName && A.ExecutorObject.GetObject() == this)
{
Registry->UnregisterAction(ActionName);
break;
}
}
}
}
RegisteredActionNames.Reset();
if (OutlinePostProcess)
{
OutlinePostProcess->DestroyComponent();
OutlinePostProcess = nullptr;
}
FinishAnimationImmediately();
UndoStack.Reset();
OriginalStates.Reset();
Super::EndPlay(EndPlayReason);
}
void UVoiceObjectEditorComponent::TickComponent(float DeltaTime,
ELevelTick TickType,
FActorComponentTickFunction* ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
if (!IsAnimating() || !IsValid(AnimatingActor)) return;
AnimElapsed += DeltaTime;
const float Alpha = (AnimDuration > SMALL_NUMBER) ? FMath::Clamp(AnimElapsed / AnimDuration, 0.f, 1.f) : 1.f;
const float SmoothAlpha = FMath::SmoothStep(0.f, 1.f, Alpha);
if (bAnimatingLocation)
{
const FVector NewLoc = FMath::Lerp(AnimStartLocation, AnimTargetLocation, SmoothAlpha);
AnimatingActor->SetActorLocation(NewLoc);
}
if (bAnimatingRotation)
{
const FQuat StartQuat = AnimStartRotation.Quaternion();
const FQuat TargetQuat = AnimTargetRotation.Quaternion();
const FQuat NewQuat = FQuat::Slerp(StartQuat, TargetQuat, SmoothAlpha);
AnimatingActor->SetActorRotation(NewQuat.Rotator());
}
if (Alpha >= 1.f)
{
if (bAnimatingLocation) AnimatingActor->SetActorLocation(AnimTargetLocation);
if (bAnimatingRotation) AnimatingActor->SetActorRotation(AnimTargetRotation);
bAnimatingLocation = false;
bAnimatingRotation = false;
AnimatingActor = nullptr;
AnimElapsed = 0.f;
}
}
// ============================================================================
// Voice dispatch
// ============================================================================
bool UVoiceObjectEditorComponent::ExecuteVoiceAction_Implementation(const FString& ActionName,
const FString& ParamsJson)
{
// Undo variants don't require selection or params.
if (ActionName == TEXT("undo"))
{
ExecuteUndo();
return true;
}
if (ActionName == TEXT("undo_all"))
{
ExecuteUndoAll();
return true;
}
// Parse params once.
TSharedPtr<FJsonObject> Params;
if (!ParamsJson.IsEmpty())
{
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(ParamsJson);
FJsonSerializer::Deserialize(Reader, Params);
}
if (ActionName == TEXT("move"))
{
FString Direction = TEXT("up");
double Distance = 50.0;
if (Params.IsValid())
{
Params->TryGetStringField(TEXT("direction"), Direction);
Params->TryGetNumberField(TEXT("distance"), Distance);
}
// LLM emits distance 0 for a bare "move left" with no spoken number; fall back to a usable step.
if (Distance <= 0.0) { Distance = 50.0; }
ExecuteMove(Direction, static_cast<float>(Distance));
return true;
}
if (ActionName == TEXT("rotate"))
{
FString Axis = TEXT("yaw");
double Angle = 90.0;
if (Params.IsValid())
{
Params->TryGetStringField(TEXT("axis"), Axis);
Params->TryGetNumberField(TEXT("angle"), Angle);
}
// LLM emits angle 0 for a bare "rotate" with no spoken number; fall back to a quarter turn.
if (FMath::IsNearlyZero(Angle)) { Angle = 90.0; }
ExecuteRotate(Axis, static_cast<float>(Angle));
return true;
}
if (ActionName == TEXT("apply_material"))
{
FString MaterialName;
if (Params.IsValid())
{
Params->TryGetStringField(TEXT("material"), MaterialName);
}
ExecuteApplyMaterial(MaterialName);
return true;
}
if (ActionName == TEXT("set_time"))
{
double Hours = 12.0;
if (Params.IsValid())
{
Params->TryGetNumberField(TEXT("hours"), Hours);
}
ExecuteSetTime(static_cast<float>(Hours));
return true;
}
return false;
}
// ============================================================================
// Selection
// ============================================================================
bool UVoiceObjectEditorComponent::GetSelectionCameraRay(FVector& OutStart, FVector& OutEnd) const
{
// 1) explicit camera
if (SelectionCameraComponent)
{
OutStart = SelectionCameraComponent->GetComponentLocation();
OutEnd = OutStart + SelectionCameraComponent->GetForwardVector() * SelectionTraceDistance;
return true;
}
// 2) auto-find UCameraComponent on owner
if (AActor* Owner = GetOwner())
{
if (UCameraComponent* Cam = Owner->FindComponentByClass<UCameraComponent>())
{
OutStart = Cam->GetComponentLocation();
OutEnd = OutStart + Cam->GetForwardVector() * SelectionTraceDistance;
return true;
}
}
// 3) fall back to PlayerController view
if (APlayerController* PC = UGameplayStatics::GetPlayerController(this, 0))
{
FVector Loc;
FRotator Rot;
PC->GetPlayerViewPoint(Loc, Rot);
OutStart = Loc;
OutEnd = Loc + Rot.Vector() * SelectionTraceDistance;
return true;
}
return false;
}
void UVoiceObjectEditorComponent::DoSelect()
{
// Locked to InitialSelection (or a BP SetSelectedActor call): ignore raycast selection entirely.
if (bLockSelection)
{
return;
}
FVector Start, End;
if (!GetSelectionCameraRay(Start, End))
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent::DoSelect: no camera source available."));
return;
}
UWorld* World = GetWorld();
if (World == nullptr) return;
FHitResult Hit;
FCollisionQueryParams Params(SCENE_QUERY_STAT(VoiceEditorSelect), false);
if (AActor* Owner = GetOwner()) Params.AddIgnoredActor(Owner);
if (World->LineTraceSingleByChannel(Hit, Start, End, SelectionTraceChannel.GetValue(), Params))
{
SetSelectedActorInternal(Hit.GetActor(), bEnforceMovable);
}
else
{
SetSelectedActorInternal(nullptr, bEnforceMovable);
}
}
void UVoiceObjectEditorComponent::SetSelectedActor(AActor* NewActor)
{
SetSelectedActorInternal(NewActor, bEnforceMovable);
}
void UVoiceObjectEditorComponent::ClearSelection() { SetSelectedActorInternal(nullptr, bEnforceMovable); }
void UVoiceObjectEditorComponent::SetSelectedActorInternal(AActor* NewActor, bool bRequireMovable)
{
// No-op if same actor — avoids redundant highlight toggle and spurious OnSelectionChanged broadcast.
if (NewActor == SelectedActor)
{
return;
}
if (NewActor != nullptr && bRequireMovable)
{
bool bHasMovable = false;
TArray<USceneComponent*> SceneComps;
NewActor->GetComponents<USceneComponent>(SceneComps);
for (USceneComponent* Comp : SceneComps)
{
if (Comp && Comp->Mobility == EComponentMobility::Movable)
{
bHasMovable = true;
break;
}
}
if (!bHasMovable)
{
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: %s is not movable, skipping."),
*NewActor->GetName());
return;
}
}
AActor* PreviousActor = SelectedActor;
// Disable highlight on previous selection (if any).
if (PreviousActor)
{
TArray<UPrimitiveComponent*> PrevPrims;
PreviousActor->GetComponents<UPrimitiveComponent>(PrevPrims);
for (UPrimitiveComponent* Comp : PrevPrims)
{
if (Comp != nullptr) Comp->SetRenderCustomDepth(false);
}
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Deselected %s"),
*PreviousActor->GetName());
}
SelectedActor = NewActor;
// Enable highlight on new selection. Outline rendering depends on a post-process material
// reading CustomDepth; without OutlineMaterial bound the stencil is set but invisible.
if (NewActor)
{
TArray<UPrimitiveComponent*> PrimitiveComponents;
NewActor->GetComponents<UPrimitiveComponent>(PrimitiveComponents);
for (UPrimitiveComponent* Comp : PrimitiveComponents)
{
if (Comp == nullptr) continue;
Comp->SetRenderCustomDepth(true);
Comp->SetCustomDepthStencilValue(1);
}
UE_LOG(LogBasemashVoiceCommander, Log, TEXT("VoiceObjectEditorComponent: Selected %s"), *NewActor->GetName());
}
OnSelectionChanged.Broadcast(PreviousActor, NewActor);
}
// ============================================================================
// Action execution
// ============================================================================
static void DisableOutlineOn(AActor* Actor)
{
if (Actor == nullptr) return;
TArray<UPrimitiveComponent*> Comps;
Actor->GetComponents<UPrimitiveComponent>(Comps);
for (UPrimitiveComponent* Comp : Comps)
{
if (Comp != nullptr) Comp->SetRenderCustomDepth(false);
}
}
void UVoiceObjectEditorComponent::ExecuteMove(const FString& Direction, float Distance)
{
if (!SelectedActor)
{
UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent::ExecuteMove: no selected actor."));
return;
}
USceneComponent* RootComp = SelectedActor->GetRootComponent();
if (RootComp && RootComp->Mobility != EComponentMobility::Movable)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: Cannot move %s - actor is not movable!"),
*SelectedActor->GetName());
return;
}
// Commit any in-flight animation first so the undo snapshot captures the final
// transform of the prior command, not its pre-lerp start (m1).
FinishAnimationImmediately();
DisableOutlineOn(SelectedActor);
SaveOriginalStateIfNeeded(SelectedActor);
SaveUndoState(SelectedActor, EUndoActionType::Move);
// Determine camera-relative direction: prefer owner pawn's ControlRotation
// so LLM-emitted "left/right/forward/backward" are viewport-relative.
FRotator CameraRot = FRotator::ZeroRotator;
if (APawn* OwnerPawn = Cast<APawn>(GetOwner()))
{
CameraRot = OwnerPawn->GetControlRotation();
}
else if (AActor* Owner = GetOwner())
{
CameraRot = Owner->GetActorRotation();
}
const FRotator YawOnly(0.f, CameraRot.Yaw, 0.f);
const FVector CamForward = FRotationMatrix(YawOnly).GetUnitAxis(EAxis::X);
const FVector CamRight = FRotationMatrix(YawOnly).GetUnitAxis(EAxis::Y);
// Top-down billiard view: every direction stays in the table (XY) plane.
// "up"/"down" map to the camera-forward axis (screen up/down), never world Z,
// so the ball slides across the table instead of lifting off it.
FVector CamDir = FVector::ZeroVector;
if (Direction == TEXT("up"))
CamDir = CamForward;
else if (Direction == TEXT("down"))
CamDir = -CamForward;
else if (Direction == TEXT("right"))
CamDir = CamRight;
else if (Direction == TEXT("left"))
CamDir = -CamRight;
else
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: Unknown direction '%s', defaulting to up."),
*Direction);
CamDir = CamForward;
}
// Snap to nearest world axis so the ball slides along the X/Y grid.
const FVector WorldAxes[] = {FVector(1, 0, 0), FVector(-1, 0, 0), FVector(0, 1, 0), FVector(0, -1, 0)};
float BestDot = -2.f;
FVector MoveDir = FVector::ZeroVector;
for (const FVector& Axis : WorldAxes)
{
const float Dot = FVector::DotProduct(CamDir, Axis);
if (Dot > BestDot)
{
BestDot = Dot;
MoveDir = Axis;
}
}
const FVector TargetLocation = SelectedActor->GetActorLocation() + MoveDir * Distance;
StartSmoothMove(SelectedActor, TargetLocation);
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Moving %s %s by %.1f cm (smooth)."),
*SelectedActor->GetName(),
*Direction,
Distance);
}
void UVoiceObjectEditorComponent::ExecuteRotate(const FString& Axis, float Angle)
{
if (!SelectedActor)
{
UE_LOG(
LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent::ExecuteRotate: no selected actor."));
return;
}
USceneComponent* RootComp = SelectedActor->GetRootComponent();
if (RootComp && RootComp->Mobility != EComponentMobility::Movable)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: Cannot rotate %s - actor is not movable!"),
*SelectedActor->GetName());
return;
}
// Commit any in-flight animation first so the undo snapshot captures the final
// transform of the prior command, not its pre-lerp start (m1).
FinishAnimationImmediately();
DisableOutlineOn(SelectedActor);
SaveOriginalStateIfNeeded(SelectedActor);
SaveUndoState(SelectedActor, EUndoActionType::Rotate);
FRotator DeltaRotation = FRotator::ZeroRotator;
if (Axis == TEXT("yaw"))
DeltaRotation.Yaw = Angle;
else if (Axis == TEXT("pitch"))
DeltaRotation.Pitch = Angle;
else if (Axis == TEXT("roll"))
DeltaRotation.Roll = Angle;
else
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: Unknown axis '%s', defaulting to yaw."),
*Axis);
DeltaRotation.Yaw = Angle;
}
const FRotator TargetRotation = SelectedActor->GetActorRotation() + DeltaRotation;
StartSmoothRotate(SelectedActor, TargetRotation);
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Rotating %s on %s by %.1f degrees (smooth)."),
*SelectedActor->GetName(),
*Axis,
Angle);
}
void UVoiceObjectEditorComponent::ExecuteApplyMaterial(const FString& MaterialName)
{
if (!SelectedActor)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent::ExecuteApplyMaterial: no selected actor."));
return;
}
TSoftObjectPtr<UMaterialInterface>* MatPtr = MaterialMap.Find(MaterialName);
if (!MatPtr)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: Unknown material '%s' (not in MaterialMap)."),
*MaterialName);
return;
}
UMaterialInterface* Material = MatPtr->LoadSynchronous();
if (!Material)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: Material '%s' failed to load (%s)."),
*MaterialName,
*MatPtr->ToSoftObjectPath().ToString());
return;
}
// Apply-material doesn't animate, but a prior Move/Rotate might still be mid-lerp.
// Commit it so the undo snapshot (below) captures the final transform (m1).
FinishAnimationImmediately();
DisableOutlineOn(SelectedActor);
SaveOriginalStateIfNeeded(SelectedActor);
SaveUndoState(SelectedActor, EUndoActionType::Material);
TArray<UMeshComponent*> MeshComponents;
SelectedActor->GetComponents<UMeshComponent>(MeshComponents);
for (UMeshComponent* MeshComp : MeshComponents)
{
if (MeshComp == nullptr) continue;
const int32 NumMaterials = MeshComp->GetNumMaterials();
for (int32 i = 0; i < NumMaterials; i++)
{
MeshComp->SetMaterial(i, Material);
}
}
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Applied material '%s' to %s."),
*MaterialName,
*SelectedActor->GetName());
}
void UVoiceObjectEditorComponent::ExecuteSetTime(float Hours)
{
float TimeOfDay = FMath::Fmod(Hours, 24.0f);
if (TimeOfDay < 0.f) TimeOfDay += 24.0f;
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Time set to %.2f (mod 24). Wire a Blueprint/subclass to apply to your sky "
"system."),
TimeOfDay);
}
// ============================================================================
// Undo
// ============================================================================
void UVoiceObjectEditorComponent::ExecuteUndo()
{
if (UndoStack.Num() == 0)
{
UE_LOG(LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent: Undo stack is empty!"));
return;
}
FinishAnimationImmediately();
FUndoEntry Entry = UndoStack.Pop();
AActor* Target = Entry.Actor.Get();
if (!IsValid(Target))
{
UE_LOG(
LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent: Undo target actor no longer valid."));
return;
}
switch (Entry.ActionType)
{
case EUndoActionType::Move: StartSmoothMove(Target, Entry.PrevLocation); break;
case EUndoActionType::Rotate: StartSmoothRotate(Target, Entry.PrevRotation); break;
case EUndoActionType::Material: RestoreMaterials(Target, Entry.PrevMaterials); break;
}
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Undo executed on %s. %d steps remaining."),
*Target->GetName(),
UndoStack.Num());
}
void UVoiceObjectEditorComponent::ExecuteUndoAll()
{
if (!SelectedActor)
{
UE_LOG(
LogBasemashVoiceCommander, Warning, TEXT("VoiceObjectEditorComponent: No object selected for undo_all!"));
return;
}
const TWeakObjectPtr<AActor> Key(SelectedActor);
const FOriginalState* State = OriginalStates.Find(Key);
if (!State)
{
UE_LOG(LogBasemashVoiceCommander,
Warning,
TEXT("VoiceObjectEditorComponent: No original state saved for %s!"),
*SelectedActor->GetName());
return;
}
FinishAnimationImmediately();
StartSmoothMoveAndRotate(SelectedActor, State->Location, State->Rotation);
RestoreMaterials(SelectedActor, State->Materials);
// Drop any undo entries for this actor — it's back to origin, so they'd be confusing.
AActor* SelectedRaw = SelectedActor;
UndoStack.RemoveAll([SelectedRaw](const FUndoEntry& Entry) { return Entry.Actor.Get() == SelectedRaw; });
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Reset %s to original state."),
*SelectedActor->GetName());
}
// ============================================================================
// Undo helpers
// ============================================================================
TArray<TArray<UMaterialInterface*>> UVoiceObjectEditorComponent::CaptureMaterials(AActor* Actor) const
{
TArray<TArray<UMaterialInterface*>> Result;
if (Actor == nullptr) return Result;
TArray<UMeshComponent*> MeshComponents;
Actor->GetComponents<UMeshComponent>(MeshComponents);
for (UMeshComponent* MeshComp : MeshComponents)
{
if (MeshComp == nullptr)
{
Result.AddDefaulted();
continue;
}
TArray<UMaterialInterface*> CompMaterials;
const int32 NumMats = MeshComp->GetNumMaterials();
for (int32 i = 0; i < NumMats; i++)
{
CompMaterials.Add(MeshComp->GetMaterial(i));
}
Result.Add(MoveTemp(CompMaterials));
}
return Result;
}
void UVoiceObjectEditorComponent::RestoreMaterials(AActor* Actor, const TArray<TArray<UMaterialInterface*>>& Materials)
{
if (Actor == nullptr) return;
TArray<UMeshComponent*> MeshComponents;
Actor->GetComponents<UMeshComponent>(MeshComponents);
const int32 NumComps = FMath::Min(MeshComponents.Num(), Materials.Num());
for (int32 CompIdx = 0; CompIdx < NumComps; CompIdx++)
{
UMeshComponent* MeshComp = MeshComponents[CompIdx];
if (MeshComp == nullptr) continue;
const TArray<UMaterialInterface*>& CompMats = Materials[CompIdx];
const int32 NumMats = FMath::Min(MeshComp->GetNumMaterials(), CompMats.Num());
for (int32 MatIdx = 0; MatIdx < NumMats; MatIdx++)
{
if (CompMats[MatIdx])
{
MeshComp->SetMaterial(MatIdx, CompMats[MatIdx]);
}
}
}
}
void UVoiceObjectEditorComponent::SaveOriginalStateIfNeeded(AActor* Actor)
{
if (Actor == nullptr) return;
const TWeakObjectPtr<AActor> Key(Actor);
if (OriginalStates.Contains(Key)) return;
FOriginalState State;
State.Location = Actor->GetActorLocation();
State.Rotation = Actor->GetActorRotation();
State.Materials = CaptureMaterials(Actor);
OriginalStates.Add(Key, MoveTemp(State));
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Saved original state for %s."),
*Actor->GetName());
}
void UVoiceObjectEditorComponent::SaveUndoState(AActor* Actor, EUndoActionType ActionType)
{
if (Actor == nullptr) return;
FUndoEntry Entry;
Entry.Actor = Actor;
Entry.PrevLocation = Actor->GetActorLocation();
Entry.PrevRotation = Actor->GetActorRotation();
Entry.PrevMaterials = CaptureMaterials(Actor);
Entry.ActionType = ActionType;
UndoStack.Add(MoveTemp(Entry));
if (MaxUndoSteps > 0 && UndoStack.Num() > MaxUndoSteps)
{
UndoStack.RemoveAt(0, UndoStack.Num() - MaxUndoSteps);
}
UE_LOG(LogBasemashVoiceCommander,
Log,
TEXT("VoiceObjectEditorComponent: Undo state saved (%d steps in stack)."),
UndoStack.Num());
}
// ============================================================================
// Smooth animation
// ============================================================================
void UVoiceObjectEditorComponent::FinishAnimationImmediately()
{
if (!IsAnimating()) return;
if (IsValid(AnimatingActor))
{
if (bAnimatingLocation) AnimatingActor->SetActorLocation(AnimTargetLocation);
if (bAnimatingRotation) AnimatingActor->SetActorRotation(AnimTargetRotation);
}
bAnimatingLocation = false;
bAnimatingRotation = false;
AnimatingActor = nullptr;
AnimElapsed = 0.f;
}
void UVoiceObjectEditorComponent::StartSmoothMove(AActor* Actor, const FVector& TargetLocation)
{
FinishAnimationImmediately();
if (!IsValid(Actor)) return;
AnimatingActor = Actor;
AnimStartLocation = Actor->GetActorLocation();
AnimTargetLocation = TargetLocation;
AnimStartRotation = Actor->GetActorRotation();
AnimTargetRotation = Actor->GetActorRotation();
AnimElapsed = 0.f;
bAnimatingLocation = true;
bAnimatingRotation = false;
}
void UVoiceObjectEditorComponent::StartSmoothRotate(AActor* Actor, const FRotator& TargetRotation)
{
FinishAnimationImmediately();
if (!IsValid(Actor)) return;
AnimatingActor = Actor;
AnimStartLocation = Actor->GetActorLocation();
AnimTargetLocation = Actor->GetActorLocation();
AnimStartRotation = Actor->GetActorRotation();
AnimTargetRotation = TargetRotation;
AnimElapsed = 0.f;
bAnimatingLocation = false;
bAnimatingRotation = true;
}
void UVoiceObjectEditorComponent::StartSmoothMoveAndRotate(AActor* Actor,
const FVector& TargetLocation,
const FRotator& TargetRotation)
{
FinishAnimationImmediately();
if (!IsValid(Actor)) return;
AnimatingActor = Actor;
AnimStartLocation = Actor->GetActorLocation();
AnimTargetLocation = TargetLocation;
AnimStartRotation = Actor->GetActorRotation();
AnimTargetRotation = TargetRotation;
AnimElapsed = 0.f;
bAnimatingLocation = true;
bAnimatingRotation = true;
}

View file

@ -0,0 +1,8 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Logging/LogMacros.h"
BASEMASHVOICECOMMANDER_API DECLARE_LOG_CATEGORY_EXTERN(LogBasemashVoiceCommander, Log, All);

View file

@ -0,0 +1,61 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "HAL/ThreadSafeBool.h"
#include "IPixelStreaming2AudioConsumer.h"
#include "UObject/NoExportTypes.h"
#include "PSVoiceAudioConsumer.generated.h"
/**
* Receives raw PCM audio from a Pixel Streaming 2 browser peer (microphone).
* Buffers audio while recording, then hands it off for voice command processing.
*/
UCLASS()
class BASEMASHVOICECOMMANDER_API UPSVoiceAudioConsumer : public UObject, public IPixelStreaming2AudioConsumer
{
GENERATED_BODY()
public:
//~ Begin IPixelStreaming2AudioConsumer interface
virtual void ConsumeRawPCM(const int16_t* AudioData, int InSampleRate, size_t NChannels, size_t NFrames) override;
virtual void OnAudioConsumerAdded() override;
virtual void OnAudioConsumerRemoved() override;
//~ End IPixelStreaming2AudioConsumer interface
/** Start buffering incoming audio data. */
void StartBuffering();
/** Stop buffering and retrieve the captured audio. Empties internal buffer. */
void StopBuffering(TArray<float>& OutPCMData, int32& OutSampleRate, int32& OutNumChannels);
[[nodiscard]] bool IsConnected() const { return bConnected; }
/** Reset first-data flag so we can verify audio flow after switching to a new peer. */
void ResetFirstDataFlag() { bGotFirstData = false; }
/**
* Sync fence used by the owning component during teardown. Taking + releasing the
* audio critical section ensures any in-flight ConsumeRawPCM has drained before
* the consumer/component is destroyed. Safe to call from the game thread after
* the sink's RemoveAudioConsumer().
*/
void Fence() { FScopeLock Lock(&CriticalSection); }
/** Fired when this consumer is removed from a sink (peer disconnected). Must be Unbind()'d only on game thread. */
FSimpleDelegate OnRemovedDelegate;
private:
TArray<float> Buffer;
bool bIsBuffering = false;
FCriticalSection CriticalSection;
int32 SampleRate = 0;
int32 NumChannels = 0;
// Touched by the audio thread (ConsumeRawPCM), the PS2 worker thread (OnAudioConsumer*),
// and the game thread (IsConnected / ResetFirstDataFlag) — must be atomic.
FThreadSafeBool bConnected = false;
FThreadSafeBool bGotFirstData = false;
};

View file

@ -0,0 +1,163 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "AudioCaptureCore.h"
#include "Components/ActorComponent.h"
#include "CoreMinimal.h"
#include "HAL/ThreadSafeBool.h"
#include "VoiceCaptureComponent.generated.h"
class UPSVoiceAudioConsumer;
class IPixelStreaming2AudioSink;
class UVoiceCommandSubsystem;
/**
* Audio source selection for UVoiceCaptureComponent.
* LocalMic - Capture from the default OS microphone via Audio::FAudioCapture.
* PixelStreaming - Capture from a remote browser peer microphone via PS2 audio consumer.
*/
UENUM(BlueprintType)
enum class EVoiceAudioSource : uint8
{
LocalMic UMETA(DisplayName = "Local Microphone"),
PixelStreaming UMETA(DisplayName = "Pixel Streaming (Browser Mic)")
};
/** Broadcast after successful transcription. Fired only on the component that initiated the request. */
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnVoiceCaptureTranscription, const FString&, Text);
/** Broadcast after LLM has parsed a command. Fired only on the component that initiated the request. */
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(
FOnVoiceCaptureCommand, const FString&, Action, const FString&, ParamsJson);
/** Broadcast on any error during the voice pipeline. Fired only on the component that initiated the request. */
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnVoiceCaptureError, const FString&, ErrorMessage);
/**
* UVoiceCaptureComponent
*
* Drop on any actor to give it voice-capture abilities. Handles:
* - Local mic or Pixel Streaming browser mic audio capture
* - Start/stop recording lifecycle with per-component event isolation
* - Stereo-to-mono mixdown, silence detection, WAV conversion
* - Round-trip to UVoiceCommandSubsystem (Groq STT + LLM)
*
* Transcription, parsed-command, and error events are delivered ONLY to the component
* that initiated the request (via FVoiceRequestCallbacks) multiple components can
* coexist without cross-talk.
*/
UCLASS(ClassGroup = ("VoiceCommander"), meta = (BlueprintSpawnableComponent, DisplayName = "Voice Capture"))
class BASEMASHVOICECOMMANDER_API UVoiceCaptureComponent : public UActorComponent
{
GENERATED_BODY()
public:
UVoiceCaptureComponent();
/** Which audio source to use at runtime. Change before BeginPlay for initial setup. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander")
EVoiceAudioSource AudioSource = EVoiceAudioSource::LocalMic;
/**
* Runtime-only per-component API key. Set via SetApiKey() or Blueprint at runtime.
* NOT serialized to asset use Project Settings for a persistent default.
* If non-empty, takes priority over Subsystem runtime key and Settings default.
*/
UPROPERTY(Transient, BlueprintReadWrite, Category = "Voice Commander", meta = (PasswordField = true))
FString ApiKeyOverride;
/** Blueprint helper to set the per-component API key at runtime. */
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void SetApiKey(const FString& Key) { ApiKeyOverride = Key; }
/** Silence threshold (max absolute float sample value). Below this, request is aborted with an error event. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Audio")
float SilenceThreshold = 0.001f;
/** If true, saves debug_recording.wav to ProjectSavedDir on every send. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Debug")
bool bSaveDebugRecording = false;
/** Begin capturing audio into the internal buffer. No-op if already recording. */
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void StartRecording();
/** Stop capturing, perform silence check + WAV encode, dispatch to the subsystem. */
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void StopRecordingAndSend();
/** True between StartRecording and StopRecordingAndSend. */
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander")
bool IsRecording() const { return bIsRecording; }
/**
* True if the configured audio source is ready to capture.
* - LocalMic: underlying Audio::FAudioCapture stream is open.
* - PixelStreaming: PS2 consumer is registered on an audio sink.
*/
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander")
bool IsAudioSourceReady() const;
/** Events fired after Groq round-trip — scoped to THIS component only. */
UPROPERTY(BlueprintAssignable, Category = "Voice Commander")
FOnVoiceCaptureTranscription OnTranscription;
UPROPERTY(BlueprintAssignable, Category = "Voice Commander")
FOnVoiceCaptureCommand OnCommandParsed;
UPROPERTY(BlueprintAssignable, Category = "Voice Commander")
FOnVoiceCaptureError OnError;
protected:
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
// --- Local mic state ---
Audio::FAudioCapture AudioCapture;
int32 CaptureSampleRate = 0;
int32 CaptureNumChannels = 0;
TArray<float> RecordedPCMData;
FCriticalSection RecordingCriticalSection;
// --- Pixel Streaming state ---
UPROPERTY()
UPSVoiceAudioConsumer* PSAudioConsumer = nullptr;
TWeakPtr<IPixelStreaming2AudioSink> PSAudioSink;
// Game-thread dominant, but read from the stale OnRemovedDelegate lambda path too.
FThreadSafeBool bPSAudioRegistered = false;
FTimerHandle PSRetryTimerHandle;
FString CurrentAudioPlayerId;
/** Handle for OnAudioTrackOpenNative so we can cleanly unsubscribe in EndPlay. */
FDelegateHandle PSAudioTrackOpenHandle;
// --- Recording state ---
// Written on the game thread (Start/Stop), read on the audio thread (LocalMic capture callback).
// FThreadSafeBool keeps the cross-thread read/write formally race-free without expanding the critical section.
FThreadSafeBool bIsRecording = false;
// --- Lifecycle ---
void InitLocalMic();
void CleanupLocalMic();
void InitPixelStreamingAudio();
void PSRetryRegistration();
void CleanupPixelStreamingAudio();
void SwitchAudioToPlayer(const FString& StreamerId, const FString& PlayerId);
// --- Utility ---
TArray<uint8> ConvertPCMToWav(const TArray<float>& PCMData, int32 SampleRate, int32 NumChannels) const;
// --- Subsystem callback glue ---
// Subsystem invokes these via FVoiceRequestCallbacks when this component is the request origin.
// Each one simply re-broadcasts the corresponding multicast so listeners on this component only see events for their own requests.
UFUNCTION()
void HandleTranscription(const FString& Text);
UFUNCTION()
void HandleCommandParsed(const FString& Action, const FString& ParamsJson);
UFUNCTION()
void HandleError(const FString& ErrorMessage);
};

View file

@ -0,0 +1,43 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "UObject/ScriptInterface.h"
#include "VoiceCommandExecutor.h"
#include "VoiceCommandAction.generated.h"
DECLARE_DYNAMIC_DELEGATE_RetVal_TwoParams(
bool, FVoiceActionHandler, const FString&, ActionName, const FString&, ParamsJson);
USTRUCT(BlueprintType)
struct BASEMASHVOICECOMMANDER_API FVoiceCommandAction
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander")
FString Name;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander")
FString Description;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander")
FString ParamSchema;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander")
TArray<FString> Examples;
// Prefer Handler if bound; otherwise ExecutorObject.
UPROPERTY(
EditAnywhere,
BlueprintReadWrite,
Category = "Voice Commander",
meta = (ToolTip = "Blueprint-bindable delegate path. Prefer this for one-off BP event graphs. For C++ executors that also need Blueprint override support, set ExecutorObject (IVoiceCommandExecutor interface) instead."))
FVoiceActionHandler Handler;
UPROPERTY(EditAnywhere,
BlueprintReadWrite,
Category = "Voice Commander",
meta = (ToolTip = "Native/C++ executor path. Set this when your action is implemented in a UObject that implements IVoiceCommandExecutor (supports Blueprint override via BlueprintNativeEvent). Handler takes priority when both are set."))
TScriptInterface<IVoiceCommandExecutor> ExecutorObject;
};

View file

@ -0,0 +1,28 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "UObject/Interface.h"
#include "VoiceCommandExecutor.generated.h"
UINTERFACE(MinimalAPI, Blueprintable)
class UVoiceCommandExecutor : public UInterface
{
GENERATED_BODY()
};
class BASEMASHVOICECOMMANDER_API IVoiceCommandExecutor
{
GENERATED_BODY()
public:
// Return true if this executor handled the action. Blueprint classes can implement this too.
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "Voice Commander")
bool ExecuteVoiceAction(const FString& ActionName, const FString& ParamsJson);
virtual bool ExecuteVoiceAction_Implementation(const FString& ActionName, const FString& ParamsJson)
{
return false;
}
};

View file

@ -0,0 +1,68 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Subsystems/GameInstanceSubsystem.h"
#include "VoiceCommandAction.h"
#include "VoiceCommandRegistry.generated.h"
UCLASS(BlueprintType)
class BASEMASHVOICECOMMANDER_API UVoiceCommandRegistry : public UGameInstanceSubsystem
{
GENERATED_BODY()
public:
virtual void Initialize(FSubsystemCollectionBase& Collection) override;
virtual void Deinitialize() override;
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry")
void RegisterAction(const FVoiceCommandAction& Action);
/** Convenience overload: register an action from Blueprint with a single call.
* Wraps the full FVoiceCommandAction use this when you don't need an
* IVoiceCommandExecutor ExecutorObject, just a Blueprint event handler.
* Returns true if registered (false if Name is empty or "none"). */
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry")
bool RegisterSimpleAction(const FString& Name,
const FString& Description,
const FString& ParamSchema,
const TArray<FString>& Examples,
const FVoiceActionHandler& Handler);
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry")
void UnregisterAction(const FString& ActionName);
// Replace only the Handler of an existing action, keep Name/Description/Examples/Schema.
// If action doesn't exist, does nothing. Returns true on success.
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry")
bool OverrideActionHandler(const FString& ActionName, const FVoiceActionHandler& NewHandler);
[[nodiscard]] UFUNCTION(BlueprintCallable, Category = "Voice Commander|Registry")
TArray<FVoiceCommandAction> GetRegisteredActions() const;
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Registry")
bool HasAction(const FString& ActionName) const;
/** Returns pointer to the registered FVoiceCommandAction for the given name, or null.
* The returned pointer is valid until the next Register/Unregister call. */
[[nodiscard]] const FVoiceCommandAction* FindAction(const FString& ActionName) const;
// Dispatches to Handler delegate first, then falls back to ExecutorObject.
// Returns true if either handler ran and returned true.
[[nodiscard]] bool DispatchAction(const FString& ActionName, const FString& ParamsJson);
// Builds the LLM system prompt describing all registered actions.
// Format matches the existing VoiceCommandSubsystem contract
// (LLM returns {"action": "...", "params": {...}} JSON only).
[[nodiscard]] FString BuildSystemPrompt() const;
// Convenience helper to get the registry from any UObject
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Registry", meta = (WorldContext = "WorldContextObject"))
static UVoiceCommandRegistry* Get(const UObject* WorldContextObject);
private:
UPROPERTY()
TMap<FString, FVoiceCommandAction> Actions;
};

View file

@ -0,0 +1,114 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Interfaces/IHttpRequest.h"
#include "Subsystems/GameInstanceSubsystem.h"
#include "VoiceCommandSubsystem.generated.h"
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnVoiceCommandParsed, const FString&, Action, const FString&, ParamsJson);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnTranscriptionReceived, const FString&, TranscribedText);
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnVoiceCommandError, const FString&, ErrorMessage);
/**
* Per-request callback payload. Components binding these only receive events
* from requests they initiated; other listeners see the global multicasts.
*/
USTRUCT(BlueprintType)
struct BASEMASHVOICECOMMANDER_API FVoiceRequestCallbacks
{
GENERATED_BODY()
// Weak ref guards against the caller UObject being destroyed mid-flight.
UPROPERTY()
TWeakObjectPtr<UObject> Context;
UPROPERTY()
FOnTranscriptionReceived OnTranscription;
UPROPERTY()
FOnVoiceCommandParsed OnCommandParsed;
UPROPERTY()
FOnVoiceCommandError OnError;
};
UCLASS(BlueprintType)
class BASEMASHVOICECOMMANDER_API UVoiceCommandSubsystem : public UGameInstanceSubsystem
{
GENERATED_BODY()
public:
virtual void Initialize(FSubsystemCollectionBase& Collection) override;
virtual void Deinitialize() override;
/** Runtime override for the Groq API key. Takes priority over Project Settings default.
* NOTE: Does not persist across GameInstance restarts (PIE stop/start, seamless travel).
* Set again after restart, or rely on the Settings default. */
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void SetApiKey(const FString& Key);
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander")
FString GetEffectiveApiKey(const FString& PerCallOverride) const;
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void SendAudioToGroq(const TArray<uint8>& WavAudioData,
const FString& PerCallApiKey,
const FVoiceRequestCallbacks& Callbacks);
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ParseCommandWithLLM(const FString& TranscribedText,
const FString& PerCallApiKey,
const FVoiceRequestCallbacks& Callbacks);
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander")
bool IsProcessing() const { return bIsProcessing; }
UPROPERTY(BlueprintAssignable, Category = "Voice Commander")
FOnVoiceCommandParsed OnAnyCommandParsed;
UPROPERTY(BlueprintAssignable, Category = "Voice Commander")
FOnTranscriptionReceived OnAnyTranscriptionReceived;
UPROPERTY(BlueprintAssignable, Category = "Voice Commander")
FOnVoiceCommandError OnAnyError;
private:
FString RuntimeApiKey;
bool bIsProcessing = false;
FVoiceRequestCallbacks CurrentCallbacks;
TArray<uint8> CurrentAudioData;
FString CurrentTranscription;
FDateTime RequestStartTime;
FDateTime STTCompleteTime;
// True while OnSTTResponse is chaining into ParseCommandWithLLM. Lets the LLM path
// skip the bIsProcessing guard and preserve timing set by the audio path.
bool bChainedFromSTT = false;
// Weak refs to every in-flight HTTP request we dispatched. Cleared on FinishRequest;
// iterated in Deinitialize to cancel outstanding work before teardown so BindUObject
// callbacks never fire on a destroyed subsystem.
TArray<TWeakPtr<IHttpRequest, ESPMode::ThreadSafe>> InFlightRequests;
void OnSTTResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess);
void OnLLMResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess);
FString BuildSystemPrompt() const;
void SendLogToPanel(const FString& Action,
const FString& ParamsJson,
const FString& LLMRaw,
const FString& Error = TEXT(""));
void OnLogPanelResponse(FHttpRequestPtr Req, FHttpResponsePtr Res, bool bSuccess);
void FireTranscription(const FString& Text);
void FireCommandParsed(const FString& Action, const FString& ParamsJson);
void FireError(const FString& ErrorMessage);
void FinishRequest();
};

View file

@ -0,0 +1,104 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "CoreMinimal.h"
#include "Engine/DeveloperSettings.h"
#include "UObject/Object.h"
#include "VoiceCommanderSettings.generated.h"
/**
* Project-level settings for the Basemash Voice Commander plugin.
* Surfaces under Project Settings -> Plugins -> Voice Commander.
* Values are persisted to DefaultGame.ini (config=Game, defaultconfig).
*
* NOTE: Blueprint users should treat Settings as read-only. Do NOT write to the CDO
* from Blueprint (e.g. via Get() -> SetMember) this can silently persist to
* DefaultGame.ini on editor save. To change the API key at runtime, go through
* UVoiceCommandSubsystem::SetApiKey() instead.
*/
UCLASS(config = Game, defaultconfig, BlueprintType, meta = (DisplayName = "Voice Commander"))
class BASEMASHVOICECOMMANDER_API UVoiceCommanderSettings : public UDeveloperSettings
{
GENERATED_BODY()
public:
// --- API ---
// Fallback Groq API key used when no per-user/runtime override is set.
UPROPERTY(
config,
EditAnywhere,
Category = "API",
meta = (DisplayName = "Groq API Key (Default)",
PasswordField = true,
ToolTip = "Development default. Persists to DefaultGame.ini and SHIPS IN PACKAGED BUILDS. For production, leave empty here and set at runtime via UVoiceCommandSubsystem::SetApiKey() from an environment variable, OS keychain, or backend proxy."))
FString GroqApiKey;
UPROPERTY(config, EditAnywhere, Category = "API", meta = (DisplayName = "STT Model"))
FString SttModel = TEXT("whisper-large-v3-turbo");
UPROPERTY(config, EditAnywhere, Category = "API", meta = (DisplayName = "LLM Model"))
FString LlmModel = TEXT("llama-3.3-70b-versatile");
UPROPERTY(config, EditAnywhere, Category = "API", meta = (DisplayName = "STT Language"))
FString SttLanguage = TEXT("en");
// --- Logging ---
UPROPERTY(config, EditAnywhere, Category = "Logging", meta = (DisplayName = "Log Panel URL"))
FString LogPanelUrl = TEXT("http://localhost:3001/api/log");
UPROPERTY(config, EditAnywhere, Category = "Logging", meta = (DisplayName = "Log To Panel"))
bool bLogToPanel = true;
// --- Builtin Actions ---
UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Move"))
bool bEnableBuiltinMove = true;
UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Rotate"))
bool bEnableBuiltinRotate = true;
UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Apply Material"))
bool bEnableBuiltinApplyMaterial = true;
UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Undo"))
bool bEnableBuiltinUndo = true;
UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Undo All"))
bool bEnableBuiltinUndoAll = true;
UPROPERTY(config, EditAnywhere, Category = "Builtin Actions", meta = (DisplayName = "Enable Set Time"))
bool bEnableBuiltinSetTime = true;
/** Convenience accessor for C++ and Blueprint callers. Returns the CDO.
* Blueprint users: the const is stripped at the BP layer do NOT write
* through this pointer. Use SetApiKey on the subsystem for runtime key
* changes, and use the GetX() helpers below for read access. */
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static const UVoiceCommanderSettings* Get();
// --- Safe per-property BP getters (return by copy, cannot mutate the CDO) ---
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static FString GetGroqApiKey();
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static FString GetSttModel();
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static FString GetLlmModel();
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static FString GetSttLanguage();
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static FString GetLogPanelUrl();
[[nodiscard]] UFUNCTION(BlueprintPure, Category = "Voice Commander|Settings")
static bool GetLogToPanel();
// UDeveloperSettings interface
virtual FName GetCategoryName() const override;
};

View file

@ -0,0 +1,229 @@
// Copyright (c) Basemash. All Rights Reserved.
#pragma once
#include "Components/ActorComponent.h"
#include "CoreMinimal.h"
#include "Engine/EngineTypes.h"
#include "UObject/SoftObjectPtr.h"
#include "VoiceCommandExecutor.h"
#include "VoiceObjectEditorComponent.generated.h"
class UCameraComponent;
class UPostProcessComponent;
class UMaterialInterface;
class UVoiceCaptureComponent;
// Tip akcije za undo sistem - koji tip operacije je izveden
UENUM()
enum class EUndoActionType : uint8
{
Move,
Rotate,
Material
};
// Undo stack entry. Raw TArray<TArray<>> can't be USTRUCT/UPROPERTY-reflected,
// so we leave these as plain C++ structs (no USTRUCT), matching the original code.
struct FUndoEntry
{
TWeakObjectPtr<AActor> Actor;
FVector PrevLocation = FVector::ZeroVector;
FRotator PrevRotation = FRotator::ZeroRotator;
TArray<TArray<UMaterialInterface*>> PrevMaterials;
EUndoActionType ActionType = EUndoActionType::Move;
};
// Originalno stanje objekta - cuva se pri prvom kontaktu, za "vrati u originalno"
struct FOriginalState
{
FVector Location = FVector::ZeroVector;
FRotator Rotation = FRotator::ZeroRotator;
TArray<TArray<UMaterialInterface*>> Materials;
};
/** Fired whenever the selected actor changes (raycast hit, BP-driven SetSelectedActor, or ClearSelection).
* PreviousActor / NewActor may be null. Use this to sync HUD/UI/SFX to selection state. */
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnSelectionChanged, AActor*, PreviousActor, AActor*, NewActor);
/**
* UVoiceObjectEditorComponent
*
* Opinionated sample executor that implements the "object editor" voice actions
* (move / rotate / apply_material / set_time / undo / undo_all) on top of the
* BasemashVoiceCommander pipeline.
*
* Drop onto any pawn-ish actor alongside a UVoiceCaptureComponent to get
* voice-driven manipulation of any movable actor picked by the selection raycast.
*
* Registers itself as an IVoiceCommandExecutor with UVoiceCommandRegistry for
* every enabled built-in action from UVoiceCommanderSettings. Registry dispatch
* is the only path into this component does not bind to
* UVoiceCaptureComponent::OnCommandParsed (dual-binding caused double-dispatch
* on move/rotate).
*/
UCLASS(ClassGroup = ("VoiceCommander"), meta = (BlueprintSpawnableComponent, DisplayName = "Voice Object Editor"))
class BASEMASHVOICECOMMANDER_API UVoiceObjectEditorComponent : public UActorComponent, public IVoiceCommandExecutor
{
GENERATED_BODY()
public:
UVoiceObjectEditorComponent();
// --- Wiring ---
/** Optional reference to the capture component on the same actor. Not used internally by the editor component; reserved for BP/subclass queries (e.g., IsAudioSourceReady(), StartRecording from BP). */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander")
UVoiceCaptureComponent* CaptureComponent = nullptr;
/** Optional explicit camera for selection raycast. If null: auto-find UCameraComponent on owner, else fall back to PlayerController view. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection")
UCameraComponent* SelectionCameraComponent = nullptr;
// --- Selection config ---
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection")
float SelectionTraceDistance = 10000.f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection")
TEnumAsByte<ECollisionChannel> SelectionTraceChannel = ECC_Visibility;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Voice Commander|Selection")
AActor* SelectedActor = nullptr;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection")
bool bEnforceMovable = false;
/** Optional. If set, this actor is auto-selected at BeginPlay (e.g. the cue ball). Ideal for
* Blueprint-only projects with no character: drop this component's actor in the level and point
* this at the object voice should control. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection")
TSoftObjectPtr<AActor> InitialSelection;
/** If true, DoSelect() (the camera raycast) is disabled so the selection cannot change.
* Combine with InitialSelection to lock voice control onto a single object. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Selection")
bool bLockSelection = false;
/** Fires when SelectedActor changes (DoSelect raycast, SetSelectedActor BP call, or ClearSelection).
* Both arguments may be null (e.g., NewActor is null when selection is cleared). */
UPROPERTY(BlueprintAssignable, Category = "Voice Commander|Selection")
FOnSelectionChanged OnSelectionChanged;
// --- Outline post process ---
/** If true, a UPostProcessComponent will be created on the owner at BeginPlay and driven by OutlineMaterial. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Outline")
bool bCreateOutlineComponent = true;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Outline")
TSoftObjectPtr<UMaterialInterface> OutlineMaterial;
// --- Material map (replaces hardcoded path list) ---
/** Map from voice-recognized material name (e.g. "Mat_Cigla") to material asset reference. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Materials")
TMap<FString, TSoftObjectPtr<UMaterialInterface>> MaterialMap;
// --- Animation config ---
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Animation")
float AnimDuration = 0.8f;
// --- Undo config ---
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice Commander|Undo")
int32 MaxUndoSteps = 20;
// --- Blueprint-callable actions ---
/** Camera-raycast selection: line-trace from SelectionCameraComponent (or auto-found camera) forward
* by SelectionTraceDistance, and select the first hit actor with at least one Movable scene component.
* Wraps SetSelectedActorInternal with bEnforceMovable=true. */
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Selection")
void DoSelect();
/** Programmatic selection setter — bypass raycast and select ANY actor (Movable or not).
* Use this for click-to-select, tag/class lookup, UI list pick, custom proximity rules, etc.
* Pass nullptr to clear selection (equivalent to ClearSelection). Fires OnSelectionChanged
* unless the actor is already selected. */
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Selection")
void SetSelectedActor(AActor* NewActor);
UFUNCTION(BlueprintCallable, Category = "Voice Commander|Selection")
void ClearSelection();
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ExecuteMove(const FString& Direction, float Distance);
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ExecuteRotate(const FString& Axis, float Angle);
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ExecuteApplyMaterial(const FString& MaterialName);
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ExecuteSetTime(float Hours);
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ExecuteUndo();
UFUNCTION(BlueprintCallable, Category = "Voice Commander")
void ExecuteUndoAll();
// --- IVoiceCommandExecutor ---
virtual bool ExecuteVoiceAction_Implementation(const FString& ActionName, const FString& ParamsJson) override;
protected:
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void
TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
// --- Undo + state tracking ---
TArray<FUndoEntry> UndoStack;
TMap<TWeakObjectPtr<AActor>, FOriginalState> OriginalStates;
void SaveUndoState(AActor* Actor, EUndoActionType ActionType);
void SaveOriginalStateIfNeeded(AActor* Actor);
[[nodiscard]] TArray<TArray<UMaterialInterface*>> CaptureMaterials(AActor* Actor) const;
void RestoreMaterials(AActor* Actor, const TArray<TArray<UMaterialInterface*>>& Materials);
// --- Smooth animation ---
bool bAnimatingLocation = false;
bool bAnimatingRotation = false;
inline bool IsAnimating() const { return bAnimatingRotation || bAnimatingLocation; }
FVector AnimStartLocation = FVector::ZeroVector;
FVector AnimTargetLocation = FVector::ZeroVector;
FRotator AnimStartRotation = FRotator::ZeroRotator;
FRotator AnimTargetRotation = FRotator::ZeroRotator;
float AnimElapsed = 0.f;
UPROPERTY()
AActor* AnimatingActor = nullptr;
void StartSmoothMove(AActor* Actor, const FVector& TargetLocation);
void StartSmoothRotate(AActor* Actor, const FRotator& TargetRotation);
void StartSmoothMoveAndRotate(AActor* Actor, const FVector& TargetLocation, const FRotator& TargetRotation);
void FinishAnimationImmediately();
// --- Outline PP (created at runtime if bCreateOutlineComponent) ---
UPROPERTY()
UPostProcessComponent* OutlinePostProcess = nullptr;
// --- Selection helpers ---
[[nodiscard]] bool GetSelectionCameraRay(FVector& OutStart, FVector& OutEnd) const;
/** Single source of truth for selection state changes. Both DoSelect and SetSelectedActor
* funnel through this. Handles Movable enforcement (only when bEnforceMovable=true,
* set by DoSelect's raycast path), CustomDepth highlight toggle, and OnSelectionChanged
* broadcast. No-op if NewActor matches current SelectedActor. */
void SetSelectedActorInternal(AActor* NewActor, bool bRequireMovable);
// --- Registration bookkeeping ---
// Names of actions we registered so we can cleanly unregister on EndPlay
// without stepping on actions registered by other executors.
TArray<FString> RegisteredActionNames;
};

View file

@ -0,0 +1,215 @@
@Rem Copyright Epic Games, Inc. All Rights Reserved.
@echo off
@Rem Set script location as working directory for commands.
pushd "%~dp0"
@Rem Turned on delayed expansion so we have variables evaluated in nested scope
setlocal EnableDelayedExpansion
@Rem Unset all our variables as these persist between cmd sessions if cmd not closed.
SET "PSInfraOrg="
SET "PSInfraRepo="
SET "PSInfraTagOrBranch="
SET "ReleaseVersion="
SET "ReleaseUrl="
SET "IsTag="
SET "RefType="
SET "Url="
SET "DownloadVersion="
SET "FlagPassed="
:arg_loop_start
SET ARG=%1
if DEFINED ARG (
if "%ARG%"=="/h" (
goto print_help
)
if "%ARG%"=="/v" (
SET UEVersion=%2
SET FlagPassed=1
SHIFT
)
if "%ARG%"=="/b" (
SET PSInfraTagOrBranch=%2
SET IsTag=0
SET FlagPassed=1
SHIFT
)
if "%ARG%"=="/t" (
SET PSInfraTagOrBranch=%2
SET IsTag=1
SET FlagPassed=1
SHIFT
)
if "%ARG%"=="/r" (
SET "ReleaseVersion=%2"
SET "ReleaseUrl=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/!ReleaseVersion!/!ReleaseVersion!.zip"
SET IsTag=0
SET FlagPassed=1
SHIFT
)
SHIFT
goto arg_loop_start
)
@Rem Name and version of ps-infra that we are downloading
SET PSInfraOrg=EpicGamesExt
SET PSInfraRepo=PixelStreamingInfrastructure
@Rem If a UE version is supplied set the right branch or tag to fetch for that version of UE
if DEFINED UEVersion (
if "%UEVersion%"=="4.26" (
SET PSInfraTagOrBranch=UE4.26
SET IsTag=0
)
if "%UEVersion%"=="4.27" (
SET PSInfraTagOrBranch=UE4.27
SET IsTag=0
)
if "%UEVersion%"=="5.0" (
SET PSInfraTagOrBranch=UE5.0
SET IsTag=0
)
if "%UEVersion%"=="5.1" (
SET PSInfraTagOrBranch=UE5.1
SET IsTag=0
)
if "%UEVersion%"=="5.2" (
SET PSInfraTagOrBranch=UE5.2
SET IsTag=0
)
if "%UEVersion%"=="5.3" (
SET PSInfraTagOrBranch=UE5.3
SET IsTag=0
)
if "%UEVersion%"=="5.4" (
SET PSInfraTagOrBranch=UE5.4
SET IsTag=0
)
if "%UEVersion%"=="5.5" (
SET PSInfraTagOrBranch=UE5.5
SET IsTag=0
)
if "%UEVersion%"=="5.6" (
SET PSInfraTagOrBranch=UE5.6
SET IsTag=0
)
if "%UEVersion%"=="5.7" (
SET PSInfraTagOrBranch=UE5.7
SET IsTag=0
)
)
@Rem If no arguments select a specific version, fetch the appropriate default
if NOT DEFINED PSInfraTagOrBranch (
SET PSInfraTagOrBranch=UE5.7
SET IsTag=0
)
echo Tag or branch: !PSInfraTagOrBranch!
@Rem Whether the named reference is a tag or a branch affects the Url we fetch it on
if %IsTag%==1 (
SET RefType=tags
) else (
SET RefType=heads
)
@Rem We have a branch, no user-specified release, then check repo for the presence of a RELEASE_VERSION file in the current branch
if %IsTag%==0 (
if NOT DEFINED ReleaseUrl (
@Rem We don't want to auto-set the release version if the user passed an explicit flag.
if NOT DEFINED FlagPassed (
FOR /F "tokens=* USEBACKQ" %%F IN (`curl -s -f -L https://raw.githubusercontent.com/EpicGamesExt/PixelStreamingInfrastructure/%PSInfraTagOrBranch%/RELEASE_VERSION`) DO (
SET "ReleaseVersion=!PSInfraTagOrBranch!-%%F"
SET "ReleaseUrl=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/!ReleaseVersion!/!ReleaseVersion!.zip"
)
)
)
)
@Rem Set our DownloadVersion here as we use this to check the contents of our DOWNLOAD_VERSION file shortly.
SET "DownloadVersion=%PSInfraTagOrBranch%"
if DEFINED ReleaseVersion (
SET "DownloadVersion=!ReleaseVersion!"
echo Release: !ReleaseVersion!
)
@Rem Check for the existence of a DOWNLOAD_VERSION file and if found, check its contents against our %DownloadVersion%
if exist DOWNLOAD_VERSION (
@Rem Read DOWNLOAD_VERSION file into variable
FOR /F "delims=" %%F IN ( DOWNLOAD_VERSION ) DO (
SET "PreviousDownloadVersion=%%F"
@Rem Remove whitespace
SET "PreviousDownloadVersion=!PreviousDownloadVersion: =!"
)
if !DownloadVersion! == !PreviousDownloadVersion! (
echo Downloaded version ^(!DownloadVersion!^) of PS infra matches release version ^(!PreviousDownloadVersion!^)...skipping install.
goto :EOF
) else (
echo There is a newer released version ^(!DownloadVersion!^) - had ^(!PreviousDownloadVersion!^), downloading...
)
) else (
echo DOWNLOAD_VERSION file not found...beginning ps-infra download.
)
@Rem By default set the download url to the .zip of the branch
SET Url=https://github.com/%PSInfraOrg%/%PSInfraRepo%/archive/refs/%RefType%/%PSInfraTagOrBranch%.zip
@Rem If we have a ReleaseUrl then set it to our download url
if DEFINED ReleaseUrl (
SET Url=!ReleaseUrl!
)
@Rem Download ps-infra and follow redirects.
echo Attempting downloading Pixel Streaming infrastructure from: !Url!
curl -L !Url! > ps-infra.zip
@Rem Unarchive the .zip
tar -xmf ps-infra.zip || echo bad archive, contents: && type ps-infra.zip && exit 0
@Rem Remove old infra
if exist Frontend\ ( rmdir /s /q Frontend )
if exist Matchmaker\ ( rmdir /s /q Matchmaker )
if exist SignallingWebserver\ ( rmdir /s /q SignallingWebserver )
if exist SFU\ ( rmdir /s /q SFU )
@Rem Rename the extracted, versioned, directory
for /d %%i in ("PixelStreamingInfrastructure-*") do (
for /d %%j in ("%%i/*") do (
echo "%%i\%%j"
move "%%i\%%j" .
)
for %%j in ("%%i/*") do (
echo "%%i\%%j"
move "%%i\%%j" .
)
echo "%%i"
rmdir /s /q "%%i"
)
@Rem Delete the downloaded zip
del ps-infra.zip
@Rem Create a DOWNLOAD_VERSION file, which we use as a comparison file to check if we should auto upgrade when these scripts are run again
echo %DownloadVersion%> DOWNLOAD_VERSION
goto :EOF
:print_help
echo.
echo Tool for fetching PixelStreaming Infrastructure. If no flags are set specifying a version to fetch,
echo the recommended version will be chosen as a default.
echo.
echo Usage:
echo %~n0%~x0 [^/h] [^/v ^<UE version^>] [^/b ^<branch^>] [^/t ^<tag^>] [^/r ^<release^>]
echo Where:
echo /v Specify a version of Unreal Engine to download the recommended release for
echo /b Specify a specific branch for the tool to download from repo
echo /t Specify a specific tag for the tool to download from repo
echo /r Specify a specific release url path e.g. https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/${RELEASE_HERE}.zip
echo /h Display this help message
goto :EOF

View file

@ -0,0 +1,199 @@
#!/bin/bash
# Copyright Epic Games, Inc. All Rights Reserved.
BASH_LOCATION="$(cd -P -- "$(dirname -- "$0")" && pwd -P)"
pushd "${BASH_LOCATION}" > /dev/null
print_help() {
echo "
Tool for fetching PixelStreaming Infrastructure. If no flags are set specifying a version to fetch,
the recommended version will be chosen as a default.
Usage:
${0} [-h] [-v <UE version>] [-b <branch>] [-t <tag>]
Where:
-v Specify a version of Unreal Engine to download the recommended
release for
-b Specify a specific branch for the tool to download from repo
-t Specify a specific tag for the tool to download from repo
-r Specify a specific release url path e.g. https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/<RELEASE_VERSION>/<RELEASE_VERSION>.zip
-h Display this help message
"
exit 1
}
# Set all default variables (e.g. # Name and version of ps-infra that we are downloading)
PSInfraOrg=EpicGamesExt
PSInfraRepo=PixelStreamingInfrastructure
PSInfraTagOrBranch=UE5.7
RefType=heads
IsTag=0
ReleaseUrlBase=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download
# Unset any variables that don't have defaults that we use that may have persisted between bash terminals.
unset Url
unset DownloadVersion
unset FlagPassed
unset ReleaseVersion
unset ReleaseUrl
while (($#)) ; do
case "$1" in
-h ) print_help;;
-v ) UEVersion="$2"; FlagPassed=1; shift 2;;
-b ) PSInfraTagOrBranch="$2"; FlagPassed=1; IsTag=0; shift 2;;
-t ) PSInfraTagOrBranch="$2"; FlagPassed=1; IsTag=1; shift 2;;
-r ) ReleaseVersion="$2"; FlagPassed=1; IsTag=0; ReleaseUrl=$ReleaseUrlBase/$ReleaseVersion/$ReleaseVersion.tar.gz; shift 2;;
* ) echo "Unknown command: $1"; shift;;
esac
done
# If a UE version is supplied set the right branch or tag to fetch for that version of UE
if [ ! -z "$UEVersion" ]
then
if [ "$UEVersion" = "4.26" ]
then
PSInfraTagOrBranch=UE4.26
IsTag=0
fi
if [ "$UEVersion" = "4.27" ]
then
PSInfraTagOrBranch=UE4.27
IsTag=0
fi
if [ "$UEVersion" = "5.0" ]
then
PSInfraTagOrBranch=UE5.0
IsTag=0
fi
if [ "$UEVersion" = "5.1" ]
then
PSInfraTagOrBranch=UE5.1
IsTag=0
fi
if [ "$UEVersion" = "5.2" ]
then
PSInfraTagOrBranch=UE5.2
IsTag=0
fi
if [ "$UEVersion" = "5.3" ]
then
PSInfraTagOrBranch=UE5.3
IsTag=0
fi
if [ "$UEVersion" = "5.4" ]
then
PSInfraTagOrBranch=UE5.4
IsTag=0
fi
if [ "$UEVersion" = "5.5" ]
then
PSInfraTagOrBranch=UE5.5
IsTag=0
fi
if [ "$UEVersion" = "5.6" ]
then
PSInfraTagOrBranch=UE5.6
IsTag=0
fi
if [ "$UEVersion" = "5.7" ]
then
PSInfraTagOrBranch=UE5.7
IsTag=0
fi
fi
# If no arguments select a specific version, fetch the appropriate default
if [ -z "$PSInfraTagOrBranch" ]
then
PSInfraTagOrBranch=UE5.7
IsTag=0
fi
echo "Tag or branch: $PSInfraTagOrBranch"
# Whether the named reference is a tag or a branch affects the Url we fetch it on
if [ "$IsTag" -eq 1 ]
then
RefType=tags
else
RefType=heads
fi
# We have a branch, no user-specified release, then check repo for the presence of a RELEASE_VERSION file in the current branch.
if [ "$IsTag" -eq 0 ] && [ -z "$ReleaseUrl" ] && [ -z "$FlagPassed" ]
then
RelUrl=https://raw.githubusercontent.com/EpicGamesExt/PixelStreamingInfrastructure/$PSInfraTagOrBranch/RELEASE_VERSION
if curl --output /dev/null --silent -r 0-0 --fail "$RelUrl"; then
ReleaseVersion="$PSInfraTagOrBranch-$(curl -L -s $RelUrl)"
ReleaseUrl=https://github.com/EpicGamesExt/PixelStreamingInfrastructure/releases/download/$ReleaseVersion/$ReleaseVersion.tar.gz
echo "Valid RELEASE_VERSION file found in Github repo at $RelUrl"
else
echo "RELEASE_VERSION file does not exist at: $RelUrl"
fi
else
echo "Skipping downloading RELEASE_VERSION file."
fi
#Set our DownloadVersion here as we use this to check the contents of our DOWNLOAD_VERSION file shortly.
DownloadVersion="$PSInfraTagOrBranch"
if [ ! -z "$ReleaseVersion" ]
then
DownloadVersion="$ReleaseVersion"
echo "Release: $ReleaseVersion"
fi
#Rem Check for the existence of a DOWNLOAD_VERSION file and if found, check its contents against our $DownloadVersion
if test -f DOWNLOAD_VERSION;
then
PREVIOUS_DOWNLOAD_VERSION=$(cat DOWNLOAD_VERSION)
if [ "$DownloadVersion" = "$PREVIOUS_DOWNLOAD_VERSION" ]
then
echo "Downloaded version ($DownloadVersion) of PS infra matches release version ($PREVIOUS_DOWNLOAD_VERSION)...skipping install."
exit 0
else
echo "There is a newer released version ($DownloadVersion) - had ($PREVIOUS_DOWNLOAD_VERSION), downloading..."
#Remove old infra
rm -rf Frontend
rm -rf Matchmaker
rm -rf SignallingWebServer
rm -rf SFU
fi
else
echo "DOWNLOAD_VERSION file not found..."
fi
# Pre-download - Set the download url to the .zip of the branch
Url=https://github.com/$PSInfraOrg/$PSInfraRepo/archive/refs/$RefType/$PSInfraTagOrBranch.tar.gz
#Check if ReleaseUrl is valid by CURLing with fail fast and if success then set it to our download url
if [ ! -z "$ReleaseUrl" ]
then
echo "Checking if release url is valid, url: $ReleaseUrl"
if curl --output /dev/null --silent -r 0-0 --fail "$ReleaseUrl"; then
echo "Valid release url at: $ReleaseUrl"
Url=$ReleaseUrl
else
echo "Invalid Github release url: $ReleaseUrl"
exit 1
fi
fi
# Download - download ps-infra and follow redirects.
echo "Beginning ps-infra download from: $Url"
curl -L $Url > ps-infra.tar.gz
# Unarchive the .tar
tar -xmf ps-infra.tar.gz || $(echo "bad archive, contents:" && head --lines=20 ps-infra.tar.gz && exit 0)
# Move the server folders into the current directory (WebServers) and delete the original directory
mv PixelStreamingInfrastructure-*/* .
# Copy any files and folders beginning with dot (ignored by * glob) and discard errors regarding to not being able to move "." and ".."
mv PixelStreamingInfrastructure-*/.* . 2>/dev/null || :
rm -rf PixelStreamingInfrastructure-*
# Delete the downloaded tar
rm ps-infra.tar.gz
#Create a DOWNLOAD_VERSION file, which we use as a comparison file to check if we should auto upgrade when these scripts are run again
echo "$DownloadVersion" >| DOWNLOAD_VERSION
exit 0

View file

@ -0,0 +1,12 @@
using UnrealBuildTool;
public class PSC_SharkTableTarget : TargetRules
{
public PSC_SharkTableTarget(TargetInfo Target) : base(Target)
{
DefaultBuildSettings = BuildSettingsVersion.Latest;
IncludeOrderVersion = EngineIncludeOrderVersion.Latest;
Type = TargetType.Game;
ExtraModuleNames.Add("PSC_SharkTable");
}
}

View file

@ -0,0 +1,10 @@
using UnrealBuildTool;
public class PSC_SharkTable : ModuleRules
{
public PSC_SharkTable(ReadOnlyTargetRules Target) : base(Target)
{
PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
PrivateDependencyModuleNames.Add("Core");
}
}

View file

@ -0,0 +1,4 @@
#include "CoreTypes.h"
#include "Modules/ModuleManager.h"
IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, PSC_SharkTable, "PSC_SharkTable");

View file

@ -0,0 +1,12 @@
using UnrealBuildTool;
public class PSC_SharkTableEditorTarget : TargetRules
{
public PSC_SharkTableEditorTarget(TargetInfo Target) : base(Target)
{
DefaultBuildSettings = BuildSettingsVersion.Latest;
IncludeOrderVersion = EngineIncludeOrderVersion.Latest;
Type = TargetType.Editor;
ExtraModuleNames.Add("PSC_SharkTable");
}
}

View file

@ -0,0 +1,242 @@
# UE5 Blueprint-to-C++ Project Conversion & Packaging Fix Guide
## Problem
When you copy an Unreal Engine 5 Blueprint-only project to a new machine, "Package Project" does nothing -- no error in the UI, no output. The log shows:
```
LogProjectPackagingSettings: UProjectPackagingSettings FindBestTargetInfo for '' resulted in null FTargetInfo*.
Listing targets that were searched:
End of target list.
```
This happens because:
1. The project has no `Source/` directory (BP-only projects auto-generate source in `Intermediate/Source/` during packaging)
2. Stale configs from the original machine override local settings
3. The editor can't discover build targets without compiled `.target` metadata files
## Step-by-Step Fix
### 1. Add Modules to .uproject
The `.uproject` file MUST declare a Modules section. Without it, the editor won't look for `.target` files in `Binaries/` and the target list will always be empty.
Add this before the `"Plugins"` array:
```json
{
"FileVersion": 3,
"EngineAssociation": "5.7",
"Modules": [
{
"Name": "YourProjectName",
"Type": "Runtime",
"LoadingPhase": "Default"
}
],
"Plugins": [
...
]
}
```
> **Critical:** Without this section, even if `.target` files exist in `Binaries/`, the editor's `FindBestTargetInfo` will return an empty target list.
### 2. Create Source/ Directory
Create the following structure at the project root:
```
Source/
YourProjectName.Target.cs (Game target)
YourProjectNameEditor.Target.cs (Editor target)
YourProjectName/
YourProjectName.Build.cs (Module build rules)
YourProjectName.cpp (Module implementation)
```
**YourProjectName.Target.cs:**
```csharp
using UnrealBuildTool;
public class YourProjectNameTarget : TargetRules
{
public YourProjectNameTarget(TargetInfo Target) : base(Target)
{
DefaultBuildSettings = BuildSettingsVersion.Latest;
IncludeOrderVersion = EngineIncludeOrderVersion.Latest;
Type = TargetType.Game;
ExtraModuleNames.Add("YourProjectName");
}
}
```
**YourProjectNameEditor.Target.cs:**
```csharp
using UnrealBuildTool;
public class YourProjectNameEditorTarget : TargetRules
{
public YourProjectNameEditorTarget(TargetInfo Target) : base(Target)
{
DefaultBuildSettings = BuildSettingsVersion.Latest;
IncludeOrderVersion = EngineIncludeOrderVersion.Latest;
Type = TargetType.Editor;
ExtraModuleNames.Add("YourProjectName");
}
}
```
**YourProjectName/YourProjectName.Build.cs:**
```csharp
using UnrealBuildTool;
public class YourProjectName : ModuleRules
{
public YourProjectName(ReadOnlyTargetRules Target) : base(Target)
{
PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
PrivateDependencyModuleNames.Add("Core");
}
}
```
**YourProjectName/YourProjectName.cpp:**
```cpp
#include "CoreTypes.h"
#include "Modules/ModuleManager.h"
IMPLEMENT_PRIMARY_GAME_MODULE(FDefaultModuleImpl, YourProjectName, "YourProjectName");
```
### 3. Build Both Targets from Command Line
Close the editor, then build from terminal:
```batch
:: Build Editor target (needed to open the project)
"C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat" YourProjectNameEditor Win64 Development "C:\Path\To\YourProject.uproject" -WaitMutex
:: Build Game target (needed for packaging -- THIS IS THE ONE PEOPLE FORGET)
"C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat" YourProjectName Win64 Shipping "C:\Path\To\YourProject.uproject" -WaitMutex
```
> **Critical:** You must build BOTH targets. The Editor build creates `Simulore_TirsovaEditor.target`, and the Game build creates `Simulore_Tirsova-Win64-Shipping.target`. The packaging system only looks for Game-type targets. If you only build the Editor target, the target list will still be empty.
### 4. Clean Stale Data from Copied Project
When a project is copied from another machine, these directories contain stale/incompatible data:
| Directory | Safe to delete | Regenerated by |
|---|---|---|
| `Intermediate/` | Yes | Editor on startup |
| `Binaries/` | Yes | Build step above |
| `DerivedDataCache/` | Yes | Editor on startup |
| `Saved/StagedBuilds/` | Yes | Packaging process |
| `Saved/Cooked/` | Yes | Cooking process |
| `Saved/Crashes/` | Yes | Not regenerated (just old logs) |
| `Saved/Temp/` | Yes | Editor on startup |
**Never delete:** `Config/`, `Content/`, `Source/`, `.uproject`, `Saved/Config/` (fix it instead)
### 5. Fix Hardcoded Paths (THE SILENT KILLER)
Copied projects contain paths from the original machine in multiple locations. Check ALL of these:
#### 5a. Config/DefaultGame.ini
```ini
[/Script/UnrealEd.ProjectPackagingSettings]
BuildTarget=YourProjectName
StagingDirectory=(Path="") ; <-- was hardcoded to other user's Desktop
```
#### 5b. Saved/Config/WindowsEditor/Game.ini (HIGHEST PRIORITY -- overrides DefaultGame.ini!)
This is the one that actually matters for packaging. The editor reads this file, not DefaultGame.ini:
```ini
[/Script/DeveloperToolSettings.PlatformsMenuSettings]
StagingDirectory=(Path="") ; <-- fix this
CookBuildTarget=YourProjectName ; <-- was empty
PackageBuildTarget=YourProjectName ; <-- was empty (ROOT CAUSE of "no targets")
```
> **This was the hardest bug to find.** `DefaultGame.ini` has `BuildTarget=YourProjectName` but the editor actually reads `PackageBuildTarget` from `Saved/Config/WindowsEditor/Game.ini`, which has higher priority. If this is empty, the editor passes an empty string to `FindBestTargetInfo`.
#### 5c. Saved/Config/WindowsEditor/EditorPerProjectUserSettings.ini
Search for and fix any paths referencing other machines:
```ini
SwarmIntermediateFolder= ; <-- was C:/Users/OtherUser/...
LastExecutedLaunchDevice= ; <-- was Windows@OTHER-PC
LastExecutedLaunchName= ; <-- was OTHER-PC
```
#### Quick way to find all stale paths:
```bash
grep -rn "C:/Users/" Config/ Saved/Config/ --include="*.ini" | grep -v "prime"
```
Replace `prime` with the current username.
## UE5 Config Priority System
Understanding this prevents hours of debugging:
| Priority | Source | Hex |
|---|---|---|
| 1 (lowest) | Constructor defaults | 0x00 |
| 2 | Scalability (BaseScalability.ini tiers) | 0x01 |
| 3 | GameUserSettings (in-game quality menu) | 0x02 |
| 4 | ProjectSettings (`[/Script/Engine.RendererSettings]`) | 0x03 |
| 5 | **`[SystemSettings]` and `[ConsoleVariables]` in DefaultEngine.ini** | 0x04 |
| 6 | Device Profiles | 0x05 |
| 7 | Standalone ConsoleVariables.ini (NOT loaded in Shipping!) | 0x06 |
| 8 | Command line | 0x07 |
| 9 | C++ code Set() | 0x08 |
| 10 (highest) | In-game console (~) | 0x09 |
### Where to put rendering overrides:
- **`[/Script/Engine.RendererSettings]`** -- Project-level feature toggles (Lumen on/off, RayTracing on/off, VSM on/off). These affect shader compilation.
- **`[SystemSettings]`** -- Runtime quality overrides that must survive scalability and device profile changes. Use this for forced quality settings in packaged builds.
- **`[ConsoleVariables]`** -- Same priority as SystemSettings. Good for non-rendering cvars (shader pipeline cache, etc).
- **`DefaultGameUserSettings.ini` `[ScalabilityGroups]`** -- Default scalability levels for first-time users. Gets overridden by hardware auto-detect on first run.
### Scalability override gotcha:
If you set `sg.ShadowQuality=3` in `[ScalabilityGroups]` AND `r.Shadow.MaxResolution=512` in `[SystemSettings]`, the scalability system will apply its shadow preset values first, then `[SystemSettings]` overrides specific cvars. Your individual settings win because they have higher priority.
**However:** Device Profiles (priority 0x05) can override `[SystemSettings]` (priority 0x04). If you see cvars being overridden, check `Saved/Config/WindowsEditor/` for device profile configs. The WindowsEditor device profile commonly overrides shadow and scalability settings.
## Packaging from Command Line (Bypass Editor UI)
If the editor UI still refuses to package, use RunUAT directly:
```batch
"C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\RunUAT.bat" BuildCookRun ^
-project="C:\Path\To\YourProject.uproject" ^
-platform=Win64 ^
-clientconfig=Shipping ^
-cook -build -stage -package -pak ^
-prereqs -compressed ^
-target=YourProjectName ^
-unrealexe="C:\Program Files\Epic Games\UE_5.7\Engine\Binaries\Win64\UnrealEditor-Cmd.exe"
```
This bypasses the editor's target discovery entirely and invokes UnrealBuildTool + cooker directly.
## Checklist Summary
When transferring a BP-only UE5 project to a new machine:
- [ ] Add `"Modules"` section to `.uproject`
- [ ] Create `Source/` with Target.cs, Build.cs, and .cpp files
- [ ] Build Editor target from command line
- [ ] Build Game target (Shipping) from command line
- [ ] Delete `Intermediate/`, `Binaries/` (before building), `DerivedDataCache/`, `Saved/StagedBuilds/`, `Saved/Cooked/`
- [ ] Fix `Config/DefaultGame.ini` -- `BuildTarget`, `StagingDirectory`
- [ ] Fix `Saved/Config/WindowsEditor/Game.ini` -- `PackageBuildTarget`, `CookBuildTarget`, `StagingDirectory`
- [ ] Fix `Saved/Config/WindowsEditor/EditorPerProjectUserSettings.ini` -- remove other machine's paths
- [ ] Grep all .ini files for hardcoded paths from other machines
- [ ] Open editor, verify Platforms > Windows > Package Project works