l->t('Assistant fallback'); } public function getTaskTypeId(): string { /** @psalm-suppress UndefinedClass */ return AudioToAudioChat::ID; } public function getExpectedRuntime(): int { return 60; } public function getInputShapeEnumValues(): array { return []; } public function getInputShapeDefaults(): array { return []; } public function getOptionalInputShape(): array { return []; } public function getOptionalInputShapeEnumValues(): array { return []; } public function getOptionalInputShapeDefaults(): array { return []; } public function getOutputShapeEnumValues(): array { return []; } public function getOptionalOutputShape(): array { return []; } public function getOptionalOutputShapeEnumValues(): array { return []; } public function process(?string $userId, array $input, callable $reportProgress): array { if (!isset($input['input']) || !$input['input'] instanceof File || !$input['input']->isReadable()) { throw new RuntimeException('Invalid input file'); } $inputFile = $input['input']; if (!isset($input['system_prompt']) || !is_string($input['system_prompt'])) { throw new RuntimeException('Invalid system_prompt'); } $systemPrompt = $input['system_prompt']; if (!isset($input['history']) || !is_array($input['history'])) { throw new RuntimeException('Invalid history'); } /** @var list $history */ $history = $input['history']; //////////////// 3 steps: STT -> LLM -> TTS // speech to text try { $task = new Task( AudioToText::ID, ['input' => $inputFile->getId()], Application::APP_ID . ':internal', $userId, ); $taskOutput = $this->taskProcessingService->runTaskProcessingTask($task); $inputTranscription = $taskOutput['output']; } catch (Exception $e) { $this->logger->warning('Transcription task failed with: ' . $e->getMessage(), ['exception' => $e]); throw new RuntimeException('Transcription sub task failed with: ' . $e->getMessage()); } // free prompt try { $task = new Task( TextToTextChat::ID, [ 'input' => $inputTranscription, 'system_prompt' => $systemPrompt, 'history' => $history, ], Application::APP_ID . ':internal', $userId, ); $taskOutput = $this->taskProcessingService->runTaskProcessingTask($task); $llmResult = $taskOutput['output']; } catch (Exception $e) { throw new RuntimeException('TextToText sub task failed: ' . $e->getMessage()); } // text to speech try { // this provider is not declared if TextToSpeech does not exist so we know it's fine /** @psalm-suppress UndefinedClass */ $task = new Task( TextToSpeech::ID, ['input' => $llmResult], Application::APP_ID . ':internal', $userId, ); // the setIncludeWatermark method was introduced in NC 33 if (method_exists($task, 'setIncludeWatermark')) { $task->setIncludeWatermark(false); } $taskOutput = $this->taskProcessingService->runTaskProcessingTask($task); $outputAudioFileId = $taskOutput['speech']; return [ 'output' => $this->taskProcessingService->getOutputFileContent($outputAudioFileId), 'output_transcript' => $llmResult, 'input_transcript' => $inputTranscription, ]; } catch (\Exception $e) { $this->logger->warning('Text to speech generation failed with: ' . $e->getMessage(), ['exception' => $e]); throw new RuntimeException('Text to speech sub task failed with: ' . $e->getMessage()); } } }