diff --git a/README.md b/README.md index 89476182..1368742a 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,35 @@ openssl rand -base64 32 使用场景生成、Realtime、TTS、ASR 或评分能力时，还需在 `.env` 中配置对应厂商凭证。 完整变量及安全默认值见 [`deploy/env/.env.example`](deploy/env/.env.example)。 +### 2.1 本地启用 JD 图片 OCR + +Web 的“上传图片”入口以服务端探测结果为准，不需要手动设置浏览器端的开关。后端本地运行时，先准备 Python 3.11、PaddleOCR 依赖和模型： + +```bash +./scripts/prepare-local-ocr.sh +``` + +然后从 `backend/unispeaking-server` 启动后端，并把 OCR 路径指向仓库内的本地目录： + +```bash +cd backend/unispeaking-server +OCR_ENABLED=true \ +OCR_PYTHON_EXECUTABLE="$PWD/../../.local/ocr/venv/bin/python" \ +OCR_RUNNER_PATH="$PWD/src/main/resources/ocr/paddle_ocr_runner.py" \ +OCR_MODEL_DIRECTORY="$PWD/../../.local/ocr/models" \ +MAVEN_REPO_URL=https://maven.aliyun.com/repository/public \ +./mvnw --settings docker/maven/settings.xml spring-boot:run +``` + +启动后用浏览器访问 Web，在模拟面试页面选择“上传图片”。也可以用登录后的 JWT 进行接口实测： + +```bash +OCR_ACCESS_TOKEN='登录后 localStorage 中的 unispeaking.accessToken' \ +./scripts/check-local-ocr.sh /absolute/path/to/jd.png +``` + +脚本先验证 `/api/interview-scenes/ocr/availability`，再提交图片到 `/prepare-materials`；这样可以区分“服务端未装好 OCR”和“图片上传/材料整理链路失败”。 + 注意： - 不要提交真实 `.env`。 diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/component/report/InterviewReportCoordinator.java b/backend/unispeaking-server/src/main/java/com/unispeaking/component/report/InterviewReportCoordinator.java index 9ddc62ec..7087bf4a 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/component/report/InterviewReportCoordinator.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/component/report/InterviewReportCoordinator.java @@ -517,7 +517,14 @@ private String buildLlmPrompt( - VOCABULARY_EXPRESSION (0-100) Produce an overall_score (0-100) as your comprehensive judgment across all five - dimensions, and a short summary narrative of the candidate's spoken English. + dimensions. + + LANGUAGE REQUIREMENT: Write the natural-language values of ALL "evaluation", + "advice", and "summary" fields in Simplified Chinese. Do not return those + fields in English. Keep standard English linguistic terms, quoted candidate + English, and short English examples only when they are necessary as evidence; + these exceptions must be embedded in an otherwise Chinese explanation. The JSON + property names and score values must remain exactly as specified below. Return exactly one JSON object and no Markdown or explanatory prose. The JSON shape must be: diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java index bd53e842..a2696a07 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java @@ -3,6 +3,7 @@ import com.unispeaking.common.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.LlmProvider; +import com.unispeaking.provider.LlmResponseFormat; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; @@ -13,6 +14,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -82,15 +84,26 @@ public QwenLlmProvider( @Override public String executeLlmTask(String prompt, String token) { + return executeLlmTask(prompt, token, LlmResponseFormat.TEXT); + } + + @Override + public String executeLlmTask( + String prompt, + String token, + LlmResponseFormat responseFormat) { if (apiKey.isBlank()) { throw retryableFailure( "QWEN_LLM_CREDENTIAL_MISSING", "Set DASHSCOPE_API_KEY before calling Qwen LLM"); } - return callForContent(prompt, apiKey); + return callForContent(prompt, apiKey, responseFormat); } - private String callForContent(String promptValue, String credential) { + private String callForContent( + String promptValue, + String credential, + LlmResponseFormat responseFormat) { String prompt = trim(promptValue); if (prompt.isBlank()) { throw nonRetryableFailure("INVALID_LLM_PROMPT", "LLM task prompt is required"); @@ -98,10 +111,13 @@ private String callForContent(String promptValue, String credential) { requireHttpsEndpoint(); try { - Map body = Map.of( - "model", model, - "messages", List.of(Map.of("role", "user", "content", prompt)), - "enable_thinking", false); + Map body = new LinkedHashMap<>(); + body.put("model", model); + body.put("messages", List.of(Map.of("role", "user", "content", prompt))); + body.put("enable_thinking", false); + if (responseFormat == LlmResponseFormat.JSON_OBJECT) { + body.put("response_format", Map.of("type", "json_object")); + } HttpRequest httpRequest = HttpRequest.newBuilder() .uri(endpoint) .timeout(readTimeout) diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ocr/PaddleOcrProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ocr/PaddleOcrProvider.java index 4104e551..160005c1 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ocr/PaddleOcrProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ocr/PaddleOcrProvider.java @@ -7,8 +7,12 @@ import com.unispeaking.provider.OcrProvider; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -17,6 +21,7 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -25,10 +30,16 @@ import javax.imageio.ImageIO; import javax.imageio.ImageReader; import javax.imageio.stream.ImageInputStream; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import tools.jackson.databind.ObjectMapper; public final class PaddleOcrProvider implements OcrProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(PaddleOcrProvider.class); + static final int MAX_IMAGE_COUNT = 5; static final int MAX_TOTAL_BYTES = 10 * 1024 * 1024; static final long MAX_PIXELS = 25_000_000L; @@ -39,22 +50,52 @@ public final class PaddleOcrProvider implements OcrProvider { private final OcrProperties properties; private final ObjectMapper objectMapper; + private final Object workerLock = new Object(); + private volatile Process workerProcess; + private volatile BufferedWriter workerWriter; + private volatile BufferedReader workerReader; + private volatile ExecutorService workerStderrExecutor; + private volatile boolean workerReady; public PaddleOcrProvider(OcrProperties properties, ObjectMapper objectMapper) { this.properties = Objects.requireNonNull(properties, "OCR properties are required"); this.objectMapper = Objects.requireNonNull(objectMapper, "ObjectMapper is required"); } + /** 启动后端时预加载 PaddleOCR，并让 Python Worker 常驻内存。 */ + @PostConstruct + void startWorkerOnApplicationStartup() { + if (!baseAvailable()) { + LOGGER.info("paddle OCR worker not started because OCR is not configured"); + return; + } + try { + ensureWorkerStarted(); + } + catch (OcrException exception) { + // OCR availability remains observable through the existing endpoint. The + // first OCR request will retry the worker start, so one transient startup + // failure does not prevent the web application from booting. + LOGGER.warn("paddle OCR worker failed to start during application startup"); + } + } + + @PreDestroy + void stopWorkerOnApplicationShutdown() { + synchronized (workerLock) { + stopWorker(); + } + } + @Override public String recognizeText(List images) { - ensureAvailable(); List validatedImages = validateImages(images); + ensureAvailable(); Path tempDirectory = null; try { tempDirectory = createTempDirectory(); List imagePaths = writeImages(tempDirectory, validatedImages); - ProcessResult result = runOcrProcess(imagePaths); - return parseRecognizedText(result.stdout(), validatedImages.size()); + return recognizeWithWorker(imagePaths, validatedImages.size()); } catch (OcrException exception) { throw exception; @@ -69,6 +110,11 @@ public String recognizeText(List images) { @Override public boolean available() { + return baseAvailable() && workerReady && workerProcess != null + && workerProcess.isAlive(); + } + + private boolean baseAvailable() { return properties.configured() && Files.isRegularFile(properties.runnerPath()) && modelDirectoriesAvailable(properties.modelDirectory()); @@ -81,9 +127,15 @@ private static boolean modelDirectoriesAvailable(Path modelDirectory) { } private void ensureAvailable() { - if (!available()) { + if (!baseAvailable()) { throw new OcrException(OcrErrorCode.UNAVAILABLE); } + try { + ensureWorkerStarted(); + } + catch (OcrException exception) { + throw exception; + } } private static List validateImages(List images) { @@ -197,7 +249,78 @@ private static List writeImages( return imagePaths; } - private ProcessResult runOcrProcess(List imagePaths) throws IOException { + private String recognizeWithWorker(List imagePaths, int expectedCount) { + synchronized (workerLock) { + ensureWorkerStarted(); + String requestId = UUID.randomUUID().toString(); + try { + WorkerRequest request = new WorkerRequest(requestId, + imagePaths.stream().map(Path::toString).toList()); + workerWriter.write(objectMapper.writeValueAsString(request)); + workerWriter.newLine(); + workerWriter.flush(); + String responseLine = readWorkerLine(timeoutMillis(properties.getTimeout())); + if (responseLine == null || responseLine.length() > MAX_STDOUT_BYTES) { + throw new OcrException(OcrErrorCode.RESPONSE_INVALID); + } + WorkerResponse response = objectMapper.readValue(responseLine, WorkerResponse.class); + if (response == null || !requestId.equals(response.id())) { + throw new OcrException(OcrErrorCode.RESPONSE_INVALID); + } + if (response.error() != null && !response.error().isBlank()) { + throw new OcrException(OcrErrorCode.PROCESS_FAILED); + } + return parseRecognizedText(response.results(), expectedCount); + } + catch (OcrException exception) { + if (exception.errorCode() == OcrErrorCode.TIMEOUT + || exception.errorCode() == OcrErrorCode.PROCESS_FAILED + || exception.errorCode() == OcrErrorCode.RESPONSE_INVALID) { + stopWorker(); + } + throw exception; + } + catch (Exception exception) { + stopWorker(); + throw new OcrException(OcrErrorCode.RESPONSE_INVALID); + } + } + } + + private void ensureWorkerStarted() { + synchronized (workerLock) { + if (workerReady && workerProcess != null && workerProcess.isAlive()) { + return; + } + stopWorker(); + if (!baseAvailable()) { + throw new OcrException(OcrErrorCode.UNAVAILABLE); + } + try { + startWorker(); + String readyLine = readWorkerLine(timeoutMillis(properties.getTimeout())); + if (readyLine == null || readyLine.length() > MAX_STDOUT_BYTES) { + throw new OcrException(OcrErrorCode.PROCESS_FAILED); + } + WorkerReady ready = objectMapper.readValue(readyLine, WorkerReady.class); + if (ready == null || !ready.ready()) { + throw new OcrException(OcrErrorCode.PROCESS_FAILED); + } + workerReady = true; + LOGGER.info("paddle OCR worker started and model loaded"); + } + catch (OcrException exception) { + stopWorker(); + throw exception; + } + catch (Exception exception) { + stopWorker(); + throw new OcrException(OcrErrorCode.PROCESS_FAILED); + } + } + } + + private void startWorker() throws IOException { List command = new ArrayList<>(); command.add(properties.getPythonExecutable()); command.add(properties.getRunnerPath()); @@ -210,46 +333,76 @@ private ProcessResult runOcrProcess(List imagePaths) throws IOException { command.add("--disable-doc-orientation"); command.add("--disable-doc-unwarping"); command.add("--disable-textline-orientation"); - command.add("--images"); - imagePaths.stream().map(Path::toString).forEach(command::add); + command.add("--worker"); ProcessBuilder processBuilder = new ProcessBuilder(command); processBuilder.environment().put("PADDLE_PDX_CACHE_HOME", properties.getModelDirectory()); - Process process = processBuilder.start(); - ExecutorService executor = Executors.newFixedThreadPool(2); + workerProcess = processBuilder.start(); + workerWriter = new BufferedWriter(new OutputStreamWriter( + workerProcess.getOutputStream(), StandardCharsets.UTF_8)); + workerReader = new BufferedReader(new InputStreamReader( + workerProcess.getInputStream(), StandardCharsets.UTF_8)); + workerStderrExecutor = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "paddle-ocr-worker-stderr"); + thread.setDaemon(true); + return thread; + }); + workerStderrExecutor.submit(() -> drainWorkerStderr(workerProcess.getErrorStream())); + } + + private String readWorkerLine(long timeoutMillis) throws Exception { + ExecutorService readerExecutor = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "paddle-ocr-worker-reader"); + thread.setDaemon(true); + return thread; + }); try { - Future stdout = - executor.submit(() -> readLimited(process.getInputStream(), MAX_STDOUT_BYTES)); - Future stderr = - executor.submit(() -> readLimited(process.getErrorStream(), MAX_STDERR_BYTES)); - boolean finished = process.waitFor( - timeoutMillis(properties.getTimeout()), - TimeUnit.MILLISECONDS); - if (!finished) { - terminateProcess(process); + Future line = readerExecutor.submit(workerReader::readLine); + try { + return line.get(timeoutMillis, TimeUnit.MILLISECONDS); + } + catch (java.util.concurrent.TimeoutException exception) { + line.cancel(true); throw new OcrException(OcrErrorCode.TIMEOUT); } - LimitedOutput stdoutOutput = getOutput(stdout); - LimitedOutput stderrOutput = getOutput(stderr); - if (stdoutOutput.truncated()) { - throw new OcrException(OcrErrorCode.RESPONSE_INVALID); + } + finally { + readerExecutor.shutdownNow(); + } + } + + private static void drainWorkerStderr(InputStream input) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + input, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + LOGGER.debug("paddle OCR worker: {}", line); } - if (stderrOutput.truncated()) { - throw new OcrException(OcrErrorCode.PROCESS_FAILED); + } + catch (IOException exception) { + // The worker lifecycle owns this stream; shutdown is expected to close it. + } + } + + private void stopWorker() { + workerReady = false; + if (workerWriter != null) { + try { + workerWriter.close(); } - if (process.exitValue() != 0) { - throw new OcrException(OcrErrorCode.PROCESS_FAILED); + catch (IOException ignored) { } - return new ProcessResult(stdoutOutput.text()); } - catch (InterruptedException exception) { - terminateProcess(process); - Thread.currentThread().interrupt(); - throw new OcrException(OcrErrorCode.PROCESS_FAILED, exception); + if (workerProcess != null) { + terminateProcess(workerProcess); } - finally { - executor.shutdownNow(); + if (workerStderrExecutor != null) { + workerStderrExecutor.shutdownNow(); } + workerWriter = null; + workerReader = null; + workerProcess = null; + workerStderrExecutor = null; } private static void terminateProcess(Process process) { @@ -302,21 +455,14 @@ private static LimitedOutput readLimited(InputStream input, int limit) throws IO return new LimitedOutput(output.toString(StandardCharsets.UTF_8), truncated); } - private String parseRecognizedText(String stdout, int expectedCount) { - RunnerResponse response; - try { - response = objectMapper.readValue(stdout, RunnerResponse.class); - } - catch (Exception exception) { + private String parseRecognizedText( + List results, + int expectedCount) { + if (results == null || results.size() != expectedCount) { throw new OcrException(OcrErrorCode.RESPONSE_INVALID); } - if (response == null - || response.results() == null - || response.results().size() != expectedCount) { - throw new OcrException(OcrErrorCode.RESPONSE_INVALID); - } - List texts = new ArrayList<>(response.results().size()); - for (RunnerImageResult result : response.results()) { + List texts = new ArrayList<>(results.size()); + for (RunnerImageResult result : results) { if (result == null || result.text() == null) { throw new OcrException(OcrErrorCode.RESPONSE_INVALID); } @@ -374,12 +520,21 @@ private record ValidatedImage(byte[] content, ImageType type) { private record LimitedOutput(String text, boolean truncated) { } - private record ProcessResult(String stdout) { - } - private record RunnerResponse(List results) { } private record RunnerImageResult(String text) { } + + private record WorkerRequest(String id, List images) { + } + + private record WorkerResponse( + String id, + List results, + String error) { + } + + private record WorkerReady(boolean ready) { + } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java index 4bbc3b31..c403502d 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java @@ -257,6 +257,14 @@ public String executeLlmTask(String modelId, String prompt, String token) { return getLlmProvider(modelId).executeLlmTask(prompt, token); } + public String executeLlmTask( + String modelId, + String prompt, + String token, + LlmResponseFormat responseFormat) { + return getLlmProvider(modelId).executeLlmTask(prompt, token, responseFormat); + } + public String executeLlmTask(String prompt, String token) { return executeLlmTaskRouted(prompt, token).response(); } @@ -267,6 +275,15 @@ public RoutedResult executeLlmTaskRouted(String prompt, String token) { modelId -> executeLlmTask(modelId, prompt, token)); } + public RoutedResult executeLlmTaskRouted( + String prompt, + String token, + LlmResponseFormat responseFormat) { + return invokeRouteWithResult( + AiCapability.LLM, + modelId -> executeLlmTask(modelId, prompt, token, responseFormat)); + } + public String convertAudioToText(String modelId, Byte[] audio, String token) { return getTranscriptionProvider(modelId).convertAudioToText( unboxAudio(audio), diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java index 4755c597..ec4ad3cf 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java @@ -13,4 +13,15 @@ protected LlmProvider(String providerId, Set supportedModels) { public final AiCapability capability() { return AiCapability.LLM; } + + /** + * Default overload keeps existing providers and callers in plain-text mode. + * A provider may override this only when it has a native format parameter. + */ + public String executeLlmTask( + String prompt, + String token, + LlmResponseFormat responseFormat) { + return executeLlmTask(prompt, token); + } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmResponseFormat.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmResponseFormat.java new file mode 100644 index 00000000..ce62fb79 --- /dev/null +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmResponseFormat.java @@ -0,0 +1,10 @@ +package com.unispeaking.provider; + +/** + * Optional response contract requested by an LLM caller. + * Providers that do not support a format must preserve their normal behavior. + */ +public enum LlmResponseFormat { + TEXT, + JSON_OBJECT +} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/service/scene/InterviewSceneService.java b/backend/unispeaking-server/src/main/java/com/unispeaking/service/scene/InterviewSceneService.java index 8f896452..bc833eaa 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/service/scene/InterviewSceneService.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/service/scene/InterviewSceneService.java @@ -30,6 +30,7 @@ import com.unispeaking.infrastructure.persistence.repository.scene.InterviewSceneRepository; import com.unispeaking.infrastructure.persistence.repository.session.PracticeSessionRepository; import com.unispeaking.provider.AiProviderRegistry; +import com.unispeaking.provider.LlmResponseFormat; import com.unispeaking.provider.OcrProvider; import com.unispeaking.service.auth.AuthService; import java.time.OffsetDateTime; @@ -321,7 +322,9 @@ private InterviewMaterial generateMaterial( String resumeText, boolean resumeAbsent) { String prompt = buildMaterialPrompt(jobDescriptionText, resumeText, resumeAbsent); - String content = providerRegistry.executeLlmTaskRouted(prompt, null).response(); + String content = providerRegistry + .executeLlmTaskRouted(prompt, null, LlmResponseFormat.JSON_OBJECT) + .response(); InterviewMaterialResponseNormalizer.ParseResult parsed = materialResponseNormalizer.parse(content); if (parsed.valid()) { @@ -335,7 +338,7 @@ private InterviewMaterial generateMaterial( prompt, parsed.errors()); String repairedContent = providerRegistry - .executeLlmTaskRouted(repairPrompt, null) + .executeLlmTaskRouted(repairPrompt, null, LlmResponseFormat.JSON_OBJECT) .response(); InterviewMaterialResponseNormalizer.ParseResult repaired = materialResponseNormalizer.parse(repairedContent); diff --git a/backend/unispeaking-server/src/main/resources/ocr/paddle_ocr_runner.py b/backend/unispeaking-server/src/main/resources/ocr/paddle_ocr_runner.py index 136de5d2..06cbf795 100644 --- a/backend/unispeaking-server/src/main/resources/ocr/paddle_ocr_runner.py +++ b/backend/unispeaking-server/src/main/resources/ocr/paddle_ocr_runner.py @@ -21,6 +21,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--disable-doc-orientation", action="store_true") parser.add_argument("--disable-doc-unwarping", action="store_true") parser.add_argument("--disable-textline-orientation", action="store_true") + parser.add_argument("--worker", action="store_true") parser.add_argument("--images", nargs="*") return parser @@ -95,6 +96,33 @@ def recognize_batch(ocr: PaddleOCR, image_paths: list[str]) -> dict: return {"results": output} +def run_worker(ocr: PaddleOCR) -> int: + # stdout is a JSON-lines protocol. Paddle/PaddleX diagnostics belong on stderr. + print(json.dumps({"ready": True}), flush=True) + for line in sys.stdin: + request_id = None + try: + request = json.loads(line) + request_id = request.get("id") + image_paths = request.get("images") + if not isinstance(request_id, str) or not request_id: + raise ValueError("invalid request id") + if not isinstance(image_paths, list) or not image_paths: + raise ValueError("missing images") + payload = recognize_batch(ocr, [str(path) for path in image_paths]) + payload["id"] = request_id + print(json.dumps(payload, ensure_ascii=False), flush=True) + except Exception: + # Do not terminate the resident process for one bad image/request. Java + # treats the error response as a failed OCR operation and can restart + # the worker if the process itself has become unhealthy. + print(json.dumps({ + "id": request_id, + "error": "ocr-request-failed", + }), flush=True) + return 0 + + def main() -> int: parser = build_parser() args = parser.parse_args() @@ -105,6 +133,8 @@ def main() -> int: args.text_recognition_model_name, model_directory, ) + if args.worker: + return run_worker(ocr) if args.download_models: ensure_models_loaded(ocr) return 0 diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/component/report/InterviewReportCoordinatorTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/component/report/InterviewReportCoordinatorTest.java index c889870b..d426e86a 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/component/report/InterviewReportCoordinatorTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/component/report/InterviewReportCoordinatorTest.java @@ -213,6 +213,27 @@ void failsWithProviderRetryableWhenLlmProviderThrows() { verify(reportRepository, never()).markCompleted(any()); } + @Test + void asksLlmToWriteReportNarrativesInSimplifiedChinese() { + when(sessionMessageRepository.findMessagesWithAudioObjectKeys(SESSION_ID)) + .thenReturn(List.of()); + when(sessionMessageRepository.findMessages(SESSION_ID)) + .thenReturn(List.of(new Message(1, "I am a backend engineer", null))); + when(sceneRepository.findById(SCENE_ID)) + .thenReturn(Optional.of(sceneDefinition())); + when(providerRegistry.executeLlmTaskRouted(anyString(), isNull())) + .thenReturn(routed(validLlmJson())); + + coordinator.submit(SESSION_ID, SCENE_ID, USER_ID); + + ArgumentCaptor prompt = ArgumentCaptor.forClass(String.class); + verify(providerRegistry).executeLlmTaskRouted(prompt.capture(), isNull()); + assertTrue(prompt.getValue().contains("Simplified Chinese")); + assertTrue(prompt.getValue().contains("ALL \"evaluation\"")); + assertTrue(prompt.getValue().contains("\"advice\", and \"summary\" fields")); + assertTrue(prompt.getValue().contains("property names and score values")); + } + private LearnerMessageRecord turn(int messageNo, String content) { return new LearnerMessageRecord( messageNo, diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java index 8486cd48..cb83c1c6 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java @@ -21,6 +21,7 @@ import com.unispeaking.infrastructure.ai.iflytek.IflytekScoringProvider; import com.unispeaking.infrastructure.ai.minimax.MiniMaxTtsProvider; import com.unispeaking.infrastructure.realtime.RealtimeCredentialIssuer; +import com.unispeaking.provider.LlmResponseFormat; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.Authenticator; @@ -348,6 +349,39 @@ void executesQwenLlmTaskWithTheServerConfiguredCredential() { assertFalse(httpClient.bodyCompletedOnSubscribe); } + @Test + void addsJsonObjectResponseFormatOnlyWhenRequested() { + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse(200, utf8( + "{\"choices\":[{\"message\":{\"content\":\"{\\\"ok\\\":true}\"}}]}"))); + QwenLlmProvider provider = new QwenLlmProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen3.5-plus", + Duration.ofSeconds(20), + 1_048_576); + + provider.executeLlmTask("Return JSON.", null, LlmResponseFormat.JSON_OBJECT); + String body = readBody(httpClient.requests.getFirst()); + assertTrue(body.contains("\"response_format\":{\"type\":\"json_object\"}")); + + RecordingHttpClient textHttpClient = new RecordingHttpClient( + new QueuedResponse(200, utf8( + "{\"choices\":[{\"message\":{\"content\":\"plain\"}}]}"))); + QwenLlmProvider textProvider = new QwenLlmProvider( + textHttpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen3.5-plus", + Duration.ofSeconds(20), + 1_048_576); + textProvider.executeLlmTask("Return text.", null); + assertFalse(readBody(textHttpClient.requests.getFirst()).contains("response_format")); + } + @Test void mapsMalformedQwenResponseToABusinessError() { RecordingHttpClient httpClient = new RecordingHttpClient( diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ocr/PaddleOcrProviderTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ocr/PaddleOcrProviderTest.java index 98ca72d3..6e3c0542 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ocr/PaddleOcrProviderTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ocr/PaddleOcrProviderTest.java @@ -41,24 +41,16 @@ void recognizesPngAndJpegInInputOrderWithOneBatchProcess() throws IOException { Path script = script(""" import json, os, pathlib, sys count_file = pathlib.Path(r'%s') - current = int(count_file.read_text() if count_file.exists() else '0') - count_file.write_text(str(current + 1)) - required = [ - '--text-detection-model-name', 'PP-OCRv5_mobile_det', - '--text-recognition-model-name', 'PP-OCRv5_mobile_rec', - '--device', 'cpu', '--disable-doc-orientation', - '--disable-doc-unwarping', '--disable-textline-orientation', '--images' - ] - for value in required: - if value not in sys.argv: - raise SystemExit(9) - if '--model-dir' in sys.argv: - raise SystemExit(10) if os.environ.get('PADDLE_PDX_CACHE_HOME') != r'%s': raise SystemExit(11) - image_index = sys.argv.index('--images') + 1 - images = sys.argv[image_index:] - print(json.dumps({'results': [{'text': pathlib.Path(path).stem} for path in images]})) + print(json.dumps({'ready': True}), flush=True) + for line in sys.stdin: + request = json.loads(line) + current = int(count_file.read_text() if count_file.exists() else '0') + count_file.write_text(str(current + 1)) + print(json.dumps({'id': request['id'], 'results': [ + {'text': pathlib.Path(path).stem} for path in request['images'] + ]}), flush=True) """.formatted(invocationCounter, cacheHome)); PaddleOcrProvider provider = provider(script, Duration.ofSeconds(2)); @@ -128,8 +120,10 @@ void rejectsImagesAbovePixelLimit() { void mapsTimeoutToStableErrorAndCleansTempDirectory() throws IOException { Path processId = tempRoot.resolve("process-id.txt"); Path script = script(""" - import os, pathlib, time + import json, os, pathlib, sys, time pathlib.Path(r'%s').write_text(str(os.getpid())) + print(json.dumps({'ready': True}), flush=True) + sys.stdin.readline() time.sleep(10) """.formatted(processId)); PaddleOcrProvider provider = provider(script, Duration.ofMillis(500)); @@ -166,7 +160,10 @@ raise SystemExit(7) @Test void mapsInvalidJsonToStableErrorWithoutRecognitionText() throws IOException { Path script = script(""" - print('recognized secret text that must not leak') + import json, sys + print(json.dumps({'ready': True}), flush=True) + request = json.loads(sys.stdin.readline()) + print('recognized secret text that must not leak', flush=True) """); PaddleOcrProvider provider = provider(script, Duration.ofSeconds(2)); @@ -183,7 +180,10 @@ void mapsInvalidJsonToStableErrorWithoutRecognitionText() throws IOException { @Test void mapsOversizedStdoutToInvalidResponse() throws IOException { Path script = script(""" - print('x' * %d) + import json, sys + print(json.dumps({'ready': True}), flush=True) + request = json.loads(sys.stdin.readline()) + print('x' * %d, flush=True) """.formatted(PaddleOcrProvider.MAX_STDOUT_BYTES + 1)); PaddleOcrProvider provider = provider(script, Duration.ofSeconds(2)); diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/service/scene/InterviewSceneServiceTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/service/scene/InterviewSceneServiceTest.java index 017c86a8..50ba7759 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/service/scene/InterviewSceneServiceTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/service/scene/InterviewSceneServiceTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -42,6 +43,7 @@ import com.unispeaking.infrastructure.persistence.repository.session.PracticeSessionRepository; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.AiProviderRegistry.RoutedResult; +import com.unispeaking.provider.LlmResponseFormat; import com.unispeaking.provider.OcrProvider; import com.unispeaking.service.auth.AuthService; import com.unispeaking.service.scene.InterviewSceneService; @@ -198,7 +200,7 @@ void prepareMaterialsStructuresDesensitizedTextIntoMaterial() { .thenReturn("脱敏后 JD"); when(materialDesensitizer.desensitize("简历原始文本")) .thenReturn("脱敏后简历"); - when(providerRegistry.executeLlmTaskRouted(anyString(), isNull())) + when(providerRegistry.executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT))) .thenReturn(completed(validMaterial())); InterviewMaterialDraft draft = service.prepareMaterials( @@ -211,7 +213,7 @@ void prepareMaterialsStructuresDesensitizedTextIntoMaterial() { assertEquals(List.of("负责支付系统设计"), material.responsibilities()); assertEquals(List.of("掌握 Java"), material.qualificationRequirements()); ArgumentCaptor promptCaptor = ArgumentCaptor.forClass(String.class); - verify(providerRegistry).executeLlmTaskRouted(promptCaptor.capture(), isNull()); + verify(providerRegistry).executeLlmTaskRouted(promptCaptor.capture(), isNull(), eq(LlmResponseFormat.JSON_OBJECT)); String prompt = promptCaptor.getValue(); assertTrue(prompt.contains("脱敏后 JD")); assertTrue(prompt.contains("脱敏后简历")); @@ -227,14 +229,14 @@ void prepareMaterialsWithoutResumePassesNoResumeToLlm() { "JD 原始文本", null, true)); when(materialDesensitizer.desensitize("JD 原始文本")) .thenReturn("脱敏后 JD"); - when(providerRegistry.executeLlmTaskRouted(anyString(), isNull())) + when(providerRegistry.executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT))) .thenReturn(completed(validMaterial())); service.prepareMaterials( new InterviewMaterialPreparationInput(null, null, "JD 原始文本", null)); ArgumentCaptor promptCaptor = ArgumentCaptor.forClass(String.class); - verify(providerRegistry).executeLlmTaskRouted(promptCaptor.capture(), isNull()); + verify(providerRegistry).executeLlmTaskRouted(promptCaptor.capture(), isNull(), eq(LlmResponseFormat.JSON_OBJECT)); assertTrue(promptCaptor.getValue().contains("No resume was provided.")); } @@ -246,14 +248,14 @@ void prepareMaterialsRetriesWhenFirstLlmMaterialResponseIsInvalid() { "JD 文本", "简历文本", false)); when(materialDesensitizer.desensitize(anyString())) .thenAnswer(invocation -> invocation.getArgument(0)); - when(providerRegistry.executeLlmTaskRouted(anyString(), isNull())) + when(providerRegistry.executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT))) .thenReturn(completed(invalidMaterial()), completed(validMaterial())); InterviewMaterialDraft draft = service.prepareMaterials( new InterviewMaterialPreparationInput(null, null, "JD 文本", null)); assertEquals("后端开发工程师", draft.material().jobTitle()); - verify(providerRegistry, times(2)).executeLlmTaskRouted(anyString(), isNull()); + verify(providerRegistry, times(2)).executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT)); } @Test @@ -264,7 +266,7 @@ void prepareMaterialsFailsWhenAllLlmMaterialResponsesAreInvalid() { "JD 文本", "简历文本", false)); when(materialDesensitizer.desensitize(anyString())) .thenAnswer(invocation -> invocation.getArgument(0)); - when(providerRegistry.executeLlmTaskRouted(anyString(), isNull())) + when(providerRegistry.executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT))) .thenReturn(completed(invalidMaterial()), completed(invalidMaterial())); BusinessException exception = assertThrows( @@ -273,7 +275,7 @@ void prepareMaterialsFailsWhenAllLlmMaterialResponsesAreInvalid() { new InterviewMaterialPreparationInput(null, null, "JD 文本", null))); assertEquals(InterviewErrorCode.INTERVIEW_MATERIAL_SOURCE_INSUFFICIENT, exception.code()); - verify(providerRegistry, times(2)).executeLlmTaskRouted(anyString(), isNull()); + verify(providerRegistry, times(2)).executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT)); } @Test @@ -286,7 +288,7 @@ void preparesMaterialFromFallbackWhenLlmResponsesAreInvalid() { true)); when(materialDesensitizer.desensitize(anyString())) .thenAnswer(invocation -> invocation.getArgument(0)); - when(providerRegistry.executeLlmTaskRouted(anyString(), isNull())) + when(providerRegistry.executeLlmTaskRouted(anyString(), isNull(), eq(LlmResponseFormat.JSON_OBJECT))) .thenReturn(completed(invalidMaterial()), completed(invalidMaterial())); InterviewMaterial material = service.prepareMaterials( diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index df0750fe..207dc18a 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -33,7 +33,7 @@ services: DATABASE_USERNAME: ${POSTGRES_USER:-unispeaking} DATABASE_PASSWORD: ${POSTGRES_PASSWORD:-unispeaking-local-password} ports: - - "8080:8080" + - "${BACKEND_HOST_PORT:-8080}:8080" depends_on: postgres: condition: service_healthy @@ -44,6 +44,7 @@ services: args: DOCKER_IMAGE_REGISTRY: ${DOCKER_IMAGE_REGISTRY:-docker.m.daocloud.io} VITE_BACKEND_URL: ${VITE_BACKEND_URL:-/backend} + VITE_OCR_ENABLED: ${OCR_ENABLED:-true} VITE_FEEDBACK_URL: ${VITE_FEEDBACK_URL:-} VITE_ALIYUN_CAPTCHA_SCENE_ID: ${ALIYUN_CAPTCHA_SCENE_ID:-} VITE_ALIYUN_CAPTCHA_PREFIX: ${ALIYUN_CAPTCHA_PREFIX:-} diff --git a/deploy/env/.env.example b/deploy/env/.env.example index 093666ce..52e63117 100644 --- a/deploy/env/.env.example +++ b/deploy/env/.env.example @@ -6,6 +6,7 @@ # ============================================================================= SERVER_PORT=8080 VITE_BACKEND_URL=/backend +VITE_OCR_ENABLED=true ALIYUN_CAPTCHA_SCENE_ID=replace-with-captcha-scene-id ALIYUN_CAPTCHA_PREFIX=replace-with-captcha-prefix ALIYUN_CAPTCHA_REGION=cn diff --git a/deploy/env/.env.prod.example b/deploy/env/.env.prod.example index 5604249f..19245206 100644 --- a/deploy/env/.env.prod.example +++ b/deploy/env/.env.prod.example @@ -7,6 +7,7 @@ # ============================================================================= SERVER_PORT=8080 VITE_BACKEND_URL=/backend +VITE_OCR_ENABLED=true ALIYUN_CAPTCHA_SCENE_ID=replace-with-captcha-scene-id ALIYUN_CAPTCHA_PREFIX=replace-with-captcha-prefix ALIYUN_CAPTCHA_REGION=cn diff --git "a/docs/API\346\216\245\345\217\243\346\226\207\346\241\243.md" "b/docs/API\346\216\245\345\217\243\346\226\207\346\241\243.md" index 84811eb4..99f1523f 100644 --- "a/docs/API\346\216\245\345\217\243\346\226\207\346\241\243.md" +++ "b/docs/API\346\216\245\345\217\243\346\226\207\346\241\243.md" @@ -523,4 +523,4 @@ Interview（英文面试，第 4 场景，逐步实现中）： 11. `DELETE /api/interview-scenes/{sceneId}` — 后端删除：软删 `interview_scene` + 物理清该 scene 音频；practice_session/session_message/interview_report 保留（审计 + 学习日历），下游访问经软删过滤 404/403。 12. 失败码补充：`INTERVIEW_REPORT_NOT_FOUND`→404、`INTERVIEW_RECORDING_NOT_FOUND`→404、`INTERVIEW_REPORT_PERSISTENCE_FAILED`→500、`INTERVIEW_AUDIO_INVALID`→400、`INTERVIEW_SESSION_ENDED`→409。 13. `GET /api/interview-scenes/assets` — 面试学习资产列表：`List`（`sceneId/jobTitle/difficulty/latestSessionId/latestReportStatus/latestOverallScore/latestPracticedAt/practiceCount/createdAt`），复练入口。 -14. `GET /api/interview-scenes/ocr/availability` — OCR 可用性探测：`{available: boolean}`（本地未配置 PaddleOCR 时为 false，前端据此禁用 JD 图片上传）。 +14. `GET /api/interview-scenes/ocr/availability` — OCR 可用性探测：`{available: boolean}`（后端检查启用开关、Python runner 和预下载模型目录；Web 面试页启动时调用此接口，据此启用或禁用 JD 图片上传）。 diff --git a/frontend/mobile/src/features/audio/ContinuousTurnRecorder.ts b/frontend/mobile/src/features/audio/ContinuousTurnRecorder.ts index 13cf64d9..4281a648 100644 --- a/frontend/mobile/src/features/audio/ContinuousTurnRecorder.ts +++ b/frontend/mobile/src/features/audio/ContinuousTurnRecorder.ts @@ -143,6 +143,7 @@ export class ContinuousTurnRecorder { private nativeRecordingUri: string | null = null; private readonly sessionSegments: Uint8Array[] = []; private assistantChunks: Uint8Array[] = []; + private lastAssistantAudioDurationMs = 0; constructor( private readonly recorder: StreamRecorderPort, @@ -161,6 +162,7 @@ export class ContinuousTurnRecorder { this.active = null; this.sessionSegments.length = 0; this.assistantChunks = []; + this.lastAssistantAudioDurationMs = 0; this.storage.prepare(this.runId); const started = await this.recorder.startRecording({ sampleRate: SAMPLE_RATE, @@ -243,6 +245,7 @@ export class ContinuousTurnRecorder { const source = concat(this.assistantChunks); const sourceView = new DataView(source.buffer, source.byteOffset, source.byteLength); const sourceSamples = source.length / BYTES_PER_SAMPLE; + this.lastAssistantAudioDurationMs = sourceSamples / PROVIDER_SAMPLE_RATE * 1_000; const outputSamples = Math.floor(sourceSamples * SAMPLE_RATE / PROVIDER_SAMPLE_RATE); const output = new Uint8Array(outputSamples * BYTES_PER_SAMPLE); const outputView = new DataView(output.buffer); @@ -254,6 +257,20 @@ export class ContinuousTurnRecorder { this.assistantChunks = []; } + /** + * React Native WebRTC does not expose a reliable remote-audio `ended` event. + * response.done only means that the provider has finished generating audio; + * give the native receiver a bounded drain window before closing the peer. + */ + async waitForAssistantAudioDrain() { + const drainMs = Math.min( + 4_000, + Math.max(1_500, Math.round(this.lastAssistantAudioDurationMs * 0.35)), + ); + await new Promise((resolve) => setTimeout(resolve, drainMs)); + this.lastAssistantAudioDurationMs = 0; + } + saveSessionRecording(sessionId: string) { this.finishAssistantAudio(); if (!this.sessionSegments.length) return null; diff --git a/frontend/mobile/src/features/interview/InterviewSessionController.ts b/frontend/mobile/src/features/interview/InterviewSessionController.ts index e5bf4a2c..d66caead 100644 --- a/frontend/mobile/src/features/interview/InterviewSessionController.ts +++ b/frontend/mobile/src/features/interview/InterviewSessionController.ts @@ -40,7 +40,7 @@ type InterviewSocket = { }; type InterviewDependencies = { - recorder: Pick; + recorder: Pick; transport: RealtimeTransport; sessionApi: Pick; sessionSocket: InterviewSocket; @@ -90,6 +90,10 @@ export class InterviewSessionController { private configured = false; private openingRequested = false; private fullRecordingUri: string | null = null; + private responseInFlight = false; + private responseAwaitingInterviewState = false; + private responseCancelRequested = false; + private pendingResponseInstructions: string | null = null; constructor( private readonly dependencies: InterviewDependencies, @@ -209,7 +213,7 @@ export class InterviewSessionController { // Interview answers commonly contain thinking pauses. Keep the microphone // open through natural pauses and never let a new user turn cancel an // interviewer response while the candidate is still speaking. - turn_detection: { type: 'semantic_vad', threshold: 0.8, prefix_padding_ms: 1_000, silence_duration_ms: 3_000, create_response: false, interrupt_response: true }, + turn_detection: { type: 'semantic_vad', threshold: 0.8, prefix_padding_ms: 1_000, silence_duration_ms: 3_000, create_response: true, interrupt_response: true }, }, }); this.configured = true; @@ -230,6 +234,14 @@ export class InterviewSessionController { // Keep the candidate microphone live while the interviewer is speaking so // Qwen can detect a deliberate barge-in and stop its response. this.muted = false; + this.responseInFlight = true; + if (this.responseAwaitingInterviewState && !this.closingRequested && !this.responseCancelRequested) { + this.responseCancelRequested = true; + this.dependencies.transport.sendProviderEvent({ + event_id: this.createEventId(), + type: 'response.cancel', + }); + } this.state = 'active'; this.dependencies.recorder.setInputEnabled(true); this.currentQuestion = ''; @@ -240,7 +252,15 @@ export class InterviewSessionController { this.dependencies.recorder.appendAssistantAudio(event.audio); return; case 'assistant.response.completed': + this.responseInFlight = false; + this.responseCancelRequested = false; this.dependencies.recorder.finishAssistantAudio(); + if (this.pendingResponseInstructions !== null && !this.endRequested) { + const instructions = this.pendingResponseInstructions; + this.pendingResponseInstructions = null; + this.sendInterviewResponse(instructions); + return; + } if (!this.closingRequested && !this.endRequested) { this.muted = false; this.dependencies.recorder.setInputEnabled(true); @@ -248,12 +268,14 @@ export class InterviewSessionController { this.applyInput(); this.publish(); } else if (this.closingRequested) { + await this.dependencies.recorder.waitForAssistantAudioDrain(); await this.end(); } return; case 'user.speech.started': if (this.closingRequested || this.endRequested) return; this.dependencies.transport.sendProviderEvent({ event_id: this.createEventId(), type: 'response.cancel' }); + this.responseCancelRequested = true; this.dependencies.recorder.speechStarted(); return; case 'user.speech.stopped': @@ -261,6 +283,7 @@ export class InterviewSessionController { return; case 'user.transcript.completed': if (this.closingRequested || this.endRequested || !event.text.trim()) return; + this.responseAwaitingInterviewState = true; await this.processTranscriptOnce(1, event, () => { this.turnOperation = this.processTurn(event.text.trim(), event.itemId) .finally(() => { this.turnOperation = null; }); @@ -289,20 +312,14 @@ export class InterviewSessionController { // persist or submit a fragment as a complete interview turn: doing so would // advance the backend topic state (and could trigger an early end). Ask for // continuation and keep the same interview turn open instead. - if (looksLikeCutoffTranscript(transcript)) { + if (looksLikeCutoffTranscript(transcript)) { const fragmentAudio = await this.dependencies.recorder.takeTurn(this.turnNo + 1); this.dependencies.recorder.discard(fragmentAudio); this.muted = false; this.dependencies.recorder.setInputEnabled(true); this.applyInput(); - this.dependencies.transport.sendProviderEvent({ - event_id: this.createEventId(), - type: 'response.create', - response: { - instructions: CONTINUE_AFTER_CUTOFF_INSTRUCTION, - modalities: ['text', 'audio'], - }, - }); + this.responseAwaitingInterviewState = false; + this.sendInterviewResponse(CONTINUE_AFTER_CUTOFF_INSTRUCTION); this.publish(); return; } @@ -312,11 +329,8 @@ export class InterviewSessionController { this.muted = false; this.dependencies.recorder.setInputEnabled(true); this.applyInput(); - this.dependencies.transport.sendProviderEvent({ - event_id: this.createEventId(), - type: 'response.create', - response: { instructions: REPEAT_AFTER_MISSING_AUDIO_INSTRUCTION, modalities: ['text', 'audio'] }, - }); + this.responseAwaitingInterviewState = false; + this.sendInterviewResponse(REPEAT_AFTER_MISSING_AUDIO_INSTRUCTION); this.publish(); return; } @@ -329,19 +343,15 @@ export class InterviewSessionController { this.publish(); if (this.endRequested) return; if (result.state.shouldEnd) { + this.responseAwaitingInterviewState = false; this.closingRequested = true; this.muted = true; this.dependencies.recorder.setInputEnabled(false); this.applyInput(); - this.dependencies.transport.sendProviderEvent({ - event_id: this.createEventId(), type: 'response.create', - response: { instructions: result.state.controlInstruction?.trim() || CLOSING_INSTRUCTION, modalities: ['text', 'audio'] }, - }); + this.sendInterviewResponse(result.state.controlInstruction?.trim() || CLOSING_INSTRUCTION); } else { - this.dependencies.transport.sendProviderEvent({ - event_id: this.createEventId(), type: 'response.create', - response: { instructions: result.state.controlInstruction?.trim() || '', modalities: ['text', 'audio'] }, - }); + this.responseAwaitingInterviewState = false; + this.sendInterviewResponse(result.state.controlInstruction?.trim() || ''); } } finally { this.dependencies.recorder.discard(wav); @@ -397,6 +407,26 @@ export class InterviewSessionController { this.dependencies.transport.setAudioEnabled(this.state === 'active' && !this.muted && !this.userMuted && !this.closingRequested && !this.endRequested); } + private sendInterviewResponse(instructions: string) { + if (this.responseInFlight) { + this.pendingResponseInstructions = instructions; + if (!this.responseCancelRequested) { + this.responseCancelRequested = true; + this.dependencies.transport.sendProviderEvent({ + event_id: this.createEventId(), + type: 'response.cancel', + }); + } + return; + } + this.dependencies.transport.sendProviderEvent({ + event_id: this.createEventId(), + type: 'response.create', + response: { instructions, modalities: ['text', 'audio'] }, + }); + this.responseInFlight = true; + } + private assertStartActive() { if (this.endRequested) throw new Error('面试启动已取消'); } @@ -461,6 +491,7 @@ export class InterviewSessionController { private reset() { this.state = 'idle'; this.error = null; this.backend = null; this.turnNo = 0; this.interviewState = null; this.reportStatus = null; this.transcripts.length = 0; this.seenTranscriptIds.clear(); this.pendingTranscriptIds.clear(); this.endRequested = false; this.closingRequested = false; this.configured = false; this.openingRequested = false; this.endPromise = null; + this.responseInFlight = false; this.responseAwaitingInterviewState = false; this.responseCancelRequested = false; this.pendingResponseInstructions = null; } private publish() { const snapshot = this.getSnapshot(); this.listeners.forEach((listener) => listener(snapshot)); } diff --git a/frontend/mobile/src/features/interview/__tests__/InterviewSessionController.test.ts b/frontend/mobile/src/features/interview/__tests__/InterviewSessionController.test.ts index f6fb9bff..af9774b8 100644 --- a/frontend/mobile/src/features/interview/__tests__/InterviewSessionController.test.ts +++ b/frontend/mobile/src/features/interview/__tests__/InterviewSessionController.test.ts @@ -18,7 +18,7 @@ function fixture() { start: jest.fn(async () => { calls.push('recorder.start'); }), setInputEnabled: jest.fn(), speechStarted: jest.fn(), speechStopped: jest.fn(), takeTurn: jest.fn(async (turnNo: number) => ({ uri: `turn-${turnNo}.wav`, name: `turn-${turnNo}.wav`, size: 16_044, durationMs: 500 })), - appendAssistantAudio: jest.fn(), finishAssistantAudio: jest.fn(), saveSessionRecording: jest.fn(() => 'file:///full.wav'), + appendAssistantAudio: jest.fn(), finishAssistantAudio: jest.fn(), waitForAssistantAudioDrain: jest.fn(async () => undefined), saveSessionRecording: jest.fn(() => 'file:///full.wav'), discard: jest.fn(), close: jest.fn(async () => { calls.push('recorder.close'); }), }; const sessionApi = { @@ -58,6 +58,18 @@ describe('InterviewSessionController', () => { expect(test.controller.getSnapshot().state).toBe('active'); }); + it('uses automatic provider VAD responses while retaining interview orchestration', async () => { + const test = fixture(); + await test.controller.start(); + await provider(test, { type: 'session.created' }); + const update = test.transport.sendProviderEvent.mock.calls.find(([event]) => event.type === 'session.update')?.[0]; + expect(update.session.turn_detection).toEqual(expect.objectContaining({ + silence_duration_ms: 3_000, + create_response: true, + interrupt_response: true, + })); + }); + it('uses a pause-tolerant VAD configuration for interview answers', async () => { const test = fixture(); await test.controller.start(); @@ -68,7 +80,7 @@ describe('InterviewSessionController', () => { threshold: 0.8, prefix_padding_ms: 1_000, interrupt_response: true, - create_response: false, + create_response: true, })); }); @@ -104,7 +116,7 @@ describe('InterviewSessionController', () => { })); }); - it('mutes once shouldEnd is returned, sends one closing response, and ends after response.done', async () => { + it('mutes once shouldEnd is returned, drains the closing audio, and then ends', async () => { const test = fixture(); await test.controller.start(); await provider(test, { type: 'session.created' }); @@ -115,6 +127,7 @@ describe('InterviewSessionController', () => { await provider(test, { type: 'conversation.item.input_audio_transcription.completed', item_id: 'item-1', transcript: 'This is my complete answer.' }); await provider(test, { type: 'response.done', response: { status: 'completed' } }); expect(test.transport.setAudioEnabled).toHaveBeenLastCalledWith(false); + expect(test.recorder.waitForAssistantAudioDrain).toHaveBeenCalledTimes(1); expect(test.sessionApi.end).toHaveBeenCalledTimes(1); expect(test.controller.getSnapshot().state).toBe('ended'); }); diff --git a/frontend/web/Dockerfile b/frontend/web/Dockerfile index 637eaf42..14cc7d80 100644 --- a/frontend/web/Dockerfile +++ b/frontend/web/Dockerfile @@ -5,6 +5,7 @@ FROM ${DOCKER_IMAGE_REGISTRY}/library/node:22-alpine AS build WORKDIR /app ARG VITE_BACKEND_URL=/backend +ARG VITE_OCR_ENABLED=true ARG VITE_FEEDBACK_URL= ARG VITE_ALIYUN_CAPTCHA_SCENE_ID= ARG VITE_ALIYUN_CAPTCHA_PREFIX= @@ -15,6 +16,7 @@ ARG VITE_UMAMI_WEBSITE_ID= ARG VITE_UMAMI_DOMAINS= ARG NPM_REGISTRY=https://registry.npmmirror.com ENV VITE_BACKEND_URL=${VITE_BACKEND_URL} +ENV VITE_OCR_ENABLED=${VITE_OCR_ENABLED} ENV VITE_FEEDBACK_URL=${VITE_FEEDBACK_URL} ENV VITE_ALIYUN_CAPTCHA_SCENE_ID=${VITE_ALIYUN_CAPTCHA_SCENE_ID} ENV VITE_ALIYUN_CAPTCHA_PREFIX=${VITE_ALIYUN_CAPTCHA_PREFIX} diff --git a/frontend/web/scripts/check-realtime-events.mjs b/frontend/web/scripts/check-realtime-events.mjs index d199b206..beaaee88 100644 --- a/frontend/web/scripts/check-realtime-events.mjs +++ b/frontend/web/scripts/check-realtime-events.mjs @@ -240,6 +240,22 @@ assert.equal(deterministicIeltsPart.turn_detection.create_response, false); assert.equal(partTwoSession.turn_detection.create_response, false); assert.equal(partTwoSession.turn_detection.interrupt_response, false); +const interviewSession = buildRealtimeSessionConfig({ + systemPrompt: "Conduct a structured interview.", + model: "qwen3.5-omni-flash-realtime", + automaticTurnResponses: false, + silenceDurationMs: 3_000, + interruptResponse: true, + vadThreshold: 0.8, + prefixPaddingMs: 1_000, +}); +assert.equal(interviewSession.turn_detection.type, "semantic_vad"); +assert.equal(interviewSession.turn_detection.silence_duration_ms, 3_000); +assert.equal(interviewSession.turn_detection.create_response, false); +assert.equal(interviewSession.turn_detection.interrupt_response, true); +assert.equal(interviewSession.turn_detection.threshold, 0.8); +assert.equal(interviewSession.turn_detection.prefix_padding_ms, 1_000); + let segmentStartCount = 0; let segmentStopCount = 0; const expectedAudio = { type: "audio/wav" }; diff --git a/frontend/web/src/component/interview/InterviewModule.jsx b/frontend/web/src/component/interview/InterviewModule.jsx index 70b5e4d9..3d24be9b 100644 --- a/frontend/web/src/component/interview/InterviewModule.jsx +++ b/frontend/web/src/component/interview/InterviewModule.jsx @@ -22,6 +22,7 @@ import { Modal } from "../common/Modal.jsx"; import { generateInterviewScene, getInterviewAssets, + getInterviewOcrAvailability, getInterviewReport, prepareInterviewMaterials, retryInterviewReport, @@ -33,7 +34,6 @@ import { SimpleCta, TrendLineChart } from "../ielts/IeltsModule.jsx"; const cx = (...parts) => parts.filter(Boolean).join(" "); -const JD_IMAGE_OCR_ENABLED = import.meta.env.VITE_OCR_ENABLED === "true"; const DIFFICULTY_LABELS = { EASY: "简单", STANDARD: "标准", HARD: "困难" }; const speedCodeByLabel = { @@ -267,6 +267,8 @@ function MaterialEditor({ material, onChange, compact = false }) { } function InterviewHome({ onNavigate, onBack }) { + const [ocrAvailable, setOcrAvailable] = useState(false); + const [ocrAvailabilityLoading, setOcrAvailabilityLoading] = useState(true); const [jdMode, setJdMode] = useState("text"); const [jdText, setJdText] = useState(""); const [jdImage, setJdImage] = useState(null); @@ -280,12 +282,34 @@ function InterviewHome({ onNavigate, onBack }) { const [formError, setFormError] = useState(""); const [generating, setGenerating] = useState(false); const [generateError, setGenerateError] = useState(""); - const jdImageUnavailable = !JD_IMAGE_OCR_ENABLED; + const jdImageUnavailable = ocrAvailabilityLoading || !ocrAvailable; + + useEffect(() => { + let cancelled = false; + getInterviewOcrAvailability() + .then((result) => { + if (cancelled) return; + setOcrAvailable(result?.available === true); + }) + .catch(() => { + if (!cancelled) setOcrAvailable(false); + }) + .finally(() => { + if (!cancelled) setOcrAvailabilityLoading(false); + }); + return () => { + cancelled = true; + }; + }, []); const prepareMaterials = async () => { if (preparing) return; setFormError(""); - if (jdMode === "image" && jdImageUnavailable) { + if (jdMode === "image" && ocrAvailabilityLoading) { + setFormError("正在检测 OCR，请稍后再试"); + return; + } + if (jdMode === "image" && !ocrAvailable) { setFormError("OCR 暂不可用，请使用粘贴文本方式上传 JD"); return; } @@ -402,12 +426,12 @@ function InterviewHome({ onNavigate, onBack }) { 岗位描述（JD）
- +
{jdMode === "text" ?