4v5u09Z5a4Yuc/English
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve dialogue speech recognition and AI intervention

Browse Source
This commit is contained in:
shen
2026-09-19 17:45:25 -07:00
parent 5b9cba793e
commit 335d0304b1
6 changed files with 704 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kouyu_english/lib/core/ai_service.dart
+85
View File
@@ -989,6 +989,91 @@ Learner wrote: $answer''';
return decodeWritingAiFeedback(content, expectedLessonId: answerId);
}
/// Evaluates a learner's dialogue turn when it does not match local preset rules.
/// Decides whether the reply is semantically acceptable in context, or detects
/// speech-to-text (ASR) phonetic slips, typos, or minor grammar errors
/// (e.g. "I'm third today" intended for "I'm tired today").
Future<DialogueAiIntervention?> checkDialogueIntervention({
required AiProviderType provider,
required String endpoint,
required String model,
required String partnerLine,
required String taskLabel,
required String learnerText,
String? hint,
String level = 'A0',
}) async {
if (provider == AiProviderType.mock) return null;
final instruction =
'You are a supportive oral English coach evaluating an ESL beginner ($level) spoken line in a dialogue.\n'
'Context:\n'
'- Dialogue partner said: "$partnerLine"\n'
'- Current turn task/goal: "$taskLabel"\n'
'${hint != null && hint.isNotEmpty ? '- Example response: "$hint"\n' : ''}'
'- Learner spoke/wrote: "$learnerText"\n\n'
'Evaluation Instructions:\n'
'1. Semantic & Communicative Check: Does the learner\'s response make sense and fulfill the conversational goal, even if phrased differently from the example (e.g. "I feel great", "Pretty good", "Not bad at all", "I like tea")?\n'
'2. Speech-to-Text (ASR) & Typo Slip Detection: Detect common speech recognition confusions or acoustic slips (e.g. /taɪəd/ transcribed as "third", "tierd", "thx"). If the learner clearly attempted the task with a phonetic or spelling slip, identify their intended English sentence.\n'
'3. Return ONLY valid JSON with no markdown:\n'
'{\n'
' "accepted": true or false,\n'
' "suggestion": "corrected English sentence (or null if accepted as-is)",\n'
' "explanation": "concise, warm Chinese explanation (1-2 sentences)"\n'
'}\n'
'Rules:\n'
'- If it is a valid, natural reply (or minor casing/punctuation): set "accepted": true, "suggestion": null, "explanation": "表达自然得体，符合本轮交流目标。".\n'
'- If there is an ASR slip, typo, or word error (e.g. "I\'m third today" intended for "I\'m tired today"): set "accepted": false, "suggestion": "I\'m tired today.", "explanation": "识别为 third，你可能是想表达 tired（今天很累）吗？".\n'
'- If off-topic or empty: set "accepted": false, "suggestion": null, "explanation": "简要说明本轮对方在问什么，建议如何回答".';
final content = await _requestPrompt(
provider: provider,
endpoint: endpoint,
model: model,
prompt: instruction,
temperature: 0,
maxTokens: 250,
);
return _decodeDialogueIntervention(content);
}
DialogueAiIntervention? _decodeDialogueIntervention(String? content) {
if (content == null || content.trim().isEmpty) return null;
try {
var sanitized = content.trim();
if (sanitized.startsWith('```')) {
sanitized = sanitized.replaceFirst(RegExp(r'^```[a-zA-Z]*\s*'), '');
sanitized = sanitized.replaceFirst(RegExp(r'\s*```$'), '');
}
final jsonStart = sanitized.indexOf('{');
final jsonEnd = sanitized.lastIndexOf('}');
if (jsonStart >= 0 && jsonEnd > jsonStart) {
sanitized = sanitized.substring(jsonStart, jsonEnd + 1);
}
final data = jsonDecode(sanitized);
if (data is! Map<String, dynamic>) return null;
final accepted = data['accepted'] == true;
final rawSuggestion = data['suggestion'] as String?;
final suggestion =
(rawSuggestion != null &&
rawSuggestion.trim().isNotEmpty &&
rawSuggestion.trim().toLowerCase() != 'null')
? rawSuggestion.trim()
: null;
final explanation =
(data['explanation'] as String?)?.trim() ??
(accepted ? '回答符合要求。' : '建议调整表达后再试。');
return DialogueAiIntervention(
accepted: accepted,
suggestion: suggestion,
explanation: explanation,
);
} catch (_) {
return null;
}
}
/// Requests a 4-skill adaptive mini-lesson that re-teaches a failed target.
Future<GeneratedLesson?> generateAdaptiveLesson({
required AiProviderType provider,
kouyu_english/lib/core/models.dart
+18
View File
@@ -111,6 +111,24 @@ class WritingAiFeedback {
final List<String> missing;
}
/// Result of AI intervention during a dialogue turn when preset rules do not match.
class DialogueAiIntervention {
const DialogueAiIntervention({
required this.accepted,
this.suggestion,
required this.explanation,
});
/// Whether the learner's response is semantically acceptable for the turn.
final bool accepted;
/// Inferred or corrected English sentence if there was an ASR slip, typo, or minor mistake.
final String? suggestion;
/// Encouraging, concise Chinese explanation of the situation and recommendation.
final String explanation;
}
class AssessmentRecord {
const AssessmentRecord({
required this.packId,
kouyu_english/lib/core/sherpa_stt_service.dart
+1 -1
View File
@@ -164,7 +164,7 @@ class SherpaSttService {
senseVoice: sherpa_onnx.OfflineSenseVoiceModelConfig(
model: resolved['model']!,
language: 'en',
useInverseTextNormalization: true,
useInverseTextNormalization: false,
),
tokens: resolved['tokens']!,
numThreads: 2,
kouyu_english/lib/core/voice_service.dart
+37 -1
View File
@@ -14,6 +14,42 @@ class VoiceService {
VoiceService._();
static final instance = VoiceService._();
/// Reverses the inverse text normalization (ITN) performed by system speech
/// recognisers, which convert spoken numbers like "one" into "1".
/// For an English-learning app the learner needs the spelled-out words.
static String reverseItn(String text) {
// Standalone digit → word. Uses word-boundary anchors so "12" or "100" are
// left alone (those are unlikely to be single-word utterances the learner
// intended as words).
const digitToWord = {
'0': 'zero',
'1': 'one',
'2': 'two',
'3': 'three',
'4': 'four',
'5': 'five',
'6': 'six',
'7': 'seven',
'8': 'eight',
'9': 'nine',
'10': 'ten',
'11': 'eleven',
'12': 'twelve',
'13': 'thirteen',
'14': 'fourteen',
'15': 'fifteen',
'16': 'sixteen',
'17': 'seventeen',
'18': 'eighteen',
'19': 'nineteen',
'20': 'twenty',
};
return text.replaceAllMapped(
RegExp(r'\b(\d{1,2})\b'),
(m) => digitToWord[m.group(1)!] ?? m.group(0)!,
);
}
final FlutterTts _tts = FlutterTts();
final SpeechToText _stt = SpeechToText();
final AudioRecorder _recorder = AudioRecorder();
@@ -228,7 +264,7 @@ class VoiceService {
await _stt.listen(
onResult: (result) =>
onResult(result.recognizedWords, result.finalResult),
onResult(reverseItn(result.recognizedWords), result.finalResult),
listenOptions: SpeechListenOptions(
localeId: targetLocaleId,
listenFor: const Duration(seconds: 30),
kouyu_english/lib/features/dialogue/dialogue_flow.dart
+230 -10
View File
@@ -161,6 +161,8 @@ class _DialoguePageState extends State<DialoguePage>
bool checkingWithAi = false;
WritingAiFeedback? aiCheck;
String? aiCheckError;
bool interveningWithAi = false;
DialogueAiIntervention? aiIntervention;
/// Shown when the reply on screen came from the built-in script instead of
/// the AI, so a canned line is never mistaken for a real answer.
@@ -308,12 +310,66 @@ class _DialoguePageState extends State<DialoguePage>
});
}
Future<void> send() async {
final text = controller.text.trim();
if (text.isEmpty || stage >= script.prompts.length || waitingForReply) {
Future<void> send({
bool overrideValidation = false,
String? overrideText,
}) async {
final text = (overrideText ?? controller.text).trim();
if (text.isEmpty ||
stage >= script.prompts.length ||
waitingForReply ||
interveningWithAi) {
return;
}
if (!_matchesCurrentTask(text)) {
if (!overrideValidation && !_matchesCurrentTask(text)) {
if (widget.state.aiProvider != AiProviderType.mock) {
setState(() {
interveningWithAi = true;
validationError = null;
aiIntervention = null;
});
_scrollToBottom();
final partnerLine =
turns.where((turn) => !turn.isLearner).lastOrNull?.text ?? '';
final hintText =
stage < script.hints.length ? script.hints[stage] : null;
final intervention =
await AiService.instance.checkDialogueIntervention(
provider: widget.state.aiProvider,
endpoint: widget.state.aiEndpoint,
model: widget.state.aiModel,
partnerLine: partnerLine,
taskLabel: _currentTaskLabel(),
learnerText: text,
hint: hintText,
level: _languageLessonId == null
? 'A0'
: lessonLevel(_languageLessonId!),
);
if (!mounted) return;
setState(() => interveningWithAi = false);
if (intervention != null) {
if (intervention.accepted) {
// AI confirmed semantic acceptability: proceed directly!
await _executeSend(text, usedAiIntervention: true);
return;
} else {
// AI detected ASR slip / typo / off-topic: show intervention card
setState(() {
aiIntervention = intervention;
validationError = null;
});
_scrollToBottom();
return;
}
}
}
setState(
() => validationError =
'这一轮要“${_currentTaskLabel()}”，这句还没做到。'
@@ -321,6 +377,17 @@ class _DialoguePageState extends State<DialoguePage>
);
return;
}
await _executeSend(text, usedAiIntervention: overrideValidation);
}
Future<void> _executeSend(
String text, {
bool usedAiIntervention = false,
}) async {
if (usedAiIntervention) {
usedHelp = true;
}
widget.state.recordDialogueAttempt(
taskId: widget.isLessonDialogue
? 'dialogue-$_lessonSegmentId-$stage'
@@ -343,6 +410,7 @@ class _DialoguePageState extends State<DialoguePage>
waitingForReply = true;
hint = null;
validationError = null;
aiIntervention = null;
aiCheck = null;
aiCheckError = null;
});
@@ -750,6 +818,125 @@ class _DialoguePageState extends State<DialoguePage>
validationError!,
style: TextStyle(color: AppColors.warmInk),
),
if (interveningWithAi)
Padding(
padding: const EdgeInsets.symmetric(vertical: 4),
child: Row(
children: [
const SizedBox(
width: 14,
height: 14,
child: CircularProgressIndicator(strokeWidth: 2),
),
const SizedBox(width: 8),
Text(
'AI 正在理解你的回答…',
style: TextStyle(fontSize: 13, color: AppColors.muted),
),
],
),
),
if (aiIntervention != null)
SectionCard(
tint: AppColors.warm,
child: SpacedColumn(
crossAxisAlignment: CrossAxisAlignment.start,
spacing: 8,
children: [
Row(
children: [
Icon(
Icons.auto_awesome,
size: 18,
color: AppColors.warmInk,
),
const SizedBox(width: 6),
Text(
'AI 助手干预与建议',
style: TextStyle(
fontWeight: FontWeight.bold,
color: AppColors.warmInk,
),
),
],
),
Text(
aiIntervention!.explanation,
style: TextStyle(
color: AppColors.warmInk,
fontSize: 13,
),
),
if (aiIntervention!.suggestion != null &&
aiIntervention!.suggestion!.isNotEmpty) ...[
Container(
padding: const EdgeInsets.symmetric(
horizontal: 10,
vertical: 6,
),
decoration: BoxDecoration(
color: AppColors.surface,
borderRadius: BorderRadius.circular(6),
border: Border.all(color: AppColors.line),
),
child: Row(
children: [
const Text(
'建议表达：',
style: TextStyle(fontSize: 12),
),
Expanded(
child: Text(
aiIntervention!.suggestion!,
style: TextStyle(
fontWeight: FontWeight.w600,
color: AppColors.green,
),
),
),
],
),
),
Wrap(
spacing: 8,
runSpacing: 8,
children: [
FilledButton.icon(
onPressed: () {
final fix = aiIntervention!.suggestion!;
controller.text = fix;
send(overrideValidation: true, overrideText: fix);
},
icon: const Icon(Icons.check, size: 16),
label: Text(
'修正为 "${aiIntervention!.suggestion}" 并发送',
),
style: FilledButton.styleFrom(
backgroundColor: AppColors.green,
visualDensity: VisualDensity.compact,
),
),
OutlinedButton(
onPressed: () => send(overrideValidation: true),
style: OutlinedButton.styleFrom(
visualDensity: VisualDensity.compact,
),
child: const Text('仍按原样发送'),
),
],
),
] else ...[
OutlinedButton(
onPressed: () => send(overrideValidation: true),
style: OutlinedButton.styleFrom(
visualDensity: VisualDensity.compact,
),
child: const Text('仍按原样发送'),
),
],
],
),
),
TextField(
controller: controller,
onChanged: (value) {
@@ -757,13 +944,19 @@ class _DialoguePageState extends State<DialoguePage>
usedVoice &&
value != lastTranscript &&
!transcriptEdited;
final needClearAi = aiCheck != null || aiCheckError != null;
final needClearAi =
aiCheck != null ||
aiCheckError != null ||
aiIntervention != null ||
validationError != null;
if (needResetVoice || needClearAi) {
setState(() {
if (needResetVoice) transcriptEdited = true;
if (needClearAi) {
aiCheck = null;
aiCheckError = null;
aiIntervention = null;
validationError = null;
}
});
}
@@ -790,8 +983,16 @@ class _DialoguePageState extends State<DialoguePage>
onPressed: transcribing ? null : _toggleListening,
),
suffixIcon: IconButton(
icon: const Icon(Icons.send),
onPressed: waitingForReply ? null : send,
icon: interveningWithAi
? const SizedBox(
width: 18,
height: 18,
child: CircularProgressIndicator(strokeWidth: 2),
)
: const Icon(Icons.send),
onPressed: (waitingForReply || interveningWithAi)
? null
: send,
),
border: const OutlineInputBorder(),
),
@@ -802,7 +1003,8 @@ class _DialoguePageState extends State<DialoguePage>
final canCheck =
value.text.trim().isNotEmpty &&
!checkingWithAi &&
!waitingForReply;
!waitingForReply &&
!interveningWithAi;
return OutlinedButton.icon(
onPressed: canCheck ? _checkWithAi : null,
icon: checkingWithAi
@@ -828,11 +1030,29 @@ class _DialoguePageState extends State<DialoguePage>
children: [
Text('AI 检查：${aiCheck!.feedback}'),
for (final note in aiCheck!.missing) Text('· $note'),
if (aiCheck!.suggestion != null)
if (aiCheck!.suggestion != null) ...[
Text('参考改正：${aiCheck!.suggestion}'),
Align(
alignment: Alignment.centerLeft,
child: TextButton.icon(
onPressed: () {
controller.text = aiCheck!.suggestion!;
setState(() {
usedHelp = true;
aiCheck = null;
});
},
icon: const Icon(Icons.done, size: 16),
label: const Text('采纳该表达'),
style: TextButton.styleFrom(
visualDensity: VisualDensity.compact,
),
),
),
],
Text(
aiCheck!.verdict == 'accepted'
? 'AI 仅作参考；这一轮是否完成仍由本地检查判断。'
? 'AI 辅助判定；若词汇超出预设，发送时 AI 会自动进行语义理解与干预。'
: '看过改正后再发送，本次对话会记为使用过提示。',
style: TextStyle(fontSize: 12, color: AppColors.muted),
),