feat: improve dialogue speech recognition and AI intervention
This commit is contained in:
@@ -989,6 +989,91 @@ Learner wrote: $answer''';
|
||||
return decodeWritingAiFeedback(content, expectedLessonId: answerId);
|
||||
}
|
||||
|
||||
/// Evaluates a learner's dialogue turn when it does not match local preset rules.
|
||||
/// Decides whether the reply is semantically acceptable in context, or detects
|
||||
/// speech-to-text (ASR) phonetic slips, typos, or minor grammar errors
|
||||
/// (e.g. "I'm third today" intended for "I'm tired today").
|
||||
Future<DialogueAiIntervention?> checkDialogueIntervention({
|
||||
required AiProviderType provider,
|
||||
required String endpoint,
|
||||
required String model,
|
||||
required String partnerLine,
|
||||
required String taskLabel,
|
||||
required String learnerText,
|
||||
String? hint,
|
||||
String level = 'A0',
|
||||
}) async {
|
||||
if (provider == AiProviderType.mock) return null;
|
||||
final instruction =
|
||||
'You are a supportive oral English coach evaluating an ESL beginner ($level) spoken line in a dialogue.\n'
|
||||
'Context:\n'
|
||||
'- Dialogue partner said: "$partnerLine"\n'
|
||||
'- Current turn task/goal: "$taskLabel"\n'
|
||||
'${hint != null && hint.isNotEmpty ? '- Example response: "$hint"\n' : ''}'
|
||||
'- Learner spoke/wrote: "$learnerText"\n\n'
|
||||
'Evaluation Instructions:\n'
|
||||
'1. Semantic & Communicative Check: Does the learner\'s response make sense and fulfill the conversational goal, even if phrased differently from the example (e.g. "I feel great", "Pretty good", "Not bad at all", "I like tea")?\n'
|
||||
'2. Speech-to-Text (ASR) & Typo Slip Detection: Detect common speech recognition confusions or acoustic slips (e.g. /taɪəd/ transcribed as "third", "tierd", "thx"). If the learner clearly attempted the task with a phonetic or spelling slip, identify their intended English sentence.\n'
|
||||
'3. Return ONLY valid JSON with no markdown:\n'
|
||||
'{\n'
|
||||
' "accepted": true or false,\n'
|
||||
' "suggestion": "corrected English sentence (or null if accepted as-is)",\n'
|
||||
' "explanation": "concise, warm Chinese explanation (1-2 sentences)"\n'
|
||||
'}\n'
|
||||
'Rules:\n'
|
||||
'- If it is a valid, natural reply (or minor casing/punctuation): set "accepted": true, "suggestion": null, "explanation": "表达自然得体,符合本轮交流目标。".\n'
|
||||
'- If there is an ASR slip, typo, or word error (e.g. "I\'m third today" intended for "I\'m tired today"): set "accepted": false, "suggestion": "I\'m tired today.", "explanation": "识别为 third,你可能是想表达 tired(今天很累)吗?".\n'
|
||||
'- If off-topic or empty: set "accepted": false, "suggestion": null, "explanation": "简要说明本轮对方在问什么,建议如何回答".';
|
||||
|
||||
final content = await _requestPrompt(
|
||||
provider: provider,
|
||||
endpoint: endpoint,
|
||||
model: model,
|
||||
prompt: instruction,
|
||||
temperature: 0,
|
||||
maxTokens: 250,
|
||||
);
|
||||
return _decodeDialogueIntervention(content);
|
||||
}
|
||||
|
||||
DialogueAiIntervention? _decodeDialogueIntervention(String? content) {
|
||||
if (content == null || content.trim().isEmpty) return null;
|
||||
try {
|
||||
var sanitized = content.trim();
|
||||
if (sanitized.startsWith('```')) {
|
||||
sanitized = sanitized.replaceFirst(RegExp(r'^```[a-zA-Z]*\s*'), '');
|
||||
sanitized = sanitized.replaceFirst(RegExp(r'\s*```$'), '');
|
||||
}
|
||||
final jsonStart = sanitized.indexOf('{');
|
||||
final jsonEnd = sanitized.lastIndexOf('}');
|
||||
if (jsonStart >= 0 && jsonEnd > jsonStart) {
|
||||
sanitized = sanitized.substring(jsonStart, jsonEnd + 1);
|
||||
}
|
||||
final data = jsonDecode(sanitized);
|
||||
if (data is! Map<String, dynamic>) return null;
|
||||
|
||||
final accepted = data['accepted'] == true;
|
||||
final rawSuggestion = data['suggestion'] as String?;
|
||||
final suggestion =
|
||||
(rawSuggestion != null &&
|
||||
rawSuggestion.trim().isNotEmpty &&
|
||||
rawSuggestion.trim().toLowerCase() != 'null')
|
||||
? rawSuggestion.trim()
|
||||
: null;
|
||||
final explanation =
|
||||
(data['explanation'] as String?)?.trim() ??
|
||||
(accepted ? '回答符合要求。' : '建议调整表达后再试。');
|
||||
|
||||
return DialogueAiIntervention(
|
||||
accepted: accepted,
|
||||
suggestion: suggestion,
|
||||
explanation: explanation,
|
||||
);
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// Requests a 4-skill adaptive mini-lesson that re-teaches a failed target.
|
||||
Future<GeneratedLesson?> generateAdaptiveLesson({
|
||||
required AiProviderType provider,
|
||||
|
||||
@@ -111,6 +111,24 @@ class WritingAiFeedback {
|
||||
final List<String> missing;
|
||||
}
|
||||
|
||||
/// Result of AI intervention during a dialogue turn when preset rules do not match.
|
||||
class DialogueAiIntervention {
|
||||
const DialogueAiIntervention({
|
||||
required this.accepted,
|
||||
this.suggestion,
|
||||
required this.explanation,
|
||||
});
|
||||
|
||||
/// Whether the learner's response is semantically acceptable for the turn.
|
||||
final bool accepted;
|
||||
|
||||
/// Inferred or corrected English sentence if there was an ASR slip, typo, or minor mistake.
|
||||
final String? suggestion;
|
||||
|
||||
/// Encouraging, concise Chinese explanation of the situation and recommendation.
|
||||
final String explanation;
|
||||
}
|
||||
|
||||
class AssessmentRecord {
|
||||
const AssessmentRecord({
|
||||
required this.packId,
|
||||
|
||||
@@ -164,7 +164,7 @@ class SherpaSttService {
|
||||
senseVoice: sherpa_onnx.OfflineSenseVoiceModelConfig(
|
||||
model: resolved['model']!,
|
||||
language: 'en',
|
||||
useInverseTextNormalization: true,
|
||||
useInverseTextNormalization: false,
|
||||
),
|
||||
tokens: resolved['tokens']!,
|
||||
numThreads: 2,
|
||||
|
||||
@@ -14,6 +14,42 @@ class VoiceService {
|
||||
VoiceService._();
|
||||
static final instance = VoiceService._();
|
||||
|
||||
/// Reverses the inverse text normalization (ITN) performed by system speech
|
||||
/// recognisers, which convert spoken numbers like "one" into "1".
|
||||
/// For an English-learning app the learner needs the spelled-out words.
|
||||
static String reverseItn(String text) {
|
||||
// Standalone digit → word. Uses word-boundary anchors so "12" or "100" are
|
||||
// left alone (those are unlikely to be single-word utterances the learner
|
||||
// intended as words).
|
||||
const digitToWord = {
|
||||
'0': 'zero',
|
||||
'1': 'one',
|
||||
'2': 'two',
|
||||
'3': 'three',
|
||||
'4': 'four',
|
||||
'5': 'five',
|
||||
'6': 'six',
|
||||
'7': 'seven',
|
||||
'8': 'eight',
|
||||
'9': 'nine',
|
||||
'10': 'ten',
|
||||
'11': 'eleven',
|
||||
'12': 'twelve',
|
||||
'13': 'thirteen',
|
||||
'14': 'fourteen',
|
||||
'15': 'fifteen',
|
||||
'16': 'sixteen',
|
||||
'17': 'seventeen',
|
||||
'18': 'eighteen',
|
||||
'19': 'nineteen',
|
||||
'20': 'twenty',
|
||||
};
|
||||
return text.replaceAllMapped(
|
||||
RegExp(r'\b(\d{1,2})\b'),
|
||||
(m) => digitToWord[m.group(1)!] ?? m.group(0)!,
|
||||
);
|
||||
}
|
||||
|
||||
final FlutterTts _tts = FlutterTts();
|
||||
final SpeechToText _stt = SpeechToText();
|
||||
final AudioRecorder _recorder = AudioRecorder();
|
||||
@@ -228,7 +264,7 @@ class VoiceService {
|
||||
|
||||
await _stt.listen(
|
||||
onResult: (result) =>
|
||||
onResult(result.recognizedWords, result.finalResult),
|
||||
onResult(reverseItn(result.recognizedWords), result.finalResult),
|
||||
listenOptions: SpeechListenOptions(
|
||||
localeId: targetLocaleId,
|
||||
listenFor: const Duration(seconds: 30),
|
||||
|
||||
Reference in New Issue
Block a user