Files

1140 lines
42 KiB
Dart
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/// 单元内容包(assets/courses)的离线校验,规则见 Doc/COURSE-PACK-JSON.md
/// 与《学习引擎规格》5.4–5.6。供命令行工具和测试共用;match 的判定直接用
/// App 的 answer_rules.dart,校验结果和运行时判分一致。
library;
import 'dart:convert';
import 'dart:io';
import 'package:kouyu_english/core/courses/answer_rules.dart' show matchesRule;
const levelOrder = ['A0', 'A1', 'A2', 'B1', 'B2'];
int levelRank(String level) {
final index = levelOrder.indexOf(level);
return index < 0 ? 99 : index;
}
String nextLevel(String level) =>
levelOrder[(levelRank(level) + 1).clamp(0, levelOrder.length - 1)];
// ------------------------------------------------------------------ CSV
List<List<String>> parseCsv(String text) {
final rows = <List<String>>[];
var row = <String>[];
final field = StringBuffer();
var quoted = false;
for (var i = 0; i < text.length; i++) {
final char = text[i];
if (quoted) {
if (char == '"') {
if (i + 1 < text.length && text[i + 1] == '"') {
field.write('"');
i++;
} else {
quoted = false;
}
} else {
field.write(char);
}
} else if (char == '"') {
quoted = true;
} else if (char == ',') {
row.add(field.toString());
field.clear();
} else if (char == '\n' || char == '\r') {
if (char == '\r' && i + 1 < text.length && text[i + 1] == '\n') i++;
row.add(field.toString());
field.clear();
if (row.any((value) => value.isNotEmpty)) rows.add(row);
row = <String>[];
} else {
field.write(char);
}
}
if (field.isNotEmpty || row.isNotEmpty) {
row.add(field.toString());
rows.add(row);
}
return rows;
}
// ------------------------------------------------------------ 词形还原
const _irregular = {
'am': 'be', 'is': 'be', 'are': 'be', 'was': 'be', 'were': 'be',
'been': 'be', 'being': 'be', 'has': 'have', 'had': 'have',
'having': 'have', 'does': 'do', 'did': 'do', 'done': 'do', 'goes': 'go',
'went': 'go', 'gone': 'go', 'awoke': 'awake', 'born': 'bear',
'beaten': 'beat', 'became': 'become', 'began': 'begin', 'begun': 'begin',
'bent': 'bend', 'bit': 'bite', 'bitten': 'bite', 'bled': 'bleed',
'blew': 'blow', 'blown': 'blow', 'broke': 'break', 'broken': 'break',
'brought': 'bring', 'built': 'build', 'burnt': 'burn', 'bought': 'buy',
'caught': 'catch', 'chose': 'choose', 'chosen': 'choose', 'came': 'come',
'dealt': 'deal', 'dug': 'dig', 'drew': 'draw', 'drawn': 'draw',
'dreamt': 'dream', 'drank': 'drink', 'drunk': 'drink', 'drove': 'drive',
'driven': 'drive', 'ate': 'eat', 'eaten': 'eat', 'fell': 'fall',
'fallen': 'fall', 'fed': 'feed', 'felt': 'feel', 'fought': 'fight',
'found': 'find', 'flew': 'fly', 'flown': 'fly', 'forgot': 'forget',
'forgotten': 'forget', 'forgave': 'forgive', 'forgiven': 'forgive',
'froze': 'freeze', 'frozen': 'freeze', 'got': 'get', 'gotten': 'get',
'gave': 'give', 'given': 'give', 'grew': 'grow', 'grown': 'grow',
'hung': 'hang', 'heard': 'hear', 'hid': 'hide', 'hidden': 'hide',
'held': 'hold', 'kept': 'keep', 'knew': 'know', 'known': 'know',
'laid': 'lay', 'led': 'lead', 'learnt': 'learn', 'left': 'leave',
'lent': 'lend', 'lay': 'lie', 'lain': 'lie', 'lit': 'light',
'lost': 'lose', 'made': 'make', 'meant': 'mean', 'met': 'meet',
'paid': 'pay', 'rode': 'ride', 'ridden': 'ride', 'rang': 'ring',
'rung': 'ring', 'rose': 'rise', 'risen': 'rise', 'ran': 'run',
'said': 'say', 'saw': 'see', 'seen': 'see', 'sought': 'seek',
'sold': 'sell', 'sent': 'send', 'shook': 'shake', 'shaken': 'shake',
'shone': 'shine', 'shot': 'shoot', 'shown': 'show', 'sang': 'sing',
'sung': 'sing', 'sank': 'sink', 'sunk': 'sink', 'sat': 'sit',
'slept': 'sleep', 'slid': 'slide', 'spoke': 'speak', 'spoken': 'speak',
'spent': 'spend', 'spilt': 'spill', 'stood': 'stand', 'stole': 'steal',
'stolen': 'steal', 'stuck': 'stick', 'stung': 'sting', 'struck': 'strike',
'swam': 'swim', 'swum': 'swim', 'swung': 'swing', 'took': 'take',
'taken': 'take', 'taught': 'teach', 'tore': 'tear', 'torn': 'tear',
'told': 'tell', 'thought': 'think', 'threw': 'throw', 'thrown': 'throw',
'understood': 'understand', 'woke': 'wake', 'woken': 'wake',
'wore': 'wear', 'worn': 'wear', 'won': 'win', 'wrote': 'write',
'written': 'write', 'children': 'child', 'men': 'man', 'women': 'woman',
'feet': 'foot', 'teeth': 'tooth', 'mice': 'mouse', 'knives': 'knife',
'wives': 'wife', 'lives': 'life', 'leaves': 'leaf', 'shelves': 'shelf',
'halves': 'half', 'better': 'good', 'best': 'good', 'worse': 'bad',
'worst': 'bad', 'further': 'far', 'furthest': 'far', 'farther': 'far',
'less': 'little', 'least': 'little', 'more': 'much', 'most': 'much',
'an': 'a', 'cannot': 'can', 'thanks': 'thank', 'people': 'person',
'mr': 'mr', 'wifi': 'wi-fi',
};
/// 一个词可能对应的词头(包括它本身),按可能性排序。
List<String> lemmaCandidates(String word) {
final result = <String>[word];
void add(String value) {
if (value.length > 1 && !result.contains(value)) result.add(value);
}
final irregular = _irregular[word];
if (irregular != null) add(irregular);
bool doubled(String stem) =>
stem.length > 2 && stem[stem.length - 1] == stem[stem.length - 2];
if (word.endsWith('ies')) add('${word.substring(0, word.length - 3)}y');
if (word.endsWith('ves')) add('${word.substring(0, word.length - 3)}f');
if (word.endsWith('es')) add(word.substring(0, word.length - 2));
if (word.endsWith('s') && !word.endsWith('ss')) {
add(word.substring(0, word.length - 1));
}
if (word.endsWith('ied')) add('${word.substring(0, word.length - 3)}y');
if (word.endsWith('ed')) {
final stem = word.substring(0, word.length - 2);
add(stem);
add('${stem}e');
if (doubled(stem)) add(stem.substring(0, stem.length - 1));
}
if (word.endsWith('ying')) add('${word.substring(0, word.length - 4)}ie');
if (word.endsWith('ing')) {
final stem = word.substring(0, word.length - 3);
add(stem);
add('${stem}e');
if (doubled(stem)) add(stem.substring(0, stem.length - 1));
}
for (final suffix in ['er', 'est']) {
if (!word.endsWith(suffix)) continue;
final stem = word.substring(0, word.length - suffix.length);
add(stem);
add('${stem}e');
if (stem.endsWith('i')) add('${stem.substring(0, stem.length - 1)}y');
if (doubled(stem)) add(stem.substring(0, stem.length - 1));
}
if (word.endsWith('ly')) {
final stem = word.substring(0, word.length - 2);
add(stem);
add('${stem}e');
if (stem.endsWith('i')) add('${stem.substring(0, stem.length - 1)}y');
if (word.endsWith('ally')) add(word.substring(0, word.length - 4));
if (word.endsWith('ly') && !word.endsWith('lly')) {
add('${word.substring(0, word.length - 1)}e');
}
}
return result;
}
// ------------------------------------------------------------ 分词
String _normalizeQuotes(String text) => text
.replaceAll(RegExp(r'[‘’`´]'), "'")
.replaceAll(RegExp(r'[“”]'), '"');
const _negations = {
"can't": ['can', 'not'], "won't": ['will', 'not'],
"shan't": ['shall', 'not'], "ain't": ['be', 'not'],
};
/// 把文本拆成词。数字、带数字的词被跳过;缩写展开;连字符词保留整体,
/// 由调用方决定是否再拆开检查。
List<String> tokenize(String text) {
var normalized = _normalizeQuotes(text).toLowerCase();
normalized = normalized.replaceAllMapped(
RegExp(r'\b([ap])\.m\.'),
(match) => '${match[1]}m',
);
final words = <String>[];
for (final match in RegExp(
r"[a-z0-9]+(?:['-][a-z0-9]+)*",
).allMatches(normalized)) {
final raw = match[0]!;
if (RegExp(r'\d').hasMatch(raw)) continue;
if (_negations.containsKey(raw)) {
words.addAll(_negations[raw]!);
continue;
}
if (raw.endsWith("n't")) {
words
..add(raw.substring(0, raw.length - 3))
..add('not');
continue;
}
if (raw == "let's") {
words.addAll(['let', 'us']);
continue;
}
if (raw == "o'clock") {
words.add(raw);
continue;
}
final apostrophe = RegExp(r"^(.+)'(m|re|s|ll|ve|d)$").firstMatch(raw);
if (apostrophe != null) {
words.add(apostrophe[1]!);
const expansions = {
'm': 'am', 're': 'are', 'll': 'will', 've': 'have', 'd': 'would',
};
final expansion = expansions[apostrophe[2]];
if (expansion != null) words.add(expansion);
continue;
}
if (raw.endsWith("s'")) {
words.add(raw.substring(0, raw.length - 1));
continue;
}
words.add(raw);
}
return words;
}
/// 用于判定条件匹配的规范化:小写、统一引号、展开常见缩写、去掉标点。
String normalizeForMatch(String text) {
var value = _normalizeQuotes(text).toLowerCase();
value = value
.replaceAll(RegExp(r"\bcan't\b|\bcannot\b"), 'can not')
.replaceAll(RegExp(r"\bwon't\b"), 'will not')
.replaceAll("n't", ' not')
.replaceAll("'m", ' am')
.replaceAll("'re", ' are')
.replaceAll("'ll", ' will')
.replaceAll("'ve", ' have')
.replaceAll("'d", ' would')
;
value = value.replaceAllMapped(
RegExp(r"\b(it|that|what|where|here|there|he|she|how|who|when|this)'s\b"),
(match) => '${match[1]} is',
);
value = value.replaceAll(RegExp(r"[^a-z0-9' ]"), ' ');
return value.replaceAll(RegExp(r'\s+'), ' ').trim();
}
/// 规范化后的 [normalizedText] 是否以完整词语的形式含有 [phrase],用于
/// 查找选项在原文里是否出现;match 规则的判定用 answer_rules.dart。
bool containsPhrase(String normalizedText, String phrase) {
final normalizedPhrase = normalizeForMatch(phrase);
if (normalizedPhrase.isEmpty) return false;
return RegExp(
'(^| )${RegExp.escape(normalizedPhrase)}(\$| )',
).hasMatch(normalizedText);
}
// ------------------------------------------------------------ 词表
const functionPos = {
'pronoun', 'preposition', 'determiner', 'conjunction', 'modal auxiliary',
'be-verb', 'do-verb', 'have-verb', 'infinitive-to',
};
class LexiconEntry {
LexiconEntry(this.headword);
final String headword;
final Set<String> levels = {};
/// 词性 → 该词性下的最低等级。
final Map<String, String> pos = {};
String get minLevel =>
(levels.toList()..sort((a, b) => levelRank(a) - levelRank(b))).first;
}
class Lexicon {
final Map<String, LexiconEntry> entries = {};
static Lexicon load(File file) {
final lexicon = Lexicon();
for (final row in parseCsv(file.readAsStringSync()).skip(1)) {
if (row.length < 3) continue;
for (final variant in headwordVariants(row[0])) {
final entry = lexicon.entries.putIfAbsent(
variant,
() => LexiconEntry(row[0]),
);
entry.levels.add(row[2]);
final previous = entry.pos[row[1]];
if (previous == null || levelRank(row[2]) < levelRank(previous)) {
entry.pos[row[1]] = row[2];
}
}
}
return lexicon;
}
static List<String> headwordVariants(String headword) => [
for (final part in headword.split('/'))
if (part.trim().isNotEmpty)
part.trim().toLowerCase().replaceAll('.', ''),
];
final Map<String, List<String>> _lemmaCache = {};
/// 词本身是词表词头时只认它本身和不规则变化,避免 only → on 这类误还原。
List<String> lemmas(String word) => _lemmaCache.putIfAbsent(word, () {
if (!entries.containsKey(word)) return lemmaCandidates(word);
// 词头本身存在时,只接受名词复数、动词 -s/-ed/-ing 的还原。
final irregular = _irregular[word];
return [
word,
?irregular,
// bed、news、feed 这类短词本身就是词头,不能还原成 be、new、fee;
// thing 也不能还原成 the。
if (word.length >= 5 &&
!word.endsWith('eed') &&
!RegExp(r'^.{1,2}ing$').hasMatch(word) &&
RegExp(r'(s|ed|ing)$').hasMatch(word))
for (final candidate in lemmaCandidates(word).skip(1))
if (entries.containsKey(candidate) &&
!RegExp(r'(ly|er|est)$').hasMatch(word))
candidate,
];
});
LexiconEntry? lookup(String word) {
for (final candidate in lemmas(word)) {
final entry = entries[candidate];
if (entry != null) return entry;
}
return null;
}
Iterable<String> functionWordsUpTo(String level) sync* {
for (final MapEntry(:key, :value) in entries.entries) {
if (value.pos.entries.any(
(pos) =>
functionPos.contains(pos.key) &&
levelRank(pos.value) <= levelRank(level),
)) {
yield key;
}
}
}
}
class PoolWord {
const PoolWord(this.headword, this.categories, this.topic);
final String headword;
final String categories;
final String topic;
}
List<PoolWord> loadPool(File file) => [
for (final row in parseCsv(file.readAsStringSync()).skip(1))
if (row.length >= 6 && row[4] == 'pool' && row[5] != 'yes')
PoolWord(row[0], row[3], row[2]),
];
// ------------------------------------------------------------ 结果
class CourseIssue {
const CourseIssue(this.where, this.message);
final String where;
final String message;
@override
String toString() => '$where: $message';
}
class LevelCoverage {
LevelCoverage(this.level, this.poolSize, this.uncovered, this.unitsReady);
final String level;
final int poolSize;
final List<PoolWord> uncovered;
final bool unitsReady;
int get covered => poolSize - uncovered.length;
double get ratio => poolSize == 0 ? 1 : covered / poolSize;
}
class CourseReport {
final List<CourseIssue> errors = [];
final List<String> missingUnits = [];
final Map<String, LevelCoverage> coverage = {};
final Map<String, Map<String, String>> itemLevels = {};
}
// ------------------------------------------------------------ 校验
class CourseValidator {
CourseValidator({
required this.projectRoot,
required Iterable<String> a0Texts,
}) : lexicon = Lexicon.load(
File('${projectRoot.path}/tool/courses/lexicon/cefrj-vocabulary-profile-1.5.csv'),
),
grammarLevels = _loadGrammar(
File('${projectRoot.path}/tool/courses/lexicon/cefrj-grammar-profile-20180315.csv'),
) {
for (final text in a0Texts) {
for (final word in tokenize(text)) {
a0Words.addAll(lexicon.lemmas(word));
}
}
}
final Directory projectRoot;
final Lexicon lexicon;
final Map<String, String> grammarLevels;
final Set<String> a0Words = {};
static Map<String, String> _loadGrammar(File file) {
final rows = parseCsv(file.readAsStringSync()).skip(1).toList();
final levels = <String, String>{};
for (final row in rows) {
if (row.length < 5) continue;
levels[row[0]] = row[4];
}
// 变体(如 59-1)未标等级时沿用主项等级。
for (final row in rows) {
if (row.length < 5) continue;
final value = levels[row[0]] ?? '';
if (!RegExp(r'^[ABC]\d').hasMatch(value)) {
final main = row[0].split('-').first;
levels[row[0]] = RegExp(r'^[ABC]\d').hasMatch(levels[main] ?? '')
? levels[main]!
: '';
}
}
return levels;
}
CourseReport validate() {
final report = CourseReport();
final coursesDir = '${projectRoot.path}/assets/courses';
final map =
jsonDecode(File('$coursesDir/course-map.json').readAsStringSync())
as Map<String, dynamic>;
final globalAllowed = <String>{
for (final word in map['alwaysAllowedWords'] as List) word as String,
for (final word in map['names'] as List) word as String,
};
final seenIds = <String, String>{};
final taught = <String>{...a0Words};
final coveredByLevel = <String, Set<String>>{};
final coveredSoFar = <String>{...a0Words};
for (final level in (map['levels'] as List).cast<Map<String, dynamic>>()) {
final levelId = level['id'] as String;
// A0 is the frozen baseline: its words are passed in as `a0Texts`, and
// it has no word pool or unit contract of the later levels.
if (levelId == 'A0') continue;
final functionWords = <String>{...lexicon.functionWordsUpTo(levelId)};
var ready = true;
for (final entry in (level['units'] as List).cast<Map<String, dynamic>>()) {
final unitId = entry['id'] as String;
final file = File('$coursesDir/${entry['file']}');
if (!file.existsSync()) {
report.missingUnits.add(unitId);
ready = false;
continue;
}
Map<String, dynamic> unit;
try {
unit = jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
} on FormatException catch (error) {
report.errors.add(CourseIssue(unitId, 'JSON 无法解析:${error.message}'));
ready = false;
continue;
}
final context = _UnitContext(
validator: this,
report: report,
level: level,
mapEntry: entry,
unit: unit,
allowed: {...globalAllowed, ...functionWords},
taughtBefore: taught,
seenIds: seenIds,
);
context.run();
taught.addAll(context.unitTaught);
coveredSoFar.addAll(context.unitCovered);
}
coveredByLevel[levelId] = {...coveredSoFar};
final pool = loadPool(
File('${projectRoot.path}/tool/courses/pools/${levelId.toLowerCase()}-word-pool.csv'),
);
final uncovered = [
for (final word in pool)
if (!_isCovered(word.headword, coveredSoFar)) word,
];
final coverage = LevelCoverage(levelId, pool.length, uncovered, ready);
report.coverage[levelId] = coverage;
final minimum = (map['minimumPoolCoverage'] as num?)?.toDouble() ?? 0.85;
if (ready && coverage.ratio < minimum) {
report.errors.add(CourseIssue(
levelId,
'词池覆盖率 ${(coverage.ratio * 100).toStringAsFixed(1)}% 低于 '
'${(minimum * 100).round()}%${coverage.covered}/${coverage.poolSize}',
));
}
}
return report;
}
bool _isCovered(String headword, Set<String> covered) {
for (final variant in Lexicon.headwordVariants(headword)) {
final words = tokenize(variant);
if (words.isEmpty) continue;
if (words.every(covered.contains)) return true;
}
return false;
}
}
class _UnitContext {
_UnitContext({
required this.validator,
required this.report,
required this.level,
required this.mapEntry,
required this.unit,
required this.allowed,
required this.taughtBefore,
required this.seenIds,
}) : unitId = mapEntry['id'] as String,
levelId = level['id'] as String;
final CourseValidator validator;
final CourseReport report;
final Map<String, dynamic> level;
final Map<String, dynamic> mapEntry;
final Map<String, dynamic> unit;
final Set<String> allowed;
final Set<String> taughtBefore;
final Map<String, String> seenIds;
final String unitId;
final String levelId;
/// 本单元教过的词(产出核心和理解词中的全部词),供后续单元使用。
final Set<String> unitTaught = {};
/// 本单元计入词池覆盖的词。
final Set<String> unitCovered = {};
final Map<String, Map<String, dynamic>> core = {};
final Map<String, Map<String, dynamic>> receptive = {};
void error(String where, String message) =>
report.errors.add(CourseIssue('$unitId $where', message));
String _string(Map<String, dynamic> json, String key, String where) {
final value = json[key];
if (value is String && value.trim().isNotEmpty) return value;
error(where, '缺少非空字符串 $key');
return '';
}
List<Map<String, dynamic>> _objects(
Map<String, dynamic> json,
String key,
String where, {
bool required = true,
}) {
final value = json[key];
if (value == null && !required) return const [];
if (value is List && value.every((item) => item is Map<String, dynamic>)) {
return value.cast<Map<String, dynamic>>();
}
error(where, '$key 必须是对象数组');
return const [];
}
List<String> _strings(Map<String, dynamic> json, String key, String where) {
final value = json[key];
if (value is List && value.every((item) => item is String)) {
return value.cast<String>();
}
error(where, '$key 必须是字符串数组');
return const [];
}
void _registerId(String id, String where) {
final previous = seenIds[id];
if (previous != null) {
error(where, 'ID $id$previous 重复');
} else {
seenIds[id] = where;
}
}
void _addWords(Set<String> target, String text) {
for (final word in tokenize(text)) {
target.addAll(validator.lexicon.lemmas(word));
if (word.contains('-')) {
for (final part in word.split('-')) {
target.addAll(validator.lexicon.lemmas(part));
}
}
}
}
late final Set<String> known;
bool _isKnown(String word) {
bool single(String value) =>
validator.lexicon.lemmas(value).any(known.contains) || value.length == 1;
if (single(word)) return true;
return word.contains('-') && word.split('-').every(single);
}
void run() {
if (unit['schemaVersion'] != 1) error('', 'schemaVersion 必须为 1');
if (unit['id'] != unitId) error('', 'id 应为 $unitId');
if (unit['level'] != levelId) error('', 'level 应为 $levelId');
for (final key in ['title', 'canDo']) {
if (unit[key] != mapEntry[key]) error(key, '与 course-map.json 不一致');
}
if (jsonEncode(unit['categories']) != jsonEncode(mapEntry['categories'])) {
error('categories', '与 course-map.json 不一致');
}
if (unit['revision'] is! int) error('revision', '必须是整数');
if (!{'draft', 'reviewed', 'frozen'}.contains(unit['status'])) {
error('status', '必须是 draft/reviewed/frozen');
}
final meta = unit['meta'];
if (meta is! Map<String, dynamic>) {
error('meta', '缺少元数据');
} else {
_string(meta, 'draftedBy', 'meta');
_string(meta, 'draftedAt', 'meta');
if (meta['reviews'] is! List) error('meta', 'reviews 必须是数组');
}
_grammar();
_items();
known = {
...allowed,
...taughtBefore,
...unitTaught,
for (final name in (unit['names'] as List? ?? const [])) '$name'.toLowerCase(),
};
_itemLevels();
final usage = <String, Set<String>>{
for (final id in core.keys) id: <String>{},
};
final texts = <String>[];
_segments(usage, texts);
_scenes(usage, texts);
_materials(texts);
for (final text in texts) {
_addWords(unitCovered, text);
}
for (final MapEntry(:key, :value) in usage.entries) {
for (final need in ['segment', 'practice', 'scene', 'task']) {
if (!value.contains(need)) {
const labels = {
'segment': '没有被任何教学段列入 itemIds',
'practice': '没有在教学段的听、说、读、写示范里用到',
'scene': '没有在任何场景的示范回答里用到',
'task': '没有被任何任务引用',
};
error(key, labels[need]!);
}
}
}
// 直接用原文分词:normalizeForMatch 会去掉连字符,T-shirt 就再也找不到了。
final textWords = <String>{};
_addWords(textWords, texts.join('\n'));
for (final MapEntry(:key, :value) in receptive.entries) {
final words = tokenize(value['en'] as String? ?? '');
final present = words.isNotEmpty &&
words.every((word) => validator.lexicon.lemmas(word).any(textWords.contains));
if (!present) error(key, '理解词 ${value['en']} 没有出现在本单元的听读或场景文本中');
}
}
void _grammar() {
for (final (index, item) in _objects(unit, 'grammar', 'grammar').indexed) {
final where = 'grammar[$index]';
final id = '${item['id']}';
final grammarLevel = validator.grammarLevels[id];
if (grammarLevel == null) {
error(where, 'CEFR-J 语法表中没有 ID $id');
continue;
}
if (!{'active', 'receptive'}.contains(item['use'])) {
error(where, 'use 必须是 active 或 receptive');
}
final base = grammarLevel.length >= 2 ? grammarLevel.substring(0, 2) : '';
if (item['use'] == 'active' &&
levelRank(base) > levelRank(levelId) &&
'${item['reason'] ?? ''}'.trim().isEmpty) {
error(where, '语法 $id$grammarLevel,高于 $levelId,须写 reason');
}
}
}
void _items() {
final range = (level['coreRange'] as List).cast<int>();
final items = _objects(unit, 'coreItems', 'coreItems');
if (items.length < range[0] || items.length > range[1]) {
error('coreItems', '产出核心 ${items.length} 项,应为 ${range[0]}${range[1]}');
}
for (final (index, item) in items.indexed) {
final id = '${item['id']}';
final where = 'coreItems[$index]';
final type = item['type'];
final prefix = type == 'word' ? 'W' : 'P';
if (!{'word', 'phrase', 'pattern'}.contains(type)) {
error(where, 'type 必须是 word/phrase/pattern');
}
if (!RegExp('^$unitId-$prefix\\d{2}\$').hasMatch(id)) {
error(where, 'ID $id 应为 $unitId-${prefix}NN');
}
_registerId(id, where);
core[id] = item;
final en = _string(item, 'en', id);
_string(item, 'zh', id);
_addWords(unitTaught, en);
_addWords(unitCovered, en);
final match = _match(item, id);
if (en.isNotEmpty && match.isNotEmpty && !matchesRule(en, match)) {
error(id, 'en 本身不满足 match');
}
final example = item['example'];
if (example is! Map<String, dynamic>) {
error(id, '缺少 example');
} else {
final exampleEn = _string(example, 'en', '$id.example');
_string(example, 'zh', '$id.example');
if (match.isNotEmpty && !matchesRule(exampleEn, match)) {
error(id, 'example.en 不满足 match');
}
}
final review = item['review'];
if (review is! Map<String, dynamic>) {
error(id, '缺少 review');
} else {
final prompt = _string(review, 'prompt', '$id.review');
if (en.isNotEmpty && normalizeForMatch(prompt).contains(normalizeForMatch(en))) {
error(id, 'review.prompt 不能直接给出英文答案');
}
}
if (item['sceneWord'] == true && '${item['sceneReason'] ?? ''}'.trim().isEmpty) {
error(id, '场景词须写 sceneReason');
}
}
final sceneWords = items.where((item) => item['sceneWord'] == true).length;
if (sceneWords * 3 > items.length) {
error('coreItems', '场景词 $sceneWords 项,超过产出核心的 1/3');
}
final receptiveRange = (level['receptiveRange'] as List).cast<int>();
final words = _objects(unit, 'receptiveWords', 'receptiveWords');
if (words.length < receptiveRange[0] || words.length > receptiveRange[1]) {
error(
'receptiveWords',
'理解词 ${words.length} 项,应为 ${receptiveRange[0]}${receptiveRange[1]}',
);
}
for (final (index, word) in words.indexed) {
final id = '${word['id']}';
final where = 'receptiveWords[$index]';
if (!RegExp('^$unitId-R\\d{2,3}\$').hasMatch(id)) {
error(where, 'ID $id 应为 $unitId-RNN');
}
_registerId(id, where);
receptive[id] = word;
final en = _string(word, 'en', id);
_string(word, 'zh', id);
_addWords(unitTaught, en);
_addWords(unitCovered, en);
if (word['sceneWord'] == true && '${word['sceneReason'] ?? ''}'.trim().isEmpty) {
error(id, '场景词须写 sceneReason');
}
}
}
List<List<String>> _match(Map<String, dynamic> item, String id) {
final value = item['match'];
if (value is List &&
value.isNotEmpty &&
value.every(
(group) => group is List && group.isNotEmpty && group.every((term) => term is String),
)) {
return [
for (final group in value) (group as List).cast<String>(),
];
}
error(id, 'match 必须是非空的字符串数组的数组');
return const [];
}
/// 按锁定词表计算等级:本单元之前已教的词、功能词和专名不计。
String _itemLevel(String en) {
final phrase = validator.lexicon.entries[normalizeForMatch(en)];
if (phrase != null) return phrase.minLevel;
var worst = 'A0';
for (final word in tokenize(en)) {
if (allowed.contains(word) ||
validator.lexicon.lemmas(word).any(taughtBefore.contains)) {
continue;
}
final parts = word.contains('-') &&
validator.lexicon.lookup(word) == null
? word.split('-')
: [word];
for (final part in parts) {
final entry = validator.lexicon.lookup(part);
final value = entry?.minLevel ?? '未收录';
if (levelRank(value) > levelRank(worst)) worst = value;
}
}
return worst;
}
void _itemLevels() {
final levels = report.itemLevels.putIfAbsent(unitId, () => {});
for (final MapEntry(:key, :value) in core.entries) {
final computed = _itemLevel('${value['en']}');
levels[key] = computed;
if (levelRank(computed) > levelRank(levelId) && value['sceneWord'] != true) {
error(key, '${value['en']} 按 CEFR-J 为 $computed,高于 $levelId;须改写或标为场景词');
}
}
final limit = nextLevel(levelId);
for (final MapEntry(:key, :value) in receptive.entries) {
final words = tokenize('${value['en']}');
if (words.isNotEmpty &&
words.every(
(word) =>
allowed.contains(word) ||
validator.lexicon.lemmas(word).any(taughtBefore.contains),
)) {
error(key, '理解词 ${value['en']} 在 A0 或之前的单元已经教过');
}
final computed = _itemLevel('${value['en']}');
levels[key] = computed;
if (levelRank(computed) > levelRank(limit) && value['sceneWord'] != true) {
error(key, '${value['en']} 按 CEFR-J 为 $computed,高于 $limit;须换词或标为场景词');
}
}
}
void _checkText(String where, String text, {Map<String, dynamic>? glosses, double tolerance = 0}) {
final words = tokenize(text);
final unknown = <String>[];
for (final word in words) {
if (!_isKnown(word)) unknown.add(word);
}
if (unknown.isEmpty) return;
final glossed = {
for (final key in (glosses ?? const {}).keys) key.toLowerCase(),
};
final missingGloss = unknown.where((word) => !glossed.contains(word)).toSet();
if (tolerance == 0) {
error(where, '未教的词:${unknown.toSet().join(', ')}');
return;
}
if (unknown.length > words.length * tolerance) {
error(
where,
'超纲词 ${unknown.length}/${words.length} 超过 ${(tolerance * 100).round()}%'
'${unknown.toSet().join(', ')}',
);
}
if (missingGloss.isNotEmpty) {
error(where, '超纲词缺少 glosses 释义:${missingGloss.join(', ')}');
}
}
List<String> _itemRefs(Map<String, dynamic> json, String where) {
final ids = _strings(json, 'itemIds', where);
for (final id in ids) {
if (!core.containsKey(id)) error(where, '引用了不存在的产出核心 $id');
}
return ids;
}
void _checkOptions(String where, Map<String, dynamic> json, {String? sourceText}) {
final options = _strings(json, 'options', where);
if (options.length != 3) {
error(where, '必须有 3 个选项(第一个为正确答案)');
return;
}
final normalized = [for (final option in options) normalizeForMatch(option)];
final english = options.every((option) => RegExp(r'^[A-Za-z0-9 ,.?!\x27-]+$').hasMatch(option));
for (var i = 0; i < options.length; i++) {
if (options[i].trim().isEmpty) error(where, '选项不能为空');
for (var j = 0; j < options.length; j++) {
if (i == j) continue;
// 中文选项直接比较原文,避免“4 号通道最上层”被规范化成“4”。
final a = english ? normalized[i] : options[i].trim();
final b = english ? normalized[j] : options[j].trim();
if (b.contains(a)) {
error(where, '选项“${options[i]}”是“${options[j]}”的一部分');
}
}
}
if (english) {
for (final option in options) {
_checkText('$where.options', option);
}
if (sourceText != null) {
final text = normalizeForMatch(sourceText);
final present = normalized.where((option) => containsPhrase(text, option)).length;
if (present < 2) error(where, '英文选项至少两个要在原文里出现');
if (!containsPhrase(text, normalized.first)) error(where, '正确选项没有在原文中出现');
}
}
}
void _segments(Map<String, Set<String>> usage, List<String> texts) {
final segments = _objects(unit, 'segments', 'segments');
if (segments.length < 2 || segments.length > 4) {
error('segments', '教学段应为 24 段');
}
for (final (index, segment) in segments.indexed) {
final id = '${segment['id']}';
if (id != '$unitId-S${index + 1}') error('segments[$index]', 'ID 应为 $unitId-S${index + 1}');
_registerId(id, 'segments[$index]');
_string(segment, 'title', id);
final minutes = segment['minutes'];
if (minutes is! int || minutes < 8 || minutes > 15) {
error(id, 'minutes 应为 815 的整数');
}
final itemIds = _itemRefs(segment, id);
for (final itemId in itemIds) {
usage[itemId]?.add('segment');
}
if (itemIds.length > 6) error(id, '一段最多 6 个新产出核心');
final practice = <String>[];
final listening = segment['listening'];
if (listening is Map<String, dynamic>) {
final text = _string(listening, 'text', '$id.listening');
_string(listening, 'question', '$id.listening');
_checkText('$id.listening.text', text);
_checkOptions('$id.listening', listening);
practice.add(text);
} else {
error(id, '缺少 listening');
}
final speaking = _objects(segment, 'speaking', id);
if (speaking.isEmpty) error(id, 'speaking 至少 1 句');
for (final (i, line) in speaking.indexed) {
final text = _string(line, 'text', '$id.speaking[$i]');
_checkText('$id.speaking[$i]', text);
practice.add(text);
}
final reading = segment['reading'];
if (reading is Map<String, dynamic>) {
final text = _string(reading, 'text', '$id.reading');
_string(reading, 'question', '$id.reading');
_checkText('$id.reading.text', text);
_checkOptions('$id.reading', reading, sourceText: text);
practice.add(text);
} else {
error(id, '缺少 reading');
}
final writing = segment['writing'];
if (writing is Map<String, dynamic>) {
_string(writing, 'prompt', '$id.writing');
final example = _string(writing, 'example', '$id.writing');
_checkText('$id.writing.example', example);
for (final ref in _itemRefs(writing, '$id.writing')) {
final item = core[ref];
if (item != null && !matchesRule(example, _match(item, ref))) {
error('$id.writing', '示范没有用到 $ref');
}
}
practice.add(example);
} else {
error(id, '缺少 writing');
}
final independent = segment['independent'];
if (independent is Map<String, dynamic>) {
_string(independent, 'prompt', '$id.independent');
if (_itemRefs(independent, '$id.independent').isEmpty) {
error('$id.independent', 'itemIds 不能为空');
}
} else {
error(id, '缺少 independent');
}
for (final itemId in itemIds) {
final item = core[itemId];
if (item == null) continue;
if (practice.any((text) => matchesRule(text, _match(item, itemId)))) {
usage[itemId]?.add('practice');
}
}
texts.addAll(practice);
}
}
void _scenes(Map<String, Set<String>> usage, List<String> texts) {
final scenes = _objects(unit, 'scenes', 'scenes');
if (scenes.length < 3 || scenes.length > 4) error('scenes', '应为 1 个主场景加 2–3 个变式场景');
if (scenes.where((scene) => scene['kind'] == 'main').length != 1) {
error('scenes', '必须恰好有 1 个 kind=main 的主场景');
}
final sceneIds = <String>{};
for (final (index, scene) in scenes.indexed) {
final id = '${scene['id']}';
if (id != '$unitId-C${index + 1}') error('scenes[$index]', 'ID 应为 $unitId-C${index + 1}');
_registerId(id, 'scenes[$index]');
sceneIds.add(id);
if (!{'main', 'variant'}.contains(scene['kind'])) error(id, 'kind 必须是 main/variant');
for (final key in ['title', 'learnerRole', 'aiRole', 'setting']) {
_string(scene, key, id);
}
final turns = _objects(scene, 'turns', id);
if (turns.length < 4 || turns.length > 7) error(id, '场景应有 47 轮');
for (final (i, turn) in turns.indexed) {
final where = '$id.turns[$i]';
final ai = _string(turn, 'ai', where);
_string(turn, 'goal', where);
final model = _string(turn, 'model', where);
_string(turn, 'zh', where);
_checkText('$where.ai', ai);
_checkText('$where.model', model);
texts
..add(ai)
..add(model);
for (final ref in _itemRefs(turn, where)) {
final item = core[ref];
if (item == null) continue;
if (matchesRule(model, _match(item, ref))) {
usage[ref]?.add('scene');
} else {
error(where, 'model 没有用到 $ref');
}
}
}
}
final tasks = _objects(unit, 'tasks', 'tasks');
final tasked = <String>{};
for (final (index, task) in tasks.indexed) {
final id = '${task['id']}';
if (id != '$unitId-T${index + 1}') error('tasks[$index]', 'ID 应为 $unitId-T${index + 1}');
_registerId(id, 'tasks[$index]');
_string(task, 'goal', id);
final sceneId = '${task['sceneId']}';
if (!sceneIds.contains(sceneId)) error(id, '引用了不存在的场景 $sceneId');
tasked.add(sceneId);
if (_strings(task, 'slots', id).isEmpty) error(id, 'slots 至少 1 个');
for (final ref in _itemRefs(task, id)) {
usage[ref]?.add('task');
}
}
for (final sceneId in sceneIds.difference(tasked)) {
error(sceneId, '没有对应的任务');
}
}
void _materials(List<String> texts) {
final spec = level['materials'];
final materials = _objects(unit, 'materials', 'materials', required: spec != null);
if (spec is! Map<String, dynamic>) {
if (materials.isNotEmpty) error('materials', '$levelId 不安排独立听读材料');
return;
}
final count = (spec['count'] as List).cast<int>();
if (materials.length < count[0] || materials.length > count[1]) {
error('materials', '听读材料 ${materials.length} 篇,应为 ${count[0]}${count[1]}');
}
final modes = <String>{};
for (final (index, material) in materials.indexed) {
final id = '${material['id']}';
if (id != '$unitId-M${index + 1}') error('materials[$index]', 'ID 应为 $unitId-M${index + 1}');
_registerId(id, 'materials[$index]');
_string(material, 'title', id);
final mode = material['mode'];
if (mode is! String || !{'listening', 'reading'}.contains(mode)) {
error(id, 'mode 必须是 listening/reading');
continue;
}
modes.add(mode);
final text = _string(material, 'text', id);
final wordCount = RegExp(r"[A-Za-z0-9'-]+").allMatches(text).length;
final range = (spec['${mode}Words'] as List).cast<int>();
if (wordCount < range[0] || wordCount > range[1]) {
error(id, '$mode 材料 $wordCount 词,应为 ${range[0]}${range[1]}');
}
final glosses = material['glosses'];
if (glosses != null && glosses is! Map<String, dynamic>) {
error(id, 'glosses 必须是对象');
}
_checkText(
'$id.text',
text,
glosses: glosses is Map<String, dynamic> ? glosses : null,
tolerance: (spec['unknownTolerance'] as num?)?.toDouble() ?? 0.02,
);
texts.add(text);
final questions = _objects(material, 'questions', id);
if (questions.length < 2 || questions.length > 4) error(id, '理解题应为 24 道');
for (final (i, question) in questions.indexed) {
final where = '$id.questions[$i]';
_string(question, 'question', where);
if (!{'gist', 'detail', 'inference'}.contains(question['type'])) {
error(where, 'type 必须是 gist/detail/inference');
}
_checkOptions(where, question);
}
}
if (materials.isNotEmpty && modes.length < 2) error('materials', '听和读的材料都要有');
final rubric = unit['rubric'];
final rubricRequired = level['openTaskRubric'] == true;
if (rubric == null) {
if (rubricRequired) {
error('rubric', '$levelId 单元须有开放任务的评分标准');
}
return;
}
if (rubric is! Map<String, dynamic>) {
error('rubric', 'rubric 必须是对象');
return;
}
final dimensions = _objects(rubric, 'dimensions', 'rubric');
if (dimensions.length < 2) error('rubric', '至少 2 个评分维度');
for (final (i, dimension) in dimensions.indexed) {
final where = 'rubric.dimensions[$i]';
_string(dimension, 'name', where);
final bands = _strings(dimension, 'bands', where);
if (bands.length < 3) error(where, '至少 3 档描述');
final pass = dimension['pass'];
if (pass is! int || pass <= 0 || pass >= bands.length) {
error(where, 'pass 必须是 bands 内、高于最低档的序号');
}
}
}
}