0e7c0577e3
- Add RDEPUBReaderSearchBarView with animated show/hide, keyword navigation, and match counting integrated into the reader controller - Restructure docs: replace scattered design docs with consolidated BUSINESS_LOGIC.md and UML_CLASS_DIAGRAMS.md; update ARCHITECTURE.md - Add SearchTests and FanrenParseTimeTest; enhance LargeBookOnDemandTests - Add scripts/run_ui_regression.sh and summarize_ui_results.py for automated UI test execution and reporting Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
466 lines
15 KiB
Python
Executable File
466 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import asdict, dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
@dataclass
|
|
class FailureItem:
|
|
test_name: str
|
|
target_name: str
|
|
category: str
|
|
summary: str
|
|
failure_text: str
|
|
test_identifier: str | None = None
|
|
|
|
|
|
def run_json_command(command: list[str]) -> dict[str, Any] | list[Any] | None:
|
|
try:
|
|
completed = subprocess.run(
|
|
command,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return None
|
|
|
|
stdout = completed.stdout.strip()
|
|
if not stdout:
|
|
return None
|
|
try:
|
|
return json.loads(stdout)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
|
|
|
|
def compact_text(text: str) -> str:
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def summarize_failure_text(text: str) -> str:
|
|
compact = compact_text(text)
|
|
if not compact:
|
|
return "No failure message captured"
|
|
|
|
for delimiter in (" - ", ". ", "\n", "…"):
|
|
if delimiter in compact:
|
|
head = compact.split(delimiter, 1)[0].strip()
|
|
if len(head) >= 12:
|
|
return head[:220]
|
|
return compact[:220]
|
|
|
|
|
|
def classify_failure(text: str, test_name: str) -> str:
|
|
haystack = f"{test_name} {text}".lower()
|
|
|
|
if any(token in haystack for token in [
|
|
"timed out",
|
|
"timeout",
|
|
"failed to fulfill",
|
|
"waitfordemoreaderstate",
|
|
"waitforexistence",
|
|
"等待",
|
|
"超时",
|
|
]):
|
|
return "Timeout"
|
|
|
|
if any(token in haystack for token in [
|
|
"no matches found",
|
|
"failed to find",
|
|
"does not exist",
|
|
"existence",
|
|
"not hittable",
|
|
"不存在",
|
|
"未出现",
|
|
]):
|
|
return "Element Missing"
|
|
|
|
if any(token in haystack for token in [
|
|
"simulator",
|
|
"lost connection",
|
|
"test runner exited",
|
|
"application state",
|
|
"quit unexpectedly",
|
|
"connection interrupted",
|
|
"xcodebuild",
|
|
]):
|
|
return "Environment"
|
|
|
|
if any(token in haystack for token in [
|
|
"page",
|
|
"pagination",
|
|
"bookpagemap",
|
|
"parsems",
|
|
"readerstate",
|
|
"location",
|
|
"页码",
|
|
"分页",
|
|
"位置",
|
|
]):
|
|
return "Navigation/State"
|
|
|
|
if any(token in haystack for token in [
|
|
"xctassert",
|
|
"assertion",
|
|
"xctfail",
|
|
"failed:",
|
|
]):
|
|
return "Assertion Failure"
|
|
|
|
return "Functional Failure"
|
|
|
|
|
|
def normalize_root_cause(text: str) -> str:
|
|
compact = summarize_failure_text(text).lower()
|
|
compact = re.sub(r"\b\d+\b", "<n>", compact)
|
|
compact = re.sub(r"'[^']+'", "'<value>'", compact)
|
|
compact = re.sub(r'"[^"]+"', '"<value>"', compact)
|
|
return compact[:180]
|
|
|
|
|
|
def load_summary_from_xcresult(xcresult_path: Path) -> dict[str, Any] | None:
|
|
return run_json_command([
|
|
"xcrun",
|
|
"xcresulttool",
|
|
"get",
|
|
"test-results",
|
|
"summary",
|
|
"--path",
|
|
str(xcresult_path),
|
|
"--compact",
|
|
])
|
|
|
|
|
|
def load_tests_from_xcresult(xcresult_path: Path) -> dict[str, Any] | None:
|
|
return run_json_command([
|
|
"xcrun",
|
|
"xcresulttool",
|
|
"get",
|
|
"test-results",
|
|
"tests",
|
|
"--path",
|
|
str(xcresult_path),
|
|
"--compact",
|
|
])
|
|
|
|
|
|
def flatten_failed_test_nodes(node: dict[str, Any], out: list[dict[str, Any]]) -> None:
|
|
node_type = node.get("nodeType")
|
|
result = node.get("result")
|
|
if node_type == "Test Case" and result == "Failed":
|
|
out.append(node)
|
|
for child in node.get("children", []) or []:
|
|
flatten_failed_test_nodes(child, out)
|
|
|
|
|
|
def extract_failures(
|
|
summary: dict[str, Any] | None,
|
|
tests_payload: dict[str, Any] | None,
|
|
raw_log_path: Path | None,
|
|
) -> list[FailureItem]:
|
|
failures: list[FailureItem] = []
|
|
|
|
raw_failures = None
|
|
if summary:
|
|
raw_failures = summary.get("testFailures")
|
|
|
|
if isinstance(raw_failures, dict):
|
|
raw_failures = [raw_failures]
|
|
|
|
if isinstance(raw_failures, list):
|
|
for item in raw_failures:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
test_name = item.get("testName") or "Unknown test"
|
|
target_name = item.get("targetName") or "Unknown target"
|
|
failure_text = item.get("failureText") or ""
|
|
failures.append(FailureItem(
|
|
test_name=test_name,
|
|
target_name=target_name,
|
|
category=classify_failure(failure_text, test_name),
|
|
summary=summarize_failure_text(failure_text),
|
|
failure_text=compact_text(failure_text),
|
|
test_identifier=item.get("testIdentifierString") or item.get("testIdentifierURL"),
|
|
))
|
|
|
|
if failures:
|
|
return dedupe_failures(failures)
|
|
|
|
if tests_payload:
|
|
failed_nodes: list[dict[str, Any]] = []
|
|
for node in tests_payload.get("testNodes", []) or []:
|
|
flatten_failed_test_nodes(node, failed_nodes)
|
|
for node in failed_nodes:
|
|
test_name = node.get("name") or "Unknown test"
|
|
details = node.get("details") or ""
|
|
failures.append(FailureItem(
|
|
test_name=test_name,
|
|
target_name="Unknown target",
|
|
category=classify_failure(details, test_name),
|
|
summary=summarize_failure_text(details),
|
|
failure_text=compact_text(details),
|
|
test_identifier=node.get("nodeIdentifier") or node.get("nodeIdentifierURL"),
|
|
))
|
|
|
|
if failures:
|
|
return dedupe_failures(failures)
|
|
|
|
if raw_log_path and raw_log_path.exists():
|
|
log_text = raw_log_path.read_text(encoding="utf-8", errors="replace")
|
|
matches = re.findall(
|
|
r"Test Case '-\[(.*?)\]' failed .*?\n(.*?)(?=\nTest Case '-\[|\Z)",
|
|
log_text,
|
|
flags=re.S,
|
|
)
|
|
for test_name, block in matches:
|
|
failure_text = compact_text(block)
|
|
failures.append(FailureItem(
|
|
test_name=test_name,
|
|
target_name="Unknown target",
|
|
category=classify_failure(failure_text, test_name),
|
|
summary=summarize_failure_text(failure_text),
|
|
failure_text=failure_text,
|
|
))
|
|
|
|
return dedupe_failures(failures)
|
|
|
|
|
|
def dedupe_failures(failures: list[FailureItem]) -> list[FailureItem]:
|
|
deduped: dict[tuple[str, str], FailureItem] = {}
|
|
for failure in failures:
|
|
key = (failure.test_name, failure.summary)
|
|
deduped.setdefault(key, failure)
|
|
return list(deduped.values())
|
|
|
|
|
|
def build_root_causes(failures: list[FailureItem]) -> list[dict[str, Any]]:
|
|
grouped: dict[str, list[FailureItem]] = defaultdict(list)
|
|
for failure in failures:
|
|
grouped[normalize_root_cause(failure.failure_text)].append(failure)
|
|
|
|
ranked = sorted(
|
|
grouped.items(),
|
|
key=lambda item: (-len(item[1]), item[0]),
|
|
)
|
|
result = []
|
|
for normalized_text, items in ranked[:5]:
|
|
result.append({
|
|
"summary": items[0].summary,
|
|
"category": items[0].category,
|
|
"count": len(items),
|
|
"tests": [failure.test_name for failure in items[:5]],
|
|
})
|
|
return result
|
|
|
|
|
|
def build_report_payload(
|
|
args: argparse.Namespace,
|
|
summary: dict[str, Any] | None,
|
|
tests_payload: dict[str, Any] | None,
|
|
failures: list[FailureItem],
|
|
) -> dict[str, Any]:
|
|
device_summaries = []
|
|
if summary:
|
|
devices_and_configurations = summary.get("devicesAndConfigurations")
|
|
if isinstance(devices_and_configurations, dict):
|
|
devices_and_configurations = [devices_and_configurations]
|
|
if isinstance(devices_and_configurations, list):
|
|
for item in devices_and_configurations:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
device = item.get("device") or {}
|
|
config = item.get("testPlanConfiguration") or {}
|
|
device_summaries.append({
|
|
"device_name": device.get("deviceName"),
|
|
"model_name": device.get("modelName"),
|
|
"os_version": device.get("osVersion"),
|
|
"configuration_name": config.get("configurationName"),
|
|
"passed_tests": item.get("passedTests"),
|
|
"failed_tests": item.get("failedTests"),
|
|
})
|
|
|
|
categories = Counter(failure.category for failure in failures)
|
|
|
|
payload = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"xcodebuild_exit_code": args.xcodebuild_exit_code,
|
|
"test_type": args.test_type,
|
|
"artifacts": {
|
|
"xcresult": str(Path(args.xcresult).resolve()) if args.xcresult else None,
|
|
"raw_log": str(Path(args.raw_log).resolve()) if args.raw_log else None,
|
|
},
|
|
"summary": {
|
|
"title": summary.get("title") if summary else None,
|
|
"environment_description": summary.get("environmentDescription") if summary else None,
|
|
"result": summary.get("result") if summary else ("Failed" if failures else "Passed"),
|
|
"total_test_count": summary.get("totalTestCount") if summary else None,
|
|
"passed_tests": summary.get("passedTests") if summary else None,
|
|
"failed_tests": summary.get("failedTests") if summary else len(failures),
|
|
"skipped_tests": summary.get("skippedTests") if summary else None,
|
|
"devices": device_summaries,
|
|
},
|
|
"failure_categories": dict(sorted(categories.items(), key=lambda item: (-item[1], item[0]))),
|
|
"top_root_causes": build_root_causes(failures),
|
|
"failures": [asdict(failure) for failure in failures],
|
|
"token_saving_notes": [
|
|
"Raw xcodebuild output is intentionally omitted from this report.",
|
|
"Use the xcresult or raw log paths only when a single failure needs deeper inspection.",
|
|
"Failures are deduplicated by test name and compact summary.",
|
|
],
|
|
}
|
|
|
|
if tests_payload and not payload["summary"]["total_test_count"]:
|
|
flat_failed_nodes: list[dict[str, Any]] = []
|
|
for node in tests_payload.get("testNodes", []) or []:
|
|
flatten_failed_test_nodes(node, flat_failed_nodes)
|
|
payload["summary"]["failed_tests"] = len(flat_failed_nodes)
|
|
|
|
return payload
|
|
|
|
|
|
def render_markdown(payload: dict[str, Any]) -> str:
|
|
summary = payload["summary"]
|
|
lines = [
|
|
"# UI Test Report",
|
|
"",
|
|
"## Snapshot",
|
|
"",
|
|
f"- Test type: `{payload.get('test_type')}`",
|
|
f"- Result: `{summary.get('result')}`",
|
|
f"- xcodebuild exit code: `{payload['xcodebuild_exit_code']}`",
|
|
f"- Total tests: `{summary.get('total_test_count')}`",
|
|
f"- Passed: `{summary.get('passed_tests')}`",
|
|
f"- Failed: `{summary.get('failed_tests')}`",
|
|
f"- Skipped: `{summary.get('skipped_tests')}`",
|
|
]
|
|
|
|
if summary.get("environment_description"):
|
|
lines.append(f"- Environment: `{summary['environment_description']}`")
|
|
|
|
artifacts = payload["artifacts"]
|
|
lines.extend([
|
|
f"- xcresult: `{artifacts.get('xcresult')}`",
|
|
f"- raw log: `{artifacts.get('raw_log')}`",
|
|
"",
|
|
])
|
|
|
|
if payload["failure_categories"]:
|
|
lines.extend([
|
|
"## Failure Categories",
|
|
"",
|
|
])
|
|
for category, count in payload["failure_categories"].items():
|
|
lines.append(f"- `{category}`: {count}")
|
|
lines.append("")
|
|
|
|
if payload["top_root_causes"]:
|
|
lines.extend([
|
|
"## Top Root Causes",
|
|
"",
|
|
])
|
|
for item in payload["top_root_causes"]:
|
|
lines.append(f"- `{item['category']}` x{item['count']}: {item['summary']}")
|
|
lines.append("")
|
|
|
|
lines.extend([
|
|
"## Failed Tests",
|
|
"",
|
|
])
|
|
if payload["failures"]:
|
|
for failure in payload["failures"]:
|
|
lines.append(
|
|
f"- `{failure['test_name']}` [{failure['category']}] {failure['summary']}"
|
|
)
|
|
else:
|
|
lines.append("- None")
|
|
lines.append("")
|
|
|
|
lines.extend([
|
|
"## Token-Saving Notes",
|
|
"",
|
|
])
|
|
for note in payload["token_saving_notes"]:
|
|
lines.append(f"- {note}")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def render_text_summary(payload: dict[str, Any]) -> str:
|
|
summary = payload["summary"]
|
|
total = summary.get("total_test_count")
|
|
passed = summary.get("passed_tests")
|
|
failed = summary.get("failed_tests")
|
|
skipped = summary.get("skipped_tests")
|
|
lines = [
|
|
"UI test summary",
|
|
f" type: {payload.get('test_type')}",
|
|
f" result: {summary.get('result')}",
|
|
f" tests: total={total} passed={passed} failed={failed} skipped={skipped}",
|
|
]
|
|
if payload["failure_categories"]:
|
|
category_text = ", ".join(
|
|
f"{name}={count}" for name, count in payload["failure_categories"].items()
|
|
)
|
|
lines.append(f" categories: {category_text}")
|
|
if payload["top_root_causes"]:
|
|
top = payload["top_root_causes"][0]
|
|
lines.append(f" top root cause: {top['summary']} ({top['count']} tests)")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Summarize UI test xcresult output into a low-token report.")
|
|
parser.add_argument("--xcresult", help="Path to .xcresult bundle", default="")
|
|
parser.add_argument("--raw-log", help="Path to xcodebuild raw log", default="")
|
|
parser.add_argument("--output-md", required=True, help="Markdown report output path")
|
|
parser.add_argument("--output-json", required=True, help="JSON report output path")
|
|
parser.add_argument("--output-txt", required=True, help="Plain-text summary output path")
|
|
parser.add_argument("--test-type", default="unknown", help="Selected UI test group")
|
|
parser.add_argument("--xcodebuild-exit-code", type=int, required=True, help="Exit code from xcodebuild")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
xcresult_path = Path(args.xcresult) if args.xcresult else None
|
|
raw_log_path = Path(args.raw_log) if args.raw_log else None
|
|
|
|
summary = None
|
|
tests_payload = None
|
|
if xcresult_path and xcresult_path.exists():
|
|
summary = load_summary_from_xcresult(xcresult_path)
|
|
tests_payload = load_tests_from_xcresult(xcresult_path)
|
|
|
|
failures = extract_failures(summary, tests_payload, raw_log_path)
|
|
payload = build_report_payload(args, summary, tests_payload, failures)
|
|
|
|
output_md = Path(args.output_md)
|
|
output_json = Path(args.output_json)
|
|
output_txt = Path(args.output_txt)
|
|
output_md.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
output_json.write_text(
|
|
json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
output_md.write_text(render_markdown(payload) + "\n", encoding="utf-8")
|
|
output_txt.write_text(render_text_summary(payload) + "\n", encoding="utf-8")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|