#!/usr/bin/env python3 from __future__ import annotations import argparse import json import re import subprocess import sys from collections import Counter, defaultdict from dataclasses import asdict, dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any @dataclass class FailureItem: test_name: str target_name: str category: str summary: str failure_text: str test_identifier: str | None = None def run_json_command(command: list[str]) -> dict[str, Any] | list[Any] | None: try: completed = subprocess.run( command, check=True, capture_output=True, text=True, ) except (OSError, subprocess.CalledProcessError): return None stdout = completed.stdout.strip() if not stdout: return None try: return json.loads(stdout) except json.JSONDecodeError: return None def compact_text(text: str) -> str: return re.sub(r"\s+", " ", text).strip() def summarize_failure_text(text: str) -> str: compact = compact_text(text) if not compact: return "No failure message captured" for delimiter in (" - ", ". ", "\n", "…"): if delimiter in compact: head = compact.split(delimiter, 1)[0].strip() if len(head) >= 12: return head[:220] return compact[:220] def classify_failure(text: str, test_name: str) -> str: haystack = f"{test_name} {text}".lower() if any(token in haystack for token in [ "timed out", "timeout", "failed to fulfill", "waitfordemoreaderstate", "waitforexistence", "等待", "超时", ]): return "Timeout" if any(token in haystack for token in [ "no matches found", "failed to find", "does not exist", "existence", "not hittable", "不存在", "未出现", ]): return "Element Missing" if any(token in haystack for token in [ "simulator", "lost connection", "test runner exited", "application state", "quit unexpectedly", "connection interrupted", "xcodebuild", ]): return "Environment" if any(token in haystack for token in [ "page", "pagination", "bookpagemap", "parsems", "readerstate", "location", "页码", "分页", "位置", ]): return "Navigation/State" if any(token in haystack for token in [ "xctassert", "assertion", "xctfail", "failed:", ]): return "Assertion Failure" return "Functional Failure" def normalize_root_cause(text: str) -> str: compact = summarize_failure_text(text).lower() compact = re.sub(r"\b\d+\b", "", compact) compact = re.sub(r"'[^']+'", "''", compact) compact = re.sub(r'"[^"]+"', '""', compact) return compact[:180] def load_summary_from_xcresult(xcresult_path: Path) -> dict[str, Any] | None: return run_json_command([ "xcrun", "xcresulttool", "get", "test-results", "summary", "--path", str(xcresult_path), "--compact", ]) def load_tests_from_xcresult(xcresult_path: Path) -> dict[str, Any] | None: return run_json_command([ "xcrun", "xcresulttool", "get", "test-results", "tests", "--path", str(xcresult_path), "--compact", ]) def flatten_failed_test_nodes(node: dict[str, Any], out: list[dict[str, Any]]) -> None: node_type = node.get("nodeType") result = node.get("result") if node_type == "Test Case" and result == "Failed": out.append(node) for child in node.get("children", []) or []: flatten_failed_test_nodes(child, out) def extract_failures( summary: dict[str, Any] | None, tests_payload: dict[str, Any] | None, raw_log_path: Path | None, ) -> list[FailureItem]: failures: list[FailureItem] = [] raw_failures = None if summary: raw_failures = summary.get("testFailures") if isinstance(raw_failures, dict): raw_failures = [raw_failures] if isinstance(raw_failures, list): for item in raw_failures: if not isinstance(item, dict): continue test_name = item.get("testName") or "Unknown test" target_name = item.get("targetName") or "Unknown target" failure_text = item.get("failureText") or "" failures.append(FailureItem( test_name=test_name, target_name=target_name, category=classify_failure(failure_text, test_name), summary=summarize_failure_text(failure_text), failure_text=compact_text(failure_text), test_identifier=item.get("testIdentifierString") or item.get("testIdentifierURL"), )) if failures: return dedupe_failures(failures) if tests_payload: failed_nodes: list[dict[str, Any]] = [] for node in tests_payload.get("testNodes", []) or []: flatten_failed_test_nodes(node, failed_nodes) for node in failed_nodes: test_name = node.get("name") or "Unknown test" details = node.get("details") or "" failures.append(FailureItem( test_name=test_name, target_name="Unknown target", category=classify_failure(details, test_name), summary=summarize_failure_text(details), failure_text=compact_text(details), test_identifier=node.get("nodeIdentifier") or node.get("nodeIdentifierURL"), )) if failures: return dedupe_failures(failures) if raw_log_path and raw_log_path.exists(): log_text = raw_log_path.read_text(encoding="utf-8", errors="replace") matches = re.findall( r"Test Case '-\[(.*?)\]' failed .*?\n(.*?)(?=\nTest Case '-\[|\Z)", log_text, flags=re.S, ) for test_name, block in matches: failure_text = compact_text(block) failures.append(FailureItem( test_name=test_name, target_name="Unknown target", category=classify_failure(failure_text, test_name), summary=summarize_failure_text(failure_text), failure_text=failure_text, )) return dedupe_failures(failures) def dedupe_failures(failures: list[FailureItem]) -> list[FailureItem]: deduped: dict[tuple[str, str], FailureItem] = {} for failure in failures: key = (failure.test_name, failure.summary) deduped.setdefault(key, failure) return list(deduped.values()) def build_root_causes(failures: list[FailureItem]) -> list[dict[str, Any]]: grouped: dict[str, list[FailureItem]] = defaultdict(list) for failure in failures: grouped[normalize_root_cause(failure.failure_text)].append(failure) ranked = sorted( grouped.items(), key=lambda item: (-len(item[1]), item[0]), ) result = [] for normalized_text, items in ranked[:5]: result.append({ "summary": items[0].summary, "category": items[0].category, "count": len(items), "tests": [failure.test_name for failure in items[:5]], }) return result def build_report_payload( args: argparse.Namespace, summary: dict[str, Any] | None, tests_payload: dict[str, Any] | None, failures: list[FailureItem], ) -> dict[str, Any]: device_summaries = [] if summary: devices_and_configurations = summary.get("devicesAndConfigurations") if isinstance(devices_and_configurations, dict): devices_and_configurations = [devices_and_configurations] if isinstance(devices_and_configurations, list): for item in devices_and_configurations: if not isinstance(item, dict): continue device = item.get("device") or {} config = item.get("testPlanConfiguration") or {} device_summaries.append({ "device_name": device.get("deviceName"), "model_name": device.get("modelName"), "os_version": device.get("osVersion"), "configuration_name": config.get("configurationName"), "passed_tests": item.get("passedTests"), "failed_tests": item.get("failedTests"), }) categories = Counter(failure.category for failure in failures) payload = { "generated_at": datetime.now(timezone.utc).isoformat(), "xcodebuild_exit_code": args.xcodebuild_exit_code, "test_type": args.test_type, "artifacts": { "xcresult": str(Path(args.xcresult).resolve()) if args.xcresult else None, "raw_log": str(Path(args.raw_log).resolve()) if args.raw_log else None, }, "summary": { "title": summary.get("title") if summary else None, "environment_description": summary.get("environmentDescription") if summary else None, "result": summary.get("result") if summary else ("Failed" if failures else "Passed"), "total_test_count": summary.get("totalTestCount") if summary else None, "passed_tests": summary.get("passedTests") if summary else None, "failed_tests": summary.get("failedTests") if summary else len(failures), "skipped_tests": summary.get("skippedTests") if summary else None, "devices": device_summaries, }, "failure_categories": dict(sorted(categories.items(), key=lambda item: (-item[1], item[0]))), "top_root_causes": build_root_causes(failures), "failures": [asdict(failure) for failure in failures], "token_saving_notes": [ "Raw xcodebuild output is intentionally omitted from this report.", "Use the xcresult or raw log paths only when a single failure needs deeper inspection.", "Failures are deduplicated by test name and compact summary.", ], } if tests_payload and not payload["summary"]["total_test_count"]: flat_failed_nodes: list[dict[str, Any]] = [] for node in tests_payload.get("testNodes", []) or []: flatten_failed_test_nodes(node, flat_failed_nodes) payload["summary"]["failed_tests"] = len(flat_failed_nodes) return payload def render_markdown(payload: dict[str, Any]) -> str: summary = payload["summary"] lines = [ "# UI Test Report", "", "## Snapshot", "", f"- Test type: `{payload.get('test_type')}`", f"- Result: `{summary.get('result')}`", f"- xcodebuild exit code: `{payload['xcodebuild_exit_code']}`", f"- Total tests: `{summary.get('total_test_count')}`", f"- Passed: `{summary.get('passed_tests')}`", f"- Failed: `{summary.get('failed_tests')}`", f"- Skipped: `{summary.get('skipped_tests')}`", ] if summary.get("environment_description"): lines.append(f"- Environment: `{summary['environment_description']}`") artifacts = payload["artifacts"] lines.extend([ f"- xcresult: `{artifacts.get('xcresult')}`", f"- raw log: `{artifacts.get('raw_log')}`", "", ]) if payload["failure_categories"]: lines.extend([ "## Failure Categories", "", ]) for category, count in payload["failure_categories"].items(): lines.append(f"- `{category}`: {count}") lines.append("") if payload["top_root_causes"]: lines.extend([ "## Top Root Causes", "", ]) for item in payload["top_root_causes"]: lines.append(f"- `{item['category']}` x{item['count']}: {item['summary']}") lines.append("") lines.extend([ "## Failed Tests", "", ]) if payload["failures"]: for failure in payload["failures"]: lines.append( f"- `{failure['test_name']}` [{failure['category']}] {failure['summary']}" ) else: lines.append("- None") lines.append("") lines.extend([ "## Token-Saving Notes", "", ]) for note in payload["token_saving_notes"]: lines.append(f"- {note}") lines.append("") return "\n".join(lines) def render_text_summary(payload: dict[str, Any]) -> str: summary = payload["summary"] total = summary.get("total_test_count") passed = summary.get("passed_tests") failed = summary.get("failed_tests") skipped = summary.get("skipped_tests") lines = [ "UI test summary", f" type: {payload.get('test_type')}", f" result: {summary.get('result')}", f" tests: total={total} passed={passed} failed={failed} skipped={skipped}", ] if payload["failure_categories"]: category_text = ", ".join( f"{name}={count}" for name, count in payload["failure_categories"].items() ) lines.append(f" categories: {category_text}") if payload["top_root_causes"]: top = payload["top_root_causes"][0] lines.append(f" top root cause: {top['summary']} ({top['count']} tests)") return "\n".join(lines) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Summarize UI test xcresult output into a low-token report.") parser.add_argument("--xcresult", help="Path to .xcresult bundle", default="") parser.add_argument("--raw-log", help="Path to xcodebuild raw log", default="") parser.add_argument("--output-md", required=True, help="Markdown report output path") parser.add_argument("--output-json", required=True, help="JSON report output path") parser.add_argument("--output-txt", required=True, help="Plain-text summary output path") parser.add_argument("--test-type", default="unknown", help="Selected UI test group") parser.add_argument("--xcodebuild-exit-code", type=int, required=True, help="Exit code from xcodebuild") return parser.parse_args() def main() -> int: args = parse_args() xcresult_path = Path(args.xcresult) if args.xcresult else None raw_log_path = Path(args.raw_log) if args.raw_log else None summary = None tests_payload = None if xcresult_path and xcresult_path.exists(): summary = load_summary_from_xcresult(xcresult_path) tests_payload = load_tests_from_xcresult(xcresult_path) failures = extract_failures(summary, tests_payload, raw_log_path) payload = build_report_payload(args, summary, tests_payload, failures) output_md = Path(args.output_md) output_json = Path(args.output_json) output_txt = Path(args.output_txt) output_md.parent.mkdir(parents=True, exist_ok=True) output_json.write_text( json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) output_md.write_text(render_markdown(payload) + "\n", encoding="utf-8") output_txt.write_text(render_text_summary(payload) + "\n", encoding="utf-8") return 0 if __name__ == "__main__": sys.exit(main())