6f4f625c56
- FastAPI backend with build queue, WebSocket logs, task management - Vue 3 frontend with build/config/history views - Xcode project build automation with IPA export - Fix: initialize build_dir before try block to ensure cleanup on early failure
591 lines
20 KiB
Python
591 lines
20 KiB
Python
"""打包服务核心"""
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import shutil
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
from ..config import (
|
||
PROJECT_ROOT,
|
||
AUTOPACKING_DIR,
|
||
BUILD_BASE_DIR,
|
||
COPY_ITEMS,
|
||
DEFAULT_BUILD_DIR_RETENTION_HOURS,
|
||
GIT_REMOTE_URL,
|
||
get_source_dir,
|
||
)
|
||
from .log_streamer import log_streamer
|
||
|
||
|
||
def _cleanup_build_dir(build_dir: Path):
|
||
"""删除整个打包目录"""
|
||
try:
|
||
if build_dir.exists():
|
||
shutil.rmtree(build_dir)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _cleanup_old_builds(db):
|
||
"""清理超过保留时间的打包目录"""
|
||
from ..models import BuildConfig
|
||
config = db.query(BuildConfig).filter(BuildConfig.key == "build_dir_retention_hours").first()
|
||
hours = int(config.value) if config else DEFAULT_BUILD_DIR_RETENTION_HOURS
|
||
cutoff = datetime.utcnow().timestamp() - hours * 3600
|
||
|
||
base = BUILD_BASE_DIR
|
||
if not base.exists():
|
||
return
|
||
for d in base.iterdir():
|
||
if d.is_dir() and d.name.startswith("build_readoor_"):
|
||
try:
|
||
if d.stat().st_mtime < cutoff:
|
||
shutil.rmtree(d)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _db_update(db, task, **fields):
|
||
"""同步更新 task 字段并 commit（在线程中调用）"""
|
||
for k, v in fields.items():
|
||
setattr(task, k, v)
|
||
db.commit()
|
||
|
||
|
||
async def update_source(task_id: str, source_dir: Path, branch: str):
|
||
"""确保分支源码目录存在且为最新"""
|
||
if not source_dir.exists():
|
||
# 首次：从远程 clone
|
||
if not GIT_REMOTE_URL:
|
||
raise Exception(f"源码目录不存在且未配置 GIT_REMOTE_URL: {source_dir}")
|
||
source_dir.parent.mkdir(parents=True, exist_ok=True)
|
||
await log_streamer.emit(task_id, f"克隆仓库: {GIT_REMOTE_URL} → {branch}")
|
||
process = await asyncio.create_subprocess_exec(
|
||
"git", "clone", "-b", branch, "--single-branch", GIT_REMOTE_URL, str(source_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
if process.returncode != 0:
|
||
raise Exception(f"git clone 失败: {branch}")
|
||
else:
|
||
# 已存在：fetch + checkout + pull
|
||
await log_streamer.emit(task_id, f"更新分支源码: {branch}")
|
||
|
||
# fetch
|
||
process = await asyncio.create_subprocess_exec(
|
||
"git", "fetch", "origin",
|
||
cwd=str(source_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
if process.returncode != 0:
|
||
raise Exception("git fetch 失败")
|
||
|
||
# checkout
|
||
process = await asyncio.create_subprocess_exec(
|
||
"git", "checkout", branch,
|
||
cwd=str(source_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
if process.returncode != 0:
|
||
raise Exception(f"git checkout {branch} 失败")
|
||
|
||
# pull
|
||
process = await asyncio.create_subprocess_exec(
|
||
"git", "pull", "origin", branch,
|
||
cwd=str(source_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
if process.returncode != 0:
|
||
raise Exception(f"git pull {branch} 失败")
|
||
|
||
await log_streamer.emit(task_id, f"源码已就绪: {source_dir}")
|
||
|
||
|
||
async def run_build_task(task_id: str):
|
||
"""执行打包任务"""
|
||
from ..database import SessionLocal
|
||
from ..models import Task
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
task = await asyncio.to_thread(
|
||
lambda: db.query(Task).filter(Task.id == task_id).first()
|
||
)
|
||
if not task:
|
||
return
|
||
|
||
# 创建日志队列（必须在 emit 之前）
|
||
log_streamer.create_queue(task_id)
|
||
|
||
# 更新状态
|
||
await asyncio.to_thread(_db_update, db, task,
|
||
status="running", started_at=datetime.utcnow())
|
||
|
||
await log_streamer.emit_step(task_id, "开始打包")
|
||
|
||
# 清理过期的打包目录
|
||
await asyncio.to_thread(_cleanup_old_builds, db)
|
||
|
||
build_dir = None
|
||
try:
|
||
# 1. 更新分支源码并拷贝
|
||
source_dir = get_source_dir(task.branch)
|
||
await asyncio.to_thread(_db_update, db, task, current_step="copy")
|
||
if GIT_REMOTE_URL:
|
||
# 配置了远程仓库，从分支目录获取源码
|
||
await update_source(task_id, source_dir, task.branch)
|
||
elif source_dir.exists():
|
||
# 分支目录已存在，直接使用
|
||
await log_streamer.emit(task_id, f"使用分支源码: {source_dir}")
|
||
else:
|
||
# 未配置远程仓库且分支目录不存在，使用默认源码目录
|
||
source_dir = PROJECT_ROOT
|
||
await log_streamer.emit(task_id, f"未配置 GIT_REMOTE_URL，使用默认源码: {source_dir}")
|
||
build_dir = await copy_source_code(task_id, task, source_dir)
|
||
await asyncio.to_thread(_db_update, db, task, build_dir=str(build_dir))
|
||
|
||
# 2. 生成配置
|
||
await asyncio.to_thread(_db_update, db, task, current_step="config")
|
||
config_data = await generate_config(task_id, task, build_dir)
|
||
await asyncio.to_thread(_db_update, db, task,
|
||
config_json=json.dumps(config_data, ensure_ascii=False))
|
||
|
||
# 3. 替换项目配置
|
||
await asyncio.to_thread(_db_update, db, task, current_step="patch")
|
||
await patch_project(task_id, task, config_data, build_dir)
|
||
|
||
# 4. 代码混淆（可选）
|
||
if task.obfuscation:
|
||
await asyncio.to_thread(_db_update, db, task, current_step="obfuscation")
|
||
await run_obfuscation(task_id, task, build_dir, source_dir)
|
||
|
||
# 5. 构建项目
|
||
await asyncio.to_thread(_db_update, db, task, current_step="build")
|
||
ipa_path = await build_project(task_id, task, build_dir)
|
||
|
||
# 查找 dSYM
|
||
dsym_path = await find_dsym(task_id, build_dir)
|
||
|
||
# 查找混淆映射表
|
||
obf_maps_path = build_dir / "obfuscation_maps"
|
||
|
||
update_fields = {"ipa_path": str(ipa_path)}
|
||
if dsym_path:
|
||
update_fields["dsym_path"] = str(dsym_path)
|
||
if obf_maps_path.exists():
|
||
update_fields["obfuscation_maps_path"] = str(obf_maps_path)
|
||
await asyncio.to_thread(_db_update, db, task, **update_fields)
|
||
|
||
# 6. 上传分发（仅 Ad_Hoc）
|
||
if task.build_type == "Ad_Hoc":
|
||
await asyncio.to_thread(_db_update, db, task, current_step="upload")
|
||
oss_url, qr_code_path = await upload_ipa(task_id, task, config_data, ipa_path)
|
||
upload_fields = {}
|
||
if oss_url:
|
||
upload_fields["oss_url"] = oss_url
|
||
if qr_code_path:
|
||
upload_fields["qr_code_path"] = qr_code_path
|
||
if upload_fields:
|
||
await asyncio.to_thread(_db_update, db, task, **upload_fields)
|
||
|
||
# 完成
|
||
await asyncio.to_thread(_db_update, db, task,
|
||
status="completed", completed_at=datetime.utcnow(),
|
||
current_step=None)
|
||
|
||
await log_streamer.emit_step(task_id, "打包完成")
|
||
await log_streamer.emit(task_id, f"IPA: {ipa_path}")
|
||
if task.oss_url:
|
||
await log_streamer.emit(task_id, f"下载链接: {task.oss_url}")
|
||
|
||
# 清理打包目录（保留产物文件，删除源码拷贝）
|
||
await log_streamer.emit(task_id, "清理临时文件...")
|
||
await asyncio.to_thread(_cleanup_build_dir, build_dir)
|
||
|
||
except asyncio.CancelledError:
|
||
await asyncio.to_thread(_db_update, db, task,
|
||
status="cancelled", completed_at=datetime.utcnow())
|
||
await log_streamer.emit(task_id, "任务已取消")
|
||
if build_dir and build_dir.exists():
|
||
await asyncio.to_thread(_cleanup_build_dir, build_dir)
|
||
|
||
except Exception as e:
|
||
await asyncio.to_thread(_db_update, db, task,
|
||
status="failed", completed_at=datetime.utcnow(),
|
||
error_message=str(e))
|
||
await log_streamer.emit_error(task_id, f"打包失败: {str(e)}")
|
||
if build_dir and build_dir.exists():
|
||
await asyncio.to_thread(_cleanup_build_dir, build_dir)
|
||
|
||
finally:
|
||
log_streamer.complete(task_id)
|
||
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
async def copy_source_code(task_id: str, task, source_dir: Path) -> Path:
|
||
"""从分支源码目录拷贝代码到打包目录"""
|
||
await log_streamer.emit_step(task_id, "拷贝代码")
|
||
|
||
build_dir = BUILD_BASE_DIR / f"build_readoor_{task_id}"
|
||
if build_dir.exists():
|
||
shutil.rmtree(build_dir)
|
||
build_dir.mkdir(parents=True)
|
||
|
||
for item in COPY_ITEMS:
|
||
src = source_dir / item
|
||
dst = build_dir / item
|
||
if src.is_dir():
|
||
await log_streamer.emit(task_id, f"拷贝目录: {item}")
|
||
shutil.copytree(src, dst, ignore=shutil.ignore_patterns(".git", "DerivedData"))
|
||
elif src.is_file():
|
||
await log_streamer.emit(task_id, f"拷贝文件: {item}")
|
||
shutil.copy2(src, dst)
|
||
|
||
await log_streamer.emit(task_id, "代码拷贝完成")
|
||
return build_dir
|
||
|
||
|
||
async def generate_config(task_id: str, task, build_dir: Path) -> dict:
|
||
"""生成打包配置"""
|
||
await log_streamer.emit_step(task_id, "生成配置")
|
||
|
||
from ..routers.config import load_config
|
||
|
||
config = load_config()
|
||
apps = config.get("apps", {})
|
||
schemes = config.get("schemes", {})
|
||
|
||
app = apps.get(task.app_id, {})
|
||
scheme = schemes.get(task.scheme_id, {})
|
||
|
||
# 读取版本号
|
||
start_build_path = AUTOPACKING_DIR / "start_build_app.py"
|
||
app_ver = "2.180.0"
|
||
build_ver = "2.180.0.0"
|
||
|
||
if start_build_path.exists():
|
||
content = start_build_path.read_text()
|
||
for line in content.split("\n"):
|
||
if line.startswith("App_Ver"):
|
||
app_ver = line.split('"')[1] if '"' in line else app_ver
|
||
elif line.startswith("Build_Ver"):
|
||
build_ver = line.split('"')[1] if '"' in line else build_ver
|
||
|
||
# 构建配置
|
||
config_data = {
|
||
"VERSION": app_ver,
|
||
"BUILD_VERSION": build_ver,
|
||
"SERVER": app.get("server", ""),
|
||
"API": app.get("API", ""),
|
||
"APPID_NAME": app.get("name", ""),
|
||
"APPID": app.get("AppGuid", app.get("AppId", "")),
|
||
"SCHEME": scheme.get("name", ""),
|
||
"OSS_FLODER": scheme.get("ossFloder", ""),
|
||
"BUILD_TYPE": task.build_type,
|
||
"ENABLE_OBFUSCATION": task.obfuscation,
|
||
}
|
||
|
||
# 证书配置
|
||
certificates = app.get("certificates", {})
|
||
cert = certificates.get(task.build_type, {})
|
||
|
||
if cert:
|
||
config_data["CERTIFICATE"] = cert.get("cer", "")
|
||
config_data["PROVISIONING_PROFILE"] = cert.get("pro", "")
|
||
config_data["BUNDLE_ID"] = cert.get("name", "")
|
||
config_data["THEME"] = cert.get("theme", "")
|
||
|
||
# 其他配置
|
||
for key in ["weixinlogin", "weixinpay", "tencent", "AssDom", "UniversalLink", "AlivcLicenseKey"]:
|
||
if key in app:
|
||
config_data[key] = app[key]
|
||
|
||
# 写入配置文件
|
||
config_output_path = build_dir / "config_output.json"
|
||
with open(config_output_path, "w", encoding="utf-8") as f:
|
||
json.dump(config_data, f, indent=2, ensure_ascii=False)
|
||
|
||
await log_streamer.emit(task_id, "配置已生成")
|
||
return config_data
|
||
|
||
|
||
async def patch_project(task_id: str, task, config_data: dict, build_dir: Path):
|
||
"""替换项目配置"""
|
||
await log_streamer.emit_step(task_id, "替换项目配置")
|
||
|
||
# 根据 scheme 选择不同的替换逻辑
|
||
if config_data.get("SCHEME") == "readoorDict":
|
||
cmd = [
|
||
"python3",
|
||
str(AUTOPACKING_DIR / "replace_build_info.py"),
|
||
"--dict-config",
|
||
json.dumps(config_data, ensure_ascii=False),
|
||
]
|
||
else:
|
||
cmd = [
|
||
"python3",
|
||
str(AUTOPACKING_DIR / "replace_build_info.py"),
|
||
"--config",
|
||
json.dumps(config_data, ensure_ascii=False),
|
||
]
|
||
|
||
# 使用 subprocess 执行替换
|
||
process = await asyncio.create_subprocess_exec(
|
||
*cmd,
|
||
cwd=str(build_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
|
||
await process.wait()
|
||
|
||
if process.returncode != 0:
|
||
raise Exception("项目配置替换失败")
|
||
|
||
await log_streamer.emit(task_id, "项目配置替换完成")
|
||
|
||
|
||
async def run_obfuscation(task_id: str, task, build_dir: Path, source_root: Path):
|
||
"""执行代码混淆"""
|
||
await log_streamer.emit_step(task_id, "代码混淆")
|
||
|
||
env = os.environ.copy()
|
||
env["TARGET_NAME"] = task.scheme_name
|
||
env["SKIP_OBF_PHASE"] = "1"
|
||
|
||
# 自检
|
||
self_check_script = AUTOPACKING_DIR / "obfuscation" / "scripts" / "obfuscation_self_check.sh"
|
||
process = await asyncio.create_subprocess_exec(
|
||
"bash", str(self_check_script), str(source_root),
|
||
cwd=str(build_dir),
|
||
env=env,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
|
||
await process.wait()
|
||
|
||
if process.returncode != 0:
|
||
raise Exception("混淆自检失败")
|
||
|
||
await log_streamer.emit(task_id, "混淆自检通过")
|
||
|
||
# 执行混淆
|
||
obfuscate_script = AUTOPACKING_DIR / "obfuscation" / "obfuscate_symbols.py"
|
||
process = await asyncio.create_subprocess_exec(
|
||
"python3", str(obfuscate_script), "--archive-mode",
|
||
cwd=str(build_dir),
|
||
env=env,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
|
||
await process.wait()
|
||
|
||
if process.returncode != 0:
|
||
raise Exception("代码混淆失败")
|
||
|
||
await log_streamer.emit(task_id, "代码混淆完成")
|
||
|
||
|
||
async def build_project(task_id: str, task, build_dir: Path) -> Path:
|
||
"""构建项目"""
|
||
await log_streamer.emit_step(task_id, "构建项目")
|
||
|
||
scheme = task.scheme_name
|
||
export_path = build_dir / "build"
|
||
archive_path = export_path / f"{scheme}.xcarchive"
|
||
workspace_path = build_dir / "readoor.xcworkspace"
|
||
|
||
export_path.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 清理
|
||
await log_streamer.emit(task_id, "清理项目...")
|
||
clean_cmd = (
|
||
f"xcodebuild clean -workspace {workspace_path.name} "
|
||
f"-scheme {scheme} "
|
||
f"-configuration Release "
|
||
f"-derivedDataPath {export_path / 'derived_data'}"
|
||
)
|
||
process = await asyncio.create_subprocess_shell(
|
||
clean_cmd,
|
||
cwd=str(build_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
|
||
# Archive
|
||
await log_streamer.emit(task_id, "开始 Archive...")
|
||
archive_cmd = (
|
||
f"xcodebuild archive -workspace {workspace_path.name} "
|
||
f"-scheme {scheme} "
|
||
f"-configuration Release "
|
||
f"-archivePath {archive_path} "
|
||
f"-derivedDataPath {export_path / 'derived_data'} "
|
||
f"-destination generic/platform=ios -quiet"
|
||
)
|
||
process = await asyncio.create_subprocess_shell(
|
||
archive_cmd,
|
||
cwd=str(build_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
|
||
if process.returncode != 0:
|
||
raise Exception("Archive 失败")
|
||
|
||
# 导出 IPA
|
||
await log_streamer.emit(task_id, "导出 IPA...")
|
||
export_plist = build_dir / "exportOptions.plist"
|
||
export_cmd = (
|
||
f"xcodebuild -exportArchive "
|
||
f"-archivePath {archive_path} "
|
||
f"-exportPath {export_path} "
|
||
f"-exportOptionsPlist {export_plist}"
|
||
)
|
||
process = await asyncio.create_subprocess_shell(
|
||
export_cmd,
|
||
cwd=str(build_dir),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
await process.wait()
|
||
|
||
if process.returncode != 0:
|
||
raise Exception("导出 IPA 失败")
|
||
|
||
# 查找 IPA 文件
|
||
ipa_files = list(export_path.glob("*.ipa"))
|
||
if not ipa_files:
|
||
raise Exception("未找到 IPA 文件")
|
||
|
||
ipa_path = ipa_files[0]
|
||
await log_streamer.emit(task_id, f"IPA 已导出: {ipa_path.name}")
|
||
return ipa_path
|
||
|
||
|
||
async def find_dsym(task_id: str, build_dir: Path) -> Path:
|
||
"""查找 dSYM 文件"""
|
||
derived_data = build_dir / "build" / "derived_data"
|
||
if not derived_data.exists():
|
||
return None
|
||
|
||
for dsym in derived_data.rglob("*.dSYM"):
|
||
return dsym
|
||
return None
|
||
|
||
|
||
async def upload_ipa(task_id: str, task, config_data: dict, ipa_path: Path) -> tuple:
|
||
"""上传 IPA，返回 (download_url, qr_code_path)"""
|
||
await log_streamer.emit_step(task_id, "上传分发平台")
|
||
|
||
qr_code_path = None
|
||
app_guid = config_data.get("APPID", "")
|
||
|
||
# 从 config.json 读取上传配置并注入
|
||
from ..routers.config import load_config
|
||
full_config = load_config()
|
||
config_data["_upload_config"] = full_config.get("upload", {})
|
||
|
||
# 使用现有的 upload_iap 脚本
|
||
upload_script = AUTOPACKING_DIR / "upload_iap.py"
|
||
if not upload_script.exists():
|
||
await log_streamer.emit_warning(task_id, "上传脚本不存在，跳过上传")
|
||
return None, None
|
||
|
||
cmd = [
|
||
"python3",
|
||
str(upload_script),
|
||
"--config",
|
||
json.dumps(config_data, ensure_ascii=False),
|
||
]
|
||
|
||
process = await asyncio.create_subprocess_exec(
|
||
*cmd,
|
||
cwd=str(PROJECT_ROOT),
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.STDOUT,
|
||
)
|
||
|
||
oss_url = None
|
||
async for line in process.stdout:
|
||
decoded = line.decode("utf-8", errors="replace").strip()
|
||
if decoded:
|
||
await log_streamer.emit(task_id, decoded)
|
||
# 解析上传结果
|
||
if decoded.startswith("UPLOAD_RESULT:"):
|
||
try:
|
||
result = json.loads(decoded[len("UPLOAD_RESULT:"):])
|
||
oss_url = result.get("download_url")
|
||
qr_code_path = result.get("qr_code_path")
|
||
except json.JSONDecodeError:
|
||
pass
|
||
|
||
await process.wait()
|
||
|
||
# 兜底：查找二维码文件
|
||
if not qr_code_path and app_guid:
|
||
expected_qr = PROJECT_ROOT / "build" / f"{app_guid}.png"
|
||
if expected_qr.exists():
|
||
qr_code_path = str(expected_qr)
|
||
|
||
if process.returncode != 0:
|
||
await log_streamer.emit_warning(task_id, "上传可能失败")
|
||
|
||
return oss_url, qr_code_path
|