feat: 支持多项目打包

- config.json 改为按项目存储(projects.<id>),上传配置全局共用;旧单项目配置自动迁移为 readoor
- 新增项目管理(管理员):Git 仓库/凭据、源码目录、Xcode 工程、拷贝条目、配置替换方式、自定义字段、替换规则
- 通用配置替换:pbxproj 版本号与签名,patch_rules 支持 define / replace_line 及 ${app.X}、${server.X} 模板
- 请求通过 X-Project 头选择项目;打包任务记录 project_id,历史按项目过滤
- 前端顶部切换项目,App/服务器环境表单按项目显示字段

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shenlei
2026-10-22 14:59:30 +09:00
co-authored by Claude Opus 5.5
parent 2bd5c7057f
commit 8f2f23682f
19 changed files with 1535 additions and 137 deletions
+120 -37
View File
@@ -3,6 +3,7 @@ import asyncio
import json
import os
import plistlib
import re
import shutil
import subprocess
import zipfile
@@ -12,16 +13,14 @@ from typing import List
from ..config import (
BUILD_BASE_DIR,
COPY_ITEMS,
DEFAULT_BUILD_DIR_RETENTION_HOURS,
BUILD_TIMEOUT_HOURS,
SKINS_DIR,
AUTOMATION_DIR,
get_source_dir,
get_shared_source_dir,
get_git_remote_url,
mask_git_remote_url,
)
from ..projects import LEGACY_PROJECT_ID, get_definition, get_project_source
from .log_streamer import log_streamer
from .distribution import DistributionError, publish_dsym, publish_ipa
from .notification import NotificationError, send_dingtalk_notification
@@ -37,9 +36,36 @@ class BuildError(Exception):
self.detail = detail
# 共享源码目录只能同时被一个任务切换、清理和复制。
# 后续在各自 build_dir 中执行的配置、依赖安装和编译不受此锁限制。
source_prepare_lock = asyncio.Lock()
# 共享源码目录只能同时被一个任务切换、清理和复制,每个源码目录一把锁,
# 不同项目的源码准备互不阻塞。后续在各自 build_dir 中执行的配置、依赖安装和编译不受此锁限制。
_source_prepare_locks: dict = {}
# 打包目录名:build_<项目 ID>_<任务 ID>
_BUILD_DIR_PATTERN = re.compile(r"^build_[a-z][a-z0-9_-]{0,31}_.+")
def _source_prepare_lock(source_dir: Path) -> asyncio.Lock:
return _source_prepare_locks.setdefault(str(source_dir.resolve()), asyncio.Lock())
def _task_project_id(task) -> str:
project_id = getattr(task, "project_id", None)
return project_id if isinstance(project_id, str) and project_id else LEGACY_PROJECT_ID
def load_task_project(task) -> dict:
"""读取任务所属项目的配置视图(见 routers.config.load_config)。"""
from fastapi import HTTPException
from ..routers.config import load_config
try:
return load_config(_task_project_id(task))
except HTTPException as exc:
raise BuildError(str(exc.detail), category="config") from exc
def get_task_definition(task) -> dict:
project_id = _task_project_id(task)
return get_definition(project_id, load_task_project(task))
def _patch_afnetworking_private_headers(build_dir: Path) -> int:
@@ -217,7 +243,7 @@ def _cleanup_old_builds(db):
_cleanup_superseded_tasks(db)
return
for d in base.iterdir():
if d.is_dir() and d.name.startswith("build_readoor_"):
if d.is_dir() and _BUILD_DIR_PATTERN.match(d.name):
try:
if d.stat().st_mtime < cutoff:
shutil.rmtree(d)
@@ -291,16 +317,21 @@ def _db_update(db, task, **fields):
db.commit()
async def update_source(task_id: str, source_dir: Path, branch: str, ref_type: str = "branch"):
"""将共享源码工作目录强制同步到指定远程分支或 Tag(ref_type="tag" 时 branch 为 Tag 名)。"""
remote_url = get_git_remote_url(with_credentials=True)
async def update_source(task_id: str, source_dir: Path, branch: str, ref_type: str = "branch",
remote_url: str = None):
"""将共享源码工作目录强制同步到指定远程分支或 Tag(ref_type="tag" 时 branch 为 Tag 名)。
remote_url 为带凭证的项目仓库地址,未传时使用 .env 中的 GIT_REMOTE_URL。
"""
if remote_url is None:
remote_url = get_git_remote_url(with_credentials=True)
remote_url_masked = mask_git_remote_url(remote_url)
git_env = _git_env()
cloned = not source_dir.exists()
if cloned:
if not remote_url:
raise Exception(f"源码目录不存在且未配置 GIT_REMOTE_URL: {source_dir}")
raise Exception(f"源码目录不存在且未配置 Git 仓库地址: {source_dir}")
source_dir.parent.mkdir(parents=True, exist_ok=True)
await log_streamer.emit(task_id, f"初始化共享源码目录: {remote_url_masked}")
process = await asyncio.create_subprocess_exec(
@@ -381,25 +412,34 @@ async def get_source_commit(source_dir: Path) -> str:
async def prepare_source_snapshot(task_id: str, task) -> tuple[Path, str]:
"""准备任务独立源码快照;远程仓库模式下此过程全局串行。"""
if not get_git_remote_url():
source_dir = get_source_dir(task.branch)
"""准备任务独立源码快照;远程仓库模式下同一源码目录的准备过程串行。"""
project_id = _task_project_id(task)
project = load_task_project(task)
definition = get_definition(project_id, project)
source = get_project_source(project_id, project)
await log_streamer.emit(task_id, f"项目: {definition['name']}({project_id})")
if not source.remote_url:
if source.manual_base is None:
raise BuildError(f"项目 {definition['name']} 未配置 Git 仓库地址", category="config")
source_dir = source.manual_base / task.branch
if not source_dir.exists():
raise Exception(
f"分支源码目录不存在: {source_dir},请先在 GIT_SOURCE_BASE 下准备分支(或 Tag)代码,或配置 GIT_REMOTE_URL"
)
await log_streamer.emit(task_id, f"使用分支源码: {source_dir}")
return await copy_source_code(task_id, task, source_dir), ""
return await copy_source_code(task_id, task, source_dir, definition), ""
source_dir = get_shared_source_dir()
source_dir = source.shared_dir
await log_streamer.emit(task_id, "等待共享源码准备队列...")
async with source_prepare_lock:
async with _source_prepare_lock(source_dir):
await log_streamer.emit(task_id, "开始准备共享源码快照")
ref_type = "tag" if task.ref_type == "tag" else "branch"
await update_source(task_id, source_dir, task.branch, ref_type)
await update_source(task_id, source_dir, task.branch, ref_type,
remote_url=source.remote_url_with_credentials)
commit = await get_source_commit(source_dir)
await log_streamer.emit(task_id, f"源码版本: {task.branch} @ {commit}")
build_dir = await copy_source_code(task_id, task, source_dir)
build_dir = await copy_source_code(task_id, task, source_dir, definition)
await log_streamer.emit(task_id, "共享源码快照已就绪,开始并行打包")
return build_dir, commit
@@ -438,6 +478,7 @@ async def run_build_task(task_id: str):
nonlocal build_dir
# 1. 串行准备源码快照;后续步骤在任务独立目录中并行执行。
await asyncio.to_thread(_db_update, db, task, current_step="source_prepare")
definition = await asyncio.to_thread(get_task_definition, task)
build_dir, source_commit = await prepare_source_snapshot(task_id, task)
await asyncio.to_thread(_db_update, db, task, build_dir=str(build_dir))
@@ -452,7 +493,7 @@ async def run_build_task(task_id: str):
# 3. 替换项目配置
await asyncio.to_thread(_db_update, db, task, current_step="patch")
await patch_project(task_id, task, config_data, build_dir)
await patch_project(task_id, task, config_data, build_dir, definition)
# 3.5 安装依赖
await asyncio.to_thread(_db_update, db, task, current_step="pod_install")
@@ -465,7 +506,7 @@ async def run_build_task(task_id: str):
# 5. 构建项目
await asyncio.to_thread(_db_update, db, task, current_step="build")
ipa_path = await build_project(task_id, task, config_data, build_dir)
ipa_path = await build_project(task_id, task, config_data, build_dir, definition["xcode"])
# 查找 dSYM
dsym_path = await find_dsym(task_id, build_dir, config_data.get("SCHEME", ""))
@@ -590,16 +631,36 @@ async def run_build_task(task_id: str):
db.close()
async def copy_source_code(task_id: str, task, source_dir: Path) -> Path:
async def copy_source_code(task_id: str, task, source_dir: Path, definition: dict = None) -> Path:
"""从分支源码目录拷贝代码到打包目录"""
await log_streamer.emit_step(task_id, "拷贝代码")
build_dir = BUILD_BASE_DIR / f"build_readoor_{task_id}"
project_id = _task_project_id(task)
if definition is None:
definition = get_definition(project_id, {})
build_dir = BUILD_BASE_DIR / f"build_{project_id}_{task_id}"
if build_dir.exists():
shutil.rmtree(build_dir)
build_dir.mkdir(parents=True)
for item in COPY_ITEMS:
# 未指定拷贝条目时拷贝整个仓库(不含 Git 数据和编译缓存)
copy_items = definition.get("copy_items") or []
if not copy_items:
await log_streamer.emit(task_id, "拷贝整个仓库")
root = str(source_dir)
def _ignore(directory, names):
ignored = {name for name in names if name in (".git", "DerivedData")}
# 仓库根目录的 build 与打包产物目录同名,不拷贝
if directory == root and "build" in names:
ignored.add("build")
return ignored
shutil.copytree(source_dir, build_dir, symlinks=True, ignore=_ignore)
await log_streamer.emit(task_id, "代码拷贝完成")
return build_dir
build_dir.mkdir(parents=True)
for item in copy_items:
src = source_dir / item
# 历史分支中该文件可能仍是小写 podfile;构建副本统一使用 Pods.xcodeproj
# 引用的标准名称 Podfile,避免大小写敏感文件系统报路径不一致。
@@ -626,7 +687,8 @@ async def generate_config(task_id: str, task, build_dir: Path) -> dict:
is_release_track, next_build_no, _ensure_track,
)
config = load_config()
project_id = _task_project_id(task)
config = load_config(project_id)
apps = config.get("apps", {})
schemes = config.get("schemes", {})
@@ -652,7 +714,7 @@ async def generate_config(task_id: str, task, build_dir: Path) -> dict:
# feature 轨始终使用手动设定的构建号。
if is_release and task.build_type == "App_Store":
async with _config_lock:
config = load_config()
config = load_config(project_id)
track = _ensure_track(config, track_name)
track.setdefault("app_ver", app_ver)
n = next_build_no(config["versions"]["tracks"], app_ver)
@@ -666,6 +728,7 @@ async def generate_config(task_id: str, task, build_dir: Path) -> dict:
# 构建配置
config_data = {
"PROJECT_ID": project_id,
"VERSION": app_ver,
"BUILD_VERSION": build_ver,
"SERVER": app.get("server", ""),
@@ -739,6 +802,16 @@ async def generate_config(task_id: str, task, build_dir: Path) -> dict:
if key in app:
config_data[key] = app[key]
# 项目自定义字段,供通用替换规则以 ${app.X} / ${server.X} 引用
definition = get_definition(project_id, config)
server = config.get("servers", {}).get(app.get("server", ""), {})
config_data["APP_VARS"] = {
field["key"]: app.get(field["key"], "") for field in definition["app_fields"] if field.get("key")
}
config_data["SERVER_VARS"] = {
field["key"]: server.get(field["key"], "") for field in definition["server_fields"] if field.get("key")
}
# 写入配置文件
config_output_path = build_dir / "config_output.json"
with open(config_output_path, "w", encoding="utf-8") as f:
@@ -748,12 +821,12 @@ async def generate_config(task_id: str, task, build_dir: Path) -> dict:
return config_data
async def patch_project(task_id: str, task, config_data: dict, build_dir: Path):
async def patch_project(task_id: str, task, config_data: dict, build_dir: Path, definition: dict = None):
"""替换项目配置"""
await log_streamer.emit_step(task_id, "替换项目配置")
try:
patch_lines = await asyncio.to_thread(apply_project_config, build_dir, config_data)
patch_lines = await asyncio.to_thread(apply_project_config, build_dir, config_data, definition)
except ProjectPatchError as exc:
raise BuildError(str(exc), category="config") from exc
(build_dir / "patch_result.log").write_text("\n".join(patch_lines), encoding="utf-8")
@@ -766,6 +839,10 @@ async def run_pod_install(task_id: str, build_dir: Path):
"""安装 CocoaPods 依赖"""
await log_streamer.emit_step(task_id, "安装依赖")
if not (build_dir / "Podfile").is_file() and not (build_dir / "podfile").is_file():
await log_streamer.emit(task_id, "项目未使用 CocoaPods,跳过")
return
process = await asyncio.create_subprocess_exec(
"pod", "install",
cwd=str(build_dir),
@@ -845,14 +922,21 @@ async def run_obfuscation(task_id: str, task, config_data: dict, build_dir: Path
await log_streamer.emit(task_id, "代码混淆完成")
async def build_project(task_id: str, task, config_data: dict, build_dir: Path) -> Path:
"""构建项目"""
async def build_project(task_id: str, task, config_data: dict, build_dir: Path, xcode: dict = None) -> Path:
"""构建项目;xcode 为项目定义中的 {workspace, project},优先使用 workspace"""
await log_streamer.emit_step(task_id, "构建项目")
scheme = config_data.get("SCHEME", "")
export_path = build_dir / "build"
archive_path = export_path / f"{scheme}.xcarchive"
workspace_path = build_dir / "readoor.xcworkspace"
if xcode is None:
xcode = get_definition(LEGACY_PROJECT_ID, {})["xcode"]
if xcode.get("workspace"):
container_args = ("-workspace", xcode["workspace"])
elif xcode.get("project"):
container_args = ("-project", xcode["project"])
else:
raise BuildError("项目未配置 Xcode workspace 或 project", category="config")
export_path.mkdir(parents=True, exist_ok=True)
@@ -860,7 +944,7 @@ async def build_project(task_id: str, task, config_data: dict, build_dir: Path)
await log_streamer.emit(task_id, "清理项目...")
process = await asyncio.create_subprocess_exec(
"xcodebuild", "clean",
"-workspace", workspace_path.name,
*container_args,
"-scheme", scheme,
"-configuration", "Release",
"-derivedDataPath", str(export_path / "derived_data"),
@@ -879,7 +963,7 @@ async def build_project(task_id: str, task, config_data: dict, build_dir: Path)
archive_output = []
process = await asyncio.create_subprocess_exec(
"xcodebuild", "archive",
"-workspace", workspace_path.name,
*container_args,
"-scheme", scheme,
"-configuration", "Release",
"-archivePath", str(archive_path),
@@ -957,9 +1041,8 @@ async def upload_ipa(task_id: str, task, config_data: dict, ipa_path: Path, buil
await log_streamer.emit_step(task_id, "上传分发平台")
# 从当前服务的 config.json 读取上传配置,不执行分支源码中的上传脚本。
from ..routers.config import load_config
full_config = load_config()
config_data["_upload_config"] = full_config.get("upload", {})
from ..routers.config import load_upload_config
config_data["_upload_config"] = load_upload_config()
try:
await log_streamer.emit(task_id, f"使用 {config_data['_upload_config'].get('mode', '')} 发布 IPA...")
return await asyncio.to_thread(publish_ipa, config_data, ipa_path, build_dir)
+140 -2
View File
@@ -135,8 +135,14 @@ def _apply_dictionary_config(build_dir: Path, config: dict, messages: list[str])
messages.append("已替换词典版本配置")
def apply_project_config(build_dir: Path, config: dict) -> list[str]:
"""写入项目配置,返回适合实时日志展示的变更说明。"""
def apply_project_config(build_dir: Path, config: dict, definition: dict = None) -> list[str]:
"""写入项目配置,返回适合实时日志展示的变更说明。
definition 为项目定义(见 backend/projects.py),未传或 patcher 为 readoor 时按 Readoor 工程处理。
"""
if definition and definition.get("patcher") != "readoor":
return apply_generic_config(build_dir, config, definition)
messages: list[str] = []
if config.get("SCHEME") == "readoorDict":
_apply_dictionary_config(build_dir, config, messages)
@@ -190,3 +196,135 @@ def apply_project_config(build_dir: Path, config: dict) -> list[str]:
_write_export_options(build_dir, config)
messages.append("工程、签名、版本和分发配置已更新")
return messages
# —— 通用工程(patcher = generic)——
_TEMPLATE_PATTERN = re.compile(r"\$\{([A-Za-z0-9_.]+)\}")
_BUILD_SETTINGS_PATTERN = re.compile(r"buildSettings = \{\n(.*?)\n\t*\};", re.DOTALL)
_PLAIN_PBX_VALUE = re.compile(r"^[A-Za-z0-9_.$/-]+$")
def _render_template(template: str, config: dict) -> str:
"""替换 ${app.X}、${server.X}、${KEY}:分别取 App 字段、服务器环境字段和打包配置;值为空时报错。"""
def _lookup(match):
name = match.group(1)
if name.startswith("app."):
value = config.get("APP_VARS", {}).get(name[4:], "")
elif name.startswith("server."):
value = config.get("SERVER_VARS", {}).get(name[7:], "")
else:
value = config.get(name, "")
if value is None or str(value) == "":
raise ProjectPatchError(f"替换规则引用的 {name} 未配置")
return str(value)
return _TEMPLATE_PATTERN.sub(_lookup, template)
def _pbx_value(value: str) -> str:
return value if _PLAIN_PBX_VALUE.match(value) else '"' + value.replace('"', '\\"') + '"'
def _patch_pbxproj(path: Path, settings: dict) -> dict:
"""在 App 目标的 buildSettings 中替换设置(含 [sdk=iphoneos*] 变体),返回每个设置的替换次数。
测试目标(含 TEST_HOST / TEST_TARGET_NAME)有自己的 Bundle ID 和签名,跳过。
"""
if not path.exists():
raise ProjectPatchError(f"工程文件不存在: {path.parent.name}/{path.name}")
counts = {key: 0 for key in settings}
def _patch_block(match):
block = match.group(1)
if "TEST_HOST" in block or "TEST_TARGET_NAME" in block:
return match.group(0)
for key, value in settings.items():
pattern = re.compile(
rf'^(\s*)("?{re.escape(key)}(?:\[sdk=iphoneos\*\])?"?) = .*;$', re.MULTILINE
)
block, n = pattern.subn(lambda m: f"{m.group(1)}{m.group(2)} = {_pbx_value(value)};", block)
counts[key] += n
return match.group(0).replace(match.group(1), block, 1)
content = _BUILD_SETTINGS_PATTERN.sub(_patch_block, path.read_text(encoding="utf-8"))
path.write_text(content, encoding="utf-8")
return counts
def _apply_define_rule(path: Path, name: str, value: str) -> int:
"""替换所有未注释的 #define NAME ... 行(#if/#else 各分支中的同名宏都会被替换)。"""
pattern = re.compile(rf"^([ \t]*#define[ \t]+{re.escape(name)})(?:[ \t].*)?$", re.MULTILINE)
content, count = pattern.subn(lambda m: f"{m.group(1)} {value}", path.read_text(encoding="utf-8"))
path.write_text(content, encoding="utf-8")
return count
def _resolve_rule_file(build_dir: Path, file: str) -> Path:
path = (build_dir / file).resolve()
if not file or not path.is_relative_to(build_dir.resolve()):
raise ProjectPatchError(f"替换规则文件路径无效: {file}")
if not path.is_file():
raise ProjectPatchError(f"替换规则文件不存在: {file}")
return path
def apply_generic_config(build_dir: Path, config: dict, definition: dict) -> list[str]:
"""通用工程:替换 pbxproj 的版本号与签名、生成 exportOptions.plist,再执行项目的 patch_rules。
patch_rules 每项:
{"type": "define", "file": "相对路径", "name": "宏名", "value": "模板"}
替换所有未注释的 #define 宏名 行
{"type": "replace_line", "file": "相对路径", "keyword": "关键字", "line": "模板"}
整行替换包含关键字且未注释的行
模板中的 ${app.X} / ${server.X} / ${KEY} 取 App 字段、服务器环境字段、打包配置。
"""
messages: list[str] = []
xcodeproj = (definition.get("xcode") or {}).get("project", "")
if not xcodeproj:
raise ProjectPatchError("项目未配置 Xcode project(.xcodeproj)")
settings = {
"MARKETING_VERSION": config.get("VERSION", ""),
"CURRENT_PROJECT_VERSION": config.get("BUILD_VERSION", ""),
}
if config.get("BUNDLE_ID"):
settings["PRODUCT_BUNDLE_IDENTIFIER"] = config["BUNDLE_ID"]
if not config.get("NO_CERT"):
if config.get("TEAM_ID"):
settings["DEVELOPMENT_TEAM"] = config["TEAM_ID"]
if config.get("PROVISIONING_NAME"):
settings["PROVISIONING_PROFILE_SPECIFIER"] = config["PROVISIONING_NAME"]
settings["CODE_SIGN_STYLE"] = "Manual"
settings["CODE_SIGN_IDENTITY"] = "iPhone Distribution"
counts = _patch_pbxproj(build_dir / xcodeproj / "project.pbxproj", settings)
missing = [key for key in ("MARKETING_VERSION", "CURRENT_PROJECT_VERSION") if counts[key] == 0]
if missing:
raise ProjectPatchError(f"未找到待替换配置: {', '.join(missing)}")
messages.append(
f"工程配置已更新: 版本 {settings['MARKETING_VERSION']} ({settings['CURRENT_PROJECT_VERSION']})"
+ (f",Bundle ID {settings['PRODUCT_BUNDLE_IDENTIFIER']}" if "PRODUCT_BUNDLE_IDENTIFIER" in settings else "")
)
for index, rule in enumerate(definition.get("patch_rules") or [], start=1):
rule_type = rule.get("type")
path = _resolve_rule_file(build_dir, rule.get("file", ""))
if rule_type == "define":
name = rule.get("name", "")
value = _render_template(rule.get("value", ""), config)
if _apply_define_rule(path, name, value) == 0:
raise ProjectPatchError(f"{rule['file']} 中未找到 #define {name}")
messages.append(f"{rule['file']}: #define {name} {value}")
elif rule_type == "replace_line":
keyword = rule.get("keyword", "")
line = _render_template(rule.get("line", ""), config)
if not keyword or _replace_lines(path, keyword, line) == 0:
raise ProjectPatchError(f"{rule['file']} 中未找到包含 {keyword} 的行")
messages.append(f"{rule['file']}: {line.strip()}")
else:
raise ProjectPatchError(f"第 {index} 条替换规则类型无效: {rule_type}")
_write_export_options(build_dir, config)
messages.append("签名和分发配置已更新")
return messages