feat: 服务端集成自动打包流程
This commit is contained in:
shen
@@ -0,0 +1,144 @@
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
def build_canonical_symbol_tables(
|
||||
swift_files: list,
|
||||
objc_files: list,
|
||||
method_files: list,
|
||||
*,
|
||||
enable_objc_symbols: bool,
|
||||
prefilter_files,
|
||||
type_pattern: str,
|
||||
method_pattern: str,
|
||||
extract_swift_symbols,
|
||||
extract_objc_symbols,
|
||||
extract_methods,
|
||||
objc_symbol_prefixes: tuple,
|
||||
already_obfuscated_re,
|
||||
symbol_whitelist,
|
||||
symbol_prefix_whitelist: tuple,
|
||||
is_system_method,
|
||||
split_swift_signature_key,
|
||||
) -> dict:
|
||||
swift_symbol_files = prefilter_files(swift_files, type_pattern)
|
||||
objc_symbol_files = prefilter_files(objc_files, type_pattern) if enable_objc_symbols else []
|
||||
method_symbol_files = prefilter_files(method_files, method_pattern)
|
||||
|
||||
swift_symbols = set()
|
||||
objc_symbols = set()
|
||||
direct_methods = set()
|
||||
objc_selectors = set()
|
||||
swift_signatures = {}
|
||||
|
||||
for file_path in swift_symbol_files:
|
||||
swift_symbols.update(extract_swift_symbols(file_path))
|
||||
|
||||
for file_path in objc_symbol_files:
|
||||
objc_symbols.update(extract_objc_symbols(file_path))
|
||||
|
||||
for file_path in method_symbol_files:
|
||||
method_data = extract_methods(file_path)
|
||||
direct_methods.update(method_data["direct_names"])
|
||||
objc_selectors.update(method_data["objc_selectors"])
|
||||
swift_signatures.update(method_data["swift_signatures"])
|
||||
|
||||
canonical = {
|
||||
"swift_symbols": sorted(swift_symbols),
|
||||
"objc_symbols": [
|
||||
name for name in sorted(objc_symbols)
|
||||
if any(name.startswith(prefix) for prefix in objc_symbol_prefixes)
|
||||
and not already_obfuscated_re.match(name)
|
||||
and name not in symbol_whitelist
|
||||
and not name.startswith(symbol_prefix_whitelist)
|
||||
],
|
||||
"direct_methods": [
|
||||
name for name in sorted(direct_methods)
|
||||
if name and not already_obfuscated_re.match(name) and name not in symbol_whitelist and not is_system_method(name)
|
||||
],
|
||||
"objc_selectors": [
|
||||
name for name in sorted(objc_selectors)
|
||||
if name and not already_obfuscated_re.match(name) and name not in symbol_whitelist and not is_system_method(name)
|
||||
],
|
||||
"swift_signatures": {},
|
||||
}
|
||||
|
||||
for signature_key in sorted(swift_signatures.keys()):
|
||||
base_name, _ = split_swift_signature_key(signature_key)
|
||||
if not base_name:
|
||||
continue
|
||||
if already_obfuscated_re.match(base_name) or base_name in symbol_whitelist or is_system_method(base_name):
|
||||
continue
|
||||
canonical["swift_signatures"][signature_key] = swift_signatures[signature_key]
|
||||
|
||||
digest_payload = {
|
||||
key: canonical[key]
|
||||
for key in ("swift_symbols", "objc_symbols", "direct_methods", "objc_selectors", "swift_signatures")
|
||||
}
|
||||
canonical["digest"] = hashlib.sha256(
|
||||
json.dumps(digest_payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
).hexdigest()
|
||||
return canonical
|
||||
|
||||
|
||||
def build_type_mappings_from_canonical_table(canonical_tables: dict, salt: str, *, enable_objc_symbols: bool, build_unique_mapping, type_prefix: str) -> tuple:
|
||||
ordered_names = []
|
||||
seen = set()
|
||||
for bucket_name in ("objc_symbols", "swift_symbols"):
|
||||
for name in canonical_tables.get(bucket_name, []):
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
ordered_names.append(name)
|
||||
|
||||
combined_mapping = build_unique_mapping(ordered_names, salt, prefix=type_prefix)
|
||||
mapping_swift = {
|
||||
name: combined_mapping[name]
|
||||
for name in canonical_tables.get("swift_symbols", [])
|
||||
if name in combined_mapping
|
||||
}
|
||||
mapping_objc = {}
|
||||
if enable_objc_symbols:
|
||||
mapping_objc = {
|
||||
name: combined_mapping[name]
|
||||
for name in canonical_tables.get("objc_symbols", [])
|
||||
if name in combined_mapping
|
||||
}
|
||||
return mapping_swift, mapping_objc
|
||||
|
||||
|
||||
def build_method_mapping_from_canonical_table(canonical_tables: dict, salt: str, *, build_unique_mapping, method_prefix: str) -> dict:
|
||||
swift_signature_mapping = canonical_tables.get("swift_signatures", {})
|
||||
ordered_names = []
|
||||
seen = set()
|
||||
for bucket in (
|
||||
canonical_tables.get("direct_methods", []),
|
||||
canonical_tables.get("objc_selectors", []),
|
||||
sorted(swift_signature_mapping.keys()),
|
||||
):
|
||||
for name in bucket:
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
ordered_names.append(name)
|
||||
|
||||
combined_tokens = build_unique_mapping(ordered_names, salt, prefix=method_prefix)
|
||||
method_mapping = {
|
||||
name: combined_tokens[name]
|
||||
for name in canonical_tables.get("direct_methods", [])
|
||||
if name in combined_tokens
|
||||
}
|
||||
method_mapping.update({
|
||||
name: combined_tokens[name]
|
||||
for name in canonical_tables.get("objc_selectors", [])
|
||||
if name in combined_tokens
|
||||
})
|
||||
|
||||
for signature_key in sorted(swift_signature_mapping.keys()):
|
||||
if signature_key not in combined_tokens:
|
||||
continue
|
||||
entry = dict(swift_signature_mapping[signature_key])
|
||||
entry["obfuscated"] = combined_tokens[signature_key]
|
||||
method_mapping[signature_key] = entry
|
||||
|
||||
return method_mapping
|
||||
@@ -0,0 +1,143 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def path_matches_dir_names(path: str, dir_names: tuple) -> bool:
|
||||
if not dir_names:
|
||||
return False
|
||||
wanted = {name.lower() for name in dir_names}
|
||||
return any(part.lower() in wanted for part in Path(path).parts)
|
||||
|
||||
|
||||
def should_exclude_path(path: str, exclude_path_fragments) -> bool:
|
||||
return any(fragment in path for fragment in exclude_path_fragments)
|
||||
|
||||
|
||||
def _run_command_capture_stdout(args: list) -> str:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
if completed.returncode not in (0, 1):
|
||||
return ""
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def _discover_files_via_os_walk(scan_dirs: list, allowed_exts: tuple, exclude_path_fragments, skip_dir_names: tuple = (), basename_prefixes: tuple = ()) -> list:
|
||||
files = []
|
||||
allowed_exts = set(allowed_exts)
|
||||
for scan_dir in scan_dirs:
|
||||
for root, dirs, filenames in os.walk(scan_dir):
|
||||
dirs[:] = [
|
||||
directory for directory in dirs
|
||||
if not any(fragment.strip("/") in directory for fragment in exclude_path_fragments)
|
||||
]
|
||||
if path_matches_dir_names(root, skip_dir_names):
|
||||
continue
|
||||
for filename in filenames:
|
||||
ext = os.path.splitext(filename)[1]
|
||||
if ext not in allowed_exts:
|
||||
continue
|
||||
name_no_ext = os.path.splitext(filename)[0]
|
||||
if basename_prefixes and not any(name_no_ext.startswith(prefix) for prefix in basename_prefixes):
|
||||
continue
|
||||
full_path = os.path.normpath(os.path.join(root, filename))
|
||||
if should_exclude_path(full_path, exclude_path_fragments):
|
||||
continue
|
||||
if path_matches_dir_names(full_path, skip_dir_names):
|
||||
continue
|
||||
files.append(full_path)
|
||||
return sorted(set(files))
|
||||
|
||||
|
||||
def _discover_files_via_rg(scan_dirs: list, allowed_exts: tuple, exclude_path_fragments, skip_dir_names: tuple = (), basename_prefixes: tuple = ()) -> list:
|
||||
rg = shutil.which("rg")
|
||||
if not rg:
|
||||
return []
|
||||
|
||||
cmd = [rg, "--files", "--no-ignore", "--hidden"]
|
||||
for ext in allowed_exts:
|
||||
cmd.extend(["-g", f"*{ext}"])
|
||||
cmd.extend(scan_dirs)
|
||||
|
||||
output = _run_command_capture_stdout(cmd)
|
||||
if not output:
|
||||
return []
|
||||
|
||||
files = []
|
||||
for line in output.splitlines():
|
||||
path = os.path.normpath(line.strip())
|
||||
if not path or should_exclude_path(path, exclude_path_fragments):
|
||||
continue
|
||||
if path_matches_dir_names(path, skip_dir_names):
|
||||
continue
|
||||
if basename_prefixes:
|
||||
name_no_ext = os.path.splitext(os.path.basename(path))[0]
|
||||
if not any(name_no_ext.startswith(prefix) for prefix in basename_prefixes):
|
||||
continue
|
||||
files.append(path)
|
||||
|
||||
return sorted(set(files))
|
||||
|
||||
|
||||
def discover_files(scan_dirs: list, allowed_exts: tuple, exclude_path_fragments, skip_dir_names: tuple = (), basename_prefixes: tuple = ()) -> list:
|
||||
files = _discover_files_via_rg(
|
||||
scan_dirs,
|
||||
allowed_exts,
|
||||
exclude_path_fragments,
|
||||
skip_dir_names=skip_dir_names,
|
||||
basename_prefixes=basename_prefixes,
|
||||
)
|
||||
if files:
|
||||
return files
|
||||
return _discover_files_via_os_walk(
|
||||
scan_dirs,
|
||||
allowed_exts,
|
||||
exclude_path_fragments,
|
||||
skip_dir_names=skip_dir_names,
|
||||
basename_prefixes=basename_prefixes,
|
||||
)
|
||||
|
||||
|
||||
def _chunked(items: list, chunk_size: int):
|
||||
for index in range(0, len(items), chunk_size):
|
||||
yield items[index:index + chunk_size]
|
||||
|
||||
|
||||
def prefilter_files_with_rg(files: list, pattern: str, chunk_size: int = 256) -> list:
|
||||
files = sorted(set(files))
|
||||
if not files:
|
||||
return []
|
||||
|
||||
rg = shutil.which("rg")
|
||||
if not rg:
|
||||
return files
|
||||
|
||||
matched = set()
|
||||
for chunk in _chunked(files, chunk_size):
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[rg, "-l", "--no-messages", "-e", pattern, *chunk],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except Exception:
|
||||
return files
|
||||
|
||||
if completed.returncode not in (0, 1):
|
||||
return files
|
||||
for line in completed.stdout.splitlines():
|
||||
path = os.path.normpath(line.strip())
|
||||
if path:
|
||||
matched.add(path)
|
||||
|
||||
return sorted(matched)
|
||||
@@ -0,0 +1,128 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_json_file(path: str, default):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return default() if callable(default) else default
|
||||
|
||||
|
||||
def save_json_file(path: str, payload):
|
||||
Path(path).write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
class IncrementalAnalysisCache:
|
||||
def __init__(self, cache_path: str, version: int):
|
||||
self.cache_path = cache_path
|
||||
self.version = version
|
||||
self._state = None
|
||||
self._dirty = False
|
||||
self._memo = {}
|
||||
|
||||
def _default_state(self) -> dict:
|
||||
return {
|
||||
"version": self.version,
|
||||
"files": {},
|
||||
}
|
||||
|
||||
def load_state(self) -> dict:
|
||||
if self._state is not None:
|
||||
return self._state
|
||||
|
||||
payload = load_json_file(self.cache_path, self._default_state)
|
||||
if payload.get("version") != self.version or not isinstance(payload.get("files"), dict):
|
||||
payload = self._default_state()
|
||||
|
||||
self._state = payload
|
||||
return self._state
|
||||
|
||||
def save_if_dirty(self):
|
||||
if not self._dirty:
|
||||
return
|
||||
|
||||
try:
|
||||
save_json_file(self.cache_path, self.load_state())
|
||||
self._dirty = False
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def build_cache_key(analysis_kind: str, path: str) -> str:
|
||||
return f"{analysis_kind}:{os.path.realpath(path)}"
|
||||
|
||||
@staticmethod
|
||||
def _read_text_with_hash(path: str):
|
||||
try:
|
||||
text = Path(path).read_text(encoding="utf-8", errors="ignore")
|
||||
except Exception:
|
||||
return None, None
|
||||
return text, hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
def get_or_compute(self, path: str, analysis_kind: str, analyzer):
|
||||
cache_key = self.build_cache_key(analysis_kind, path)
|
||||
memo_key = (analysis_kind, os.path.realpath(path))
|
||||
if memo_key in self._memo:
|
||||
return self._memo[memo_key]
|
||||
|
||||
cache_state = self.load_state()
|
||||
files_cache = cache_state.setdefault("files", {})
|
||||
cached_entry = files_cache.get(cache_key)
|
||||
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
stat_signature = {
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"size": stat.st_size,
|
||||
}
|
||||
except Exception:
|
||||
stat_signature = {
|
||||
"mtime_ns": None,
|
||||
"size": None,
|
||||
}
|
||||
|
||||
if (
|
||||
isinstance(cached_entry, dict)
|
||||
and cached_entry.get("mtime_ns") == stat_signature["mtime_ns"]
|
||||
and cached_entry.get("size") == stat_signature["size"]
|
||||
and isinstance(cached_entry.get("analysis"), dict)
|
||||
):
|
||||
analysis = cached_entry["analysis"]
|
||||
self._memo[memo_key] = analysis
|
||||
return analysis
|
||||
|
||||
text, text_hash = self._read_text_with_hash(path)
|
||||
if text is None:
|
||||
analysis = analyzer("", path)
|
||||
self._memo[memo_key] = analysis
|
||||
return analysis
|
||||
|
||||
if (
|
||||
isinstance(cached_entry, dict)
|
||||
and cached_entry.get("sha256") == text_hash
|
||||
and isinstance(cached_entry.get("analysis"), dict)
|
||||
):
|
||||
cached_entry.update(stat_signature)
|
||||
self._dirty = True
|
||||
analysis = cached_entry["analysis"]
|
||||
self._memo[memo_key] = analysis
|
||||
return analysis
|
||||
|
||||
analysis = analyzer(text, path)
|
||||
files_cache[cache_key] = {
|
||||
"path": os.path.realpath(path),
|
||||
"analysis_kind": analysis_kind,
|
||||
"mtime_ns": stat_signature["mtime_ns"],
|
||||
"size": stat_signature["size"],
|
||||
"sha256": text_hash,
|
||||
"analysis": analysis,
|
||||
}
|
||||
self._dirty = True
|
||||
self._memo[memo_key] = analysis
|
||||
return analysis
|
||||
Reference in New Issue
Block a user