import CryptoKit import Foundation public struct RDAIIndexOptions: Sendable { public var scope: RDAIReadScope public var priorityResource: RDAIResourceIdentifier? public init(scope: RDAIReadScope, priorityResource: RDAIResourceIdentifier? = nil) { self.scope = scope self.priorityResource = priorityResource } } public enum RDAIIndexState: Equatable, Sendable { case notStarted case indexing(completedResources: Int, totalResources: Int) case paused(completedResources: Int, totalResources: Int) case ready case failed(RDAIError) } public struct RDAIRetrievalOptions: Sendable { public var maximumResults: Int public var scope: RDAIReadScope public var minimumScore: Double public init(maximumResults: Int = 4, scope: RDAIReadScope, minimumScore: Double = 0.01) { self.maximumResults = max(1, maximumResults) self.scope = scope self.minimumScore = min(max(minimumScore, 0), 1) } } public struct RDAIRetrievalMatch: Sendable, Identifiable { public let id: String public let passage: RDAIPassage public let score: Double public let lexicalScore: Double public let semanticScore: Double? public init(passage: RDAIPassage, score: Double, lexicalScore: Double, semanticScore: Double? = nil) { id = passage.id self.passage = passage self.score = score self.lexicalScore = lexicalScore self.semanticScore = semanticScore } } public enum RDAIContentHasher { public static func hash(_ text: String) -> String { SHA256.hash(data: Data(text.utf8)).map { String(format: "%02x", $0) }.joined() } } public actor RDAIInMemoryIndexStore { private var resources: [RDAIDocumentIdentifier: [RDAIResourceDescriptor]] = [:] private var passages: [RDAIDocumentIdentifier: [RDAIPassage]] = [:] private var entities: [RDAIDocumentIdentifier: [RDAIEntityMention]] = [:] public init() {} public func replace( document: RDAIDocumentDescriptor, resource: RDAIResourceDescriptor, analysis: RDAIAnalysisResult ) { var documentResources = resources[document.identifier] ?? [] documentResources.removeAll { $0.identifier == resource.identifier } documentResources.append(resource) resources[document.identifier] = documentResources.sorted { $0.order < $1.order } var documentPassages = passages[document.identifier] ?? [] documentPassages.removeAll { $0.resourceIdentifier == resource.identifier } documentPassages.append(contentsOf: analysis.passages) passages[document.identifier] = documentPassages var documentEntities = entities[document.identifier] ?? [] documentEntities.removeAll { $0.locator.resourceIdentifier == resource.identifier } documentEntities.append(contentsOf: analysis.entityMentions) entities[document.identifier] = documentEntities } public func passages(for documentIdentifier: RDAIDocumentIdentifier) -> [RDAIPassage] { passages[documentIdentifier] ?? [] } public func entities(for documentIdentifier: RDAIDocumentIdentifier) -> [RDAIEntityMention] { entities[documentIdentifier] ?? [] } public func resourceOrder( for identifier: RDAIResourceIdentifier, documentIdentifier: RDAIDocumentIdentifier ) -> Int? { resources[documentIdentifier]?.first { $0.identifier == identifier }?.order } public func remove(documentIdentifier: RDAIDocumentIdentifier) { resources.removeValue(forKey: documentIdentifier) passages.removeValue(forKey: documentIdentifier) entities.removeValue(forKey: documentIdentifier) } } /// First implementation of the local indexing pipeline. Storage is deliberately /// injectable so a SQLite-backed store can replace it without changing adapters. @MainActor public final class RDAIReaderService { public private(set) var state: RDAIIndexState = .notStarted { didSet { stateContinuations.values.forEach { $0.yield(state) } } } private let contentProvider: RDAIContentProvider private let analyzer: any RDAIAnalyzing private let store: RDAIInMemoryIndexStore private let persistentStore: RDAISQLiteIndexStore? private let generativeProvider: (any RDAIGenerativeProvider)? private let semanticScorer: (any RDAISemanticScoring)? private let configuration: RDAIReaderConfiguration private let diagnostics: (any RDAIDiagnosticsRecording)? private var stateContinuations: [UUID: AsyncStream.Continuation] = [:] private var pauseRequested = false private var resumeContinuation: CheckedContinuation? public init( contentProvider: RDAIContentProvider, analyzer: some RDAIAnalyzing, semanticScorer: (any RDAISemanticScoring)? = nil, store: RDAIInMemoryIndexStore = .init(), persistentStore: RDAISQLiteIndexStore? = nil, generativeProvider: (any RDAIGenerativeProvider)? = nil, configuration: RDAIReaderConfiguration = .default, diagnostics: (any RDAIDiagnosticsRecording)? = nil ) { self.contentProvider = contentProvider self.analyzer = analyzer self.semanticScorer = semanticScorer self.store = store self.persistentStore = persistentStore self.generativeProvider = generativeProvider self.configuration = configuration self.diagnostics = diagnostics } public func stateUpdates() -> AsyncStream { let identifier = UUID() return AsyncStream { [weak self] continuation in guard let self else { continuation.finish() return } continuation.yield(self.state) self.stateContinuations[identifier] = continuation continuation.onTermination = { [weak self] _ in Task { @MainActor in self?.stateContinuations.removeValue(forKey: identifier) } } } } public func prepareIndex(options: RDAIIndexOptions) async throws { guard configuration.isAIEnabled else { throw RDAIError.disabled } let startedAt = Date() let document = contentProvider.aiDocumentDescriptor() var resources = try await contentProvider.aiResources().sorted { $0.order < $1.order } resources = prioritized(resources, preferred: options.priorityResource) let scopedResources = try await resourcesAllowed(by: options.scope, document: document, resources: resources) state = .indexing(completedResources: 0, totalResources: scopedResources.count) do { for (index, resource) in scopedResources.enumerated() { try Task.checkCancellation() await waitIfPaused(completed: index, total: scopedResources.count) try Task.checkCancellation() let sourceSnapshot = try await contentProvider.aiResourceSnapshot(for: resource.identifier) let snapshot = limited(sourceSnapshot, by: options.scope) guard !snapshot.sourceText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { state = .indexing(completedResources: index + 1, totalResources: scopedResources.count) continue } if let persistentStore, try await persistentStore.resourceHash(documentID: document.identifier, resourceID: resource.identifier) == snapshot.sourceHash { try await persistentStore.saveCheckpoint(documentID: document.identifier, resourceOrder: resource.order) state = .indexing(completedResources: index + 1, totalResources: scopedResources.count) continue } let analysis = await analyzer.analyze(document: document, snapshot: snapshot) if let persistentStore { try await persistentStore.replace(document: document, resource: resource, analysis: analysis, sourceHash: snapshot.sourceHash) try await persistentStore.saveCheckpoint(documentID: document.identifier, resourceOrder: resource.order) } else { await store.replace(document: document, resource: resource, analysis: analysis) } state = .indexing(completedResources: index + 1, totalResources: scopedResources.count) } state = .ready await record(stage: .indexing, startedAt: startedAt, count: scopedResources.count) } catch is CancellationError { state = .notStarted throw RDAIError.cancelled } catch let error as RDAIError { state = .failed(error) await record(stage: .indexing, startedAt: startedAt, errorCode: "rdai-error") throw error } catch { // Public errors must not expose paths, OCR text or SQLite details. let wrapped = RDAIError.indexingFailed("unexpected") state = .failed(wrapped) await record(stage: .indexing, startedAt: startedAt, errorCode: "indexing-failed") throw wrapped } } public func retrieve(query: String, options: RDAIRetrievalOptions) async -> [RDAIRetrievalMatch] { let document = contentProvider.aiDocumentDescriptor() let allPassages = await storedPassages(for: document.identifier) let filtered = await filter(allPassages, scope: options.scope, documentIdentifier: document.identifier) var matches: [RDAIRetrievalMatch] = [] for passage in filtered { let score = lexicalScore(query: query, text: passage.text) let semantic = await semanticScorer?.score(query: query, text: passage.text, languageCode: passage.languageCode) let finalScore = semantic.map { 0.6 * score + 0.4 * $0 } ?? score guard finalScore >= options.minimumScore else { continue } matches.append(RDAIRetrievalMatch(passage: passage, score: finalScore, lexicalScore: score, semanticScore: semantic)) } return matches.sorted { if $0.score == $1.score { return $0.passage.order < $1.passage.order } return $0.score > $1.score } .prefix(options.maximumResults) .map { $0 } } public func removeIndex() async { if let persistentStore { try? await persistentStore.remove(documentID: contentProvider.aiDocumentDescriptor().identifier) } else { await store.remove(documentIdentifier: contentProvider.aiDocumentDescriptor().identifier) } state = .notStarted } public func pauseIndexing() { pauseRequested = true } public func resumeIndexing() { pauseRequested = false resumeContinuation?.resume() resumeContinuation = nil } public func removeAllAIData() async { if let persistentStore { try? await persistentStore.removeAll() } else { await store.remove(documentIdentifier: contentProvider.aiDocumentDescriptor().identifier) } state = .notStarted } public func storageBytes() async -> Int64 { await persistentStore?.storageBytes() ?? 0 } public func showCitation(_ citation: RDAICitation) async throws { try await contentProvider.aiShowCitationHighlight(citation) } public func clearCitationHighlight() { contentProvider.aiClearCitationHighlight() } public func availability(for capability: RDAICapability, locale: Locale = .current) async -> RDAICapabilityAvailability { guard configuration.isAIEnabled else { return .unavailable(reason: .providerNotInstalled) } switch capability { case .languageAnalysis, .entityExtraction, .lexicalSearch: return .available case .semanticSearch: return semanticScorer == nil ? .degraded(reason: .embeddingAssetsUnavailable) : .available case .summarization, .questionAnswering, .characterRelationships: guard let generativeProvider else { return .degraded(reason: .providerNotInstalled) } return await generativeProvider.availability(for: capability, locale: locale) } } public func summarize(scope: RDAIReadScope, length: RDAISummaryLength) async throws -> RDAISummary { guard configuration.isAIEnabled else { throw RDAIError.disabled } let document = contentProvider.aiDocumentDescriptor() let cacheKey = artifactKey(document: document, scope: scope, suffix: "summary|\(length.rawValue)") if configuration.storesGeneratedArtifacts, let persistentStore, let data = try? await persistentStore.artifact(documentID: document.identifier, key: cacheKey), let cached = try? JSONDecoder().decode(RDAISummary.self, from: data) { return cached } let passages = try await passages(for: scope, limit: length == .brief ? 2 : (length == .standard ? 4 : 6)) let metadata = generationMetadata(provider: generativeProvider?.identifier ?? "extractive") if let generativeProvider, case .available = await generativeProvider.availability(for: .summarization, locale: .current), case let .summary(overview, statements) = try await generativeProvider.generate(.init(task: .summary(length), passages: passages, scope: scope)) { let validated = validatedStatements(statements, passages: passages) guard !validated.isEmpty else { throw RDAIError.insufficientEvidence } let result = RDAISummary(title: document.title, overview: overview, keyPoints: validated, citations: citations(for: passages), metadata: metadata) try? await saveArtifact(result, document: document, key: cacheKey) return result } let statements = passages.map { RDAISourcedStatement(text: $0.text, citationIdentifiers: [$0.id]) } let result = RDAISummary(title: document.title, overview: passages.first?.text ?? "暂无可总结的已读内容。", keyPoints: statements, citations: citations(for: passages), metadata: metadata) try? await saveArtifact(result, document: document, key: cacheKey) return result } public func answer(question: String, scope: RDAIReadScope) async throws -> RDAIAnswer { guard configuration.isAIEnabled else { throw RDAIError.disabled } let document = contentProvider.aiDocumentDescriptor() let cacheKey = artifactKey(document: document, scope: scope, suffix: "answer|\(RDAIContentHasher.hash(question))") if configuration.storesGeneratedArtifacts, let persistentStore, let data = try? await persistentStore.artifact(documentID: document.identifier, key: cacheKey), let cached = try? JSONDecoder().decode(RDAIAnswer.self, from: data) { return cached } let matches = await retrieve(query: question, options: .init(maximumResults: configuration.maximumRetrievedPassages, scope: scope)) let passages = matches.map(\.passage) let metadata = generationMetadata(provider: generativeProvider?.identifier ?? "extractive") guard !passages.isEmpty else { return RDAIAnswer(status: .insufficientEvidence, text: "在当前阅读范围内未找到足够证据。", statements: [], citations: [], metadata: metadata) } if let generativeProvider, case .available = await generativeProvider.availability(for: .questionAnswering, locale: .current), case let .answer(text, statements) = try await generativeProvider.generate(.init(task: .answer, userText: question, passages: passages, scope: scope)) { let validated = validatedStatements(statements, passages: passages) guard !validated.isEmpty else { return RDAIAnswer(status: .insufficientEvidence, text: "模型结果缺少可验证的原文证据。", statements: [], citations: [], metadata: metadata) } let result = RDAIAnswer(status: .answered, text: text, statements: validated, citations: citations(for: passages), metadata: metadata) try? await saveArtifact(result, document: document, key: cacheKey) return result } let statements = passages.map { RDAISourcedStatement(text: $0.text, citationIdentifiers: [$0.id]) } let result = RDAIAnswer(status: .answered, text: "以下内容与问题最相关:", statements: statements, citations: citations(for: passages), metadata: metadata) try? await saveArtifact(result, document: document, key: cacheKey) return result } public func characters(scope: RDAIReadScope) async -> [RDAICharacter] { let document = contentProvider.aiDocumentDescriptor() let mentions = (await storedEntities(for: document.identifier)).filter { $0.kind == .person } let grouped = Dictionary(grouping: mentions, by: { canonicalCharacterName($0.normalizedName) }) return grouped.compactMap { (name, values) -> RDAICharacter? in guard let first = values.first else { return nil } let evidence = values.prefix(3).map { mention in RDAICitation(passageIdentifier: mention.id, quote: mention.surfaceText, locator: mention.locator) } return RDAICharacter(id: RDAIContentHasher.hash(name), displayName: first.surfaceText, aliases: Array(Set(values.map(\.surfaceText))).sorted(), description: "在已读内容中出现 \(values.count) 次。", firstAppearance: evidence.first, evidence: evidence) }.sorted { $0.displayName < $1.displayName } } /// Conservative local relationship extraction: entities must co-occur in /// one allowed passage. It never promotes co-occurrence to a fact. public func relationships(scope: RDAIReadScope) async -> [RDAIRelationship] { let document = contentProvider.aiDocumentDescriptor() let passages = await filter(await storedPassages(for: document.identifier), scope: scope, documentIdentifier: document.identifier) let mentions = (await storedEntities(for: document.identifier)).filter { $0.kind == .person } var evidenceByPair: [String: [RDAICitation]] = [:] for passage in passages { let names = Array(Set(mentions.filter { $0.locator.resourceIdentifier == passage.resourceIdentifier && $0.locator.textRange.intersects(passage.locator.textRange) }.map { canonicalCharacterName($0.normalizedName) })).sorted() guard names.count > 1 else { continue } for left in names.indices { for right in names.indices where right > left { let key = "\(names[left])|\(names[right])" evidenceByPair[key, default: []].append(citation(for: passage)) } } } return evidenceByPair.compactMap { key, evidence in let names = key.split(separator: "|", maxSplits: 1).map(String.init) guard names.count == 2, !evidence.isEmpty else { return nil } return RDAIRelationship(id: RDAIContentHasher.hash(key), sourceCharacterIdentifier: RDAIContentHasher.hash(names[0]), targetCharacterIdentifier: RDAIContentHasher.hash(names[1]), label: "共同出现", status: .possible, evidence: Array(evidence.prefix(3))) }.sorted { $0.id < $1.id } } private func passages(for scope: RDAIReadScope, limit: Int) async throws -> [RDAIPassage] { let document = contentProvider.aiDocumentDescriptor() let stored = await storedPassages(for: document.identifier) let all = await filter(stored, scope: scope, documentIdentifier: document.identifier) guard !all.isEmpty else { throw RDAIError.insufficientEvidence } return Array(all.sorted { $0.order < $1.order }.prefix(limit)) } private func citations(for passages: [RDAIPassage]) -> [RDAICitation] { passages.map(citation(for:)) } private func citation(for passage: RDAIPassage) -> RDAICitation { RDAICitation(passageIdentifier: passage.id, quote: passage.text, locator: passage.locator) } private func validatedStatements(_ statements: [RDAISourcedStatement], passages: [RDAIPassage]) -> [RDAISourcedStatement] { let identifiers = Set(passages.map(\.id)) return statements.compactMap { statement in let citations = statement.citationIdentifiers.filter { identifiers.contains($0) } guard !citations.isEmpty, !statement.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } return RDAISourcedStatement(id: statement.id, text: statement.text, citationIdentifiers: citations) } } private func canonicalCharacterName(_ value: String) -> String { let compact = value.unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) || (0x4E00...0x9FFF).contains($0.value) } var name = String(String.UnicodeScalarView(compact)).lowercased() // Only remove explicit honorific suffixes; ambiguous aliases remain // distinct so the UI does not silently merge two people. for suffix in ["先生", "女士", "小姐", "老师", "教授", "博士", "将军"] where name.hasSuffix(suffix) && name.count > suffix.count { name.removeLast(suffix.count) break } return name } private func generationMetadata(provider: String) -> RDAIGenerationMetadata { .init(providerIdentifier: provider, promptIdentifier: "rdai.evidence.v1", promptVersion: 1, scopeHash: RDAIContentHasher.hash(contentProvider.aiDocumentDescriptor().identifier.rawValue)) } private func artifactKey(document: RDAIDocumentDescriptor, scope: RDAIReadScope, suffix: String) -> String { let scopeData = (try? JSONEncoder().encode(scope)).map { String(data: $0, encoding: .utf8) } ?? "" return RDAIContentHasher.hash("\(document.contentRevision)|\(scopeData)|rdai.evidence.v1|\(generativeProvider?.identifier ?? "extractive")|\(suffix)") } private func saveArtifact(_ value: T, document: RDAIDocumentDescriptor, key: String) async throws { guard configuration.storesGeneratedArtifacts, let persistentStore else { return } try await persistentStore.saveArtifact(documentID: document.identifier, key: key, payload: JSONEncoder().encode(value)) } private func record(stage: RDAIDiagnosticStage, startedAt: Date, errorCode: String? = nil, count: Int = 0) async { guard configuration.diagnosticsLevel != .disabled, let diagnostics else { return } await diagnostics.record(.init(documentHash: RDAIContentHasher.hash(contentProvider.aiDocumentDescriptor().identifier.rawValue), stage: stage, durationMilliseconds: Int(Date().timeIntervalSince(startedAt) * 1_000), errorCode: errorCode, count: count)) } private func prioritized( _ resources: [RDAIResourceDescriptor], preferred: RDAIResourceIdentifier? ) -> [RDAIResourceDescriptor] { guard let preferred else { return resources } return resources.sorted { lhs, rhs in if lhs.identifier == preferred { return true } if rhs.identifier == preferred { return false } return lhs.order < rhs.order } } private func waitIfPaused(completed: Int, total: Int) async { guard pauseRequested else { return } state = .paused(completedResources: completed, totalResources: total) await withTaskCancellationHandler(operation: { await withCheckedContinuation { continuation in resumeContinuation = continuation } }, onCancel: { [weak self] in Task { @MainActor in self?.resumeIndexing() } }) state = .indexing(completedResources: completed, totalResources: total) } private func resourcesAllowed( by scope: RDAIReadScope, document: RDAIDocumentDescriptor, resources: [RDAIResourceDescriptor] ) async throws -> [RDAIResourceDescriptor] { guard !scope.includesWholeDocument else { return resources } guard let upperBound = scope.upperBound, upperBound.documentIdentifier == document.identifier, let boundary = resources.first(where: { $0.identifier == upperBound.resourceIdentifier }) else { return [] } return resources.filter { $0.order <= boundary.order } } private func limited(_ snapshot: RDAIResourceSnapshot, by scope: RDAIReadScope) -> RDAIResourceSnapshot { guard !scope.includesWholeDocument, let upperBound = scope.upperBound, upperBound.resourceIdentifier == snapshot.descriptor.identifier else { return snapshot } let length = min(max(upperBound.textRange.upperBound, 0), snapshot.sourceText.utf16.count) let source = (snapshot.sourceText as NSString).substring(to: length) let availableRange = RDAITextRange(location: 0, length: length) let runs = snapshot.locatorRuns.filter { $0.textRange.intersects(availableRange) } return RDAIResourceSnapshot( descriptor: snapshot.descriptor, sourceText: source, sourceHash: RDAIContentHasher.hash(source), locatorRuns: runs ) } private func filter( _ passages: [RDAIPassage], scope: RDAIReadScope, documentIdentifier: RDAIDocumentIdentifier ) async -> [RDAIPassage] { guard !scope.includesWholeDocument else { return passages } guard let upperBound = scope.upperBound, upperBound.documentIdentifier == documentIdentifier, let boundaryOrder = await resourceOrder(for: upperBound.resourceIdentifier, documentIdentifier: documentIdentifier) else { return [] } return await withTaskGroup(of: (RDAIPassage, Int?).self) { group in for passage in passages { group.addTask { [store] in let order = await self.resourceOrder(for: passage.resourceIdentifier, documentIdentifier: documentIdentifier) return (passage, order) } } var allowed: [RDAIPassage] = [] for await (passage, order) in group { guard let order, order <= boundaryOrder else { continue } if passage.resourceIdentifier == upperBound.resourceIdentifier, passage.locator.textRange.location >= upperBound.textRange.upperBound { continue } allowed.append(passage) } return allowed } } private func resourceOrder(for identifier: RDAIResourceIdentifier, documentIdentifier: RDAIDocumentIdentifier) async -> Int? { if let persistentStore { return try? await persistentStore.resourceOrder(documentID: documentIdentifier, resourceID: identifier) } return await store.resourceOrder(for: identifier, documentIdentifier: documentIdentifier) } private func storedPassages(for identifier: RDAIDocumentIdentifier) async -> [RDAIPassage] { if let persistentStore { return (try? await persistentStore.passages(documentID: identifier)) ?? [] } return await store.passages(for: identifier) } private func storedEntities(for identifier: RDAIDocumentIdentifier) async -> [RDAIEntityMention] { if let persistentStore { return (try? await persistentStore.entities(documentID: identifier)) ?? [] } return await store.entities(for: identifier) } private func lexicalScore(query: String, text: String) -> Double { let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return 0 } let normalizedText = text.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current) let normalizedQuery = trimmed.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current) if normalizedText.localizedCaseInsensitiveContains(normalizedQuery) { return 1 } let terms = normalizedQuery.split(whereSeparator: { $0.isWhitespace || $0.isNewline }).map(String.init) guard !terms.isEmpty else { return 0 } let matches = terms.filter { normalizedText.localizedCaseInsensitiveContains($0) }.count return Double(matches) / Double(terms.count) } }