更新阅读器功能与示例
This commit is contained in:
shen
@@ -0,0 +1,23 @@
|
||||
import Foundation
|
||||
|
||||
public enum RDAIDiagnosticStage: String, Codable, Sendable { case indexing, retrieval, generation, storage }
|
||||
|
||||
public struct RDAIDiagnosticEvent: Codable, Sendable {
|
||||
public let documentHash: String
|
||||
public let stage: RDAIDiagnosticStage
|
||||
public let durationMilliseconds: Int
|
||||
public let errorCode: String?
|
||||
public let count: Int
|
||||
public let createdAt: Date
|
||||
public init(documentHash: String, stage: RDAIDiagnosticStage, durationMilliseconds: Int, errorCode: String? = nil, count: Int = 0, createdAt: Date = .init()) { self.documentHash = documentHash; self.stage = stage; self.durationMilliseconds = durationMilliseconds; self.errorCode = errorCode; self.count = count; self.createdAt = createdAt }
|
||||
}
|
||||
|
||||
public protocol RDAIDiagnosticsRecording: Sendable { func record(_ event: RDAIDiagnosticEvent) async }
|
||||
|
||||
public actor RDAIInMemoryDiagnosticsRecorder: RDAIDiagnosticsRecording {
|
||||
private var values: [RDAIDiagnosticEvent] = []
|
||||
public init() {}
|
||||
public func record(_ event: RDAIDiagnosticEvent) { values.append(event) }
|
||||
public func events() -> [RDAIDiagnosticEvent] { values }
|
||||
public func removeAll() { values.removeAll() }
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
|
||||
public struct RDAIIndexOptions: Sendable {
|
||||
public var scope: RDAIReadScope
|
||||
public var priorityResource: RDAIResourceIdentifier?
|
||||
|
||||
public init(scope: RDAIReadScope, priorityResource: RDAIResourceIdentifier? = nil) {
|
||||
self.scope = scope
|
||||
self.priorityResource = priorityResource
|
||||
}
|
||||
}
|
||||
|
||||
public enum RDAIIndexState: Equatable, Sendable {
|
||||
case notStarted
|
||||
case indexing(completedResources: Int, totalResources: Int)
|
||||
case paused(completedResources: Int, totalResources: Int)
|
||||
case ready
|
||||
case failed(RDAIError)
|
||||
}
|
||||
|
||||
public struct RDAIRetrievalOptions: Sendable {
|
||||
public var maximumResults: Int
|
||||
public var scope: RDAIReadScope
|
||||
public var minimumScore: Double
|
||||
|
||||
public init(maximumResults: Int = 4, scope: RDAIReadScope, minimumScore: Double = 0.01) {
|
||||
self.maximumResults = max(1, maximumResults)
|
||||
self.scope = scope
|
||||
self.minimumScore = min(max(minimumScore, 0), 1)
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIRetrievalMatch: Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let passage: RDAIPassage
|
||||
public let score: Double
|
||||
public let lexicalScore: Double
|
||||
public let semanticScore: Double?
|
||||
|
||||
public init(passage: RDAIPassage, score: Double, lexicalScore: Double, semanticScore: Double? = nil) {
|
||||
id = passage.id
|
||||
self.passage = passage
|
||||
self.score = score
|
||||
self.lexicalScore = lexicalScore
|
||||
self.semanticScore = semanticScore
|
||||
}
|
||||
}
|
||||
|
||||
public enum RDAIContentHasher {
|
||||
public static func hash(_ text: String) -> String {
|
||||
SHA256.hash(data: Data(text.utf8)).map { String(format: "%02x", $0) }.joined()
|
||||
}
|
||||
}
|
||||
|
||||
public actor RDAIInMemoryIndexStore {
|
||||
private var resources: [RDAIDocumentIdentifier: [RDAIResourceDescriptor]] = [:]
|
||||
private var passages: [RDAIDocumentIdentifier: [RDAIPassage]] = [:]
|
||||
private var entities: [RDAIDocumentIdentifier: [RDAIEntityMention]] = [:]
|
||||
|
||||
public init() {}
|
||||
|
||||
public func replace(
|
||||
document: RDAIDocumentDescriptor,
|
||||
resource: RDAIResourceDescriptor,
|
||||
analysis: RDAIAnalysisResult
|
||||
) {
|
||||
var documentResources = resources[document.identifier] ?? []
|
||||
documentResources.removeAll { $0.identifier == resource.identifier }
|
||||
documentResources.append(resource)
|
||||
resources[document.identifier] = documentResources.sorted { $0.order < $1.order }
|
||||
|
||||
var documentPassages = passages[document.identifier] ?? []
|
||||
documentPassages.removeAll { $0.resourceIdentifier == resource.identifier }
|
||||
documentPassages.append(contentsOf: analysis.passages)
|
||||
passages[document.identifier] = documentPassages
|
||||
|
||||
var documentEntities = entities[document.identifier] ?? []
|
||||
documentEntities.removeAll { $0.locator.resourceIdentifier == resource.identifier }
|
||||
documentEntities.append(contentsOf: analysis.entityMentions)
|
||||
entities[document.identifier] = documentEntities
|
||||
}
|
||||
|
||||
public func passages(for documentIdentifier: RDAIDocumentIdentifier) -> [RDAIPassage] {
|
||||
passages[documentIdentifier] ?? []
|
||||
}
|
||||
|
||||
public func entities(for documentIdentifier: RDAIDocumentIdentifier) -> [RDAIEntityMention] {
|
||||
entities[documentIdentifier] ?? []
|
||||
}
|
||||
|
||||
public func resourceOrder(
|
||||
for identifier: RDAIResourceIdentifier,
|
||||
documentIdentifier: RDAIDocumentIdentifier
|
||||
) -> Int? {
|
||||
resources[documentIdentifier]?.first { $0.identifier == identifier }?.order
|
||||
}
|
||||
|
||||
public func remove(documentIdentifier: RDAIDocumentIdentifier) {
|
||||
resources.removeValue(forKey: documentIdentifier)
|
||||
passages.removeValue(forKey: documentIdentifier)
|
||||
entities.removeValue(forKey: documentIdentifier)
|
||||
}
|
||||
}
|
||||
|
||||
/// First implementation of the local indexing pipeline. Storage is deliberately
|
||||
/// injectable so a SQLite-backed store can replace it without changing adapters.
|
||||
@MainActor
|
||||
public final class RDAIReaderService {
|
||||
public private(set) var state: RDAIIndexState = .notStarted {
|
||||
didSet { stateContinuations.values.forEach { $0.yield(state) } }
|
||||
}
|
||||
|
||||
private let contentProvider: RDAIContentProvider
|
||||
private let analyzer: any RDAIAnalyzing
|
||||
private let store: RDAIInMemoryIndexStore
|
||||
private let persistentStore: RDAISQLiteIndexStore?
|
||||
private let generativeProvider: (any RDAIGenerativeProvider)?
|
||||
private let semanticScorer: (any RDAISemanticScoring)?
|
||||
private let configuration: RDAIReaderConfiguration
|
||||
private let diagnostics: (any RDAIDiagnosticsRecording)?
|
||||
private var stateContinuations: [UUID: AsyncStream<RDAIIndexState>.Continuation] = [:]
|
||||
private var pauseRequested = false
|
||||
private var resumeContinuation: CheckedContinuation<Void, Never>?
|
||||
|
||||
public init(
|
||||
contentProvider: RDAIContentProvider,
|
||||
analyzer: some RDAIAnalyzing,
|
||||
semanticScorer: (any RDAISemanticScoring)? = nil,
|
||||
store: RDAIInMemoryIndexStore = .init(),
|
||||
persistentStore: RDAISQLiteIndexStore? = nil,
|
||||
generativeProvider: (any RDAIGenerativeProvider)? = nil,
|
||||
configuration: RDAIReaderConfiguration = .default,
|
||||
diagnostics: (any RDAIDiagnosticsRecording)? = nil
|
||||
) {
|
||||
self.contentProvider = contentProvider
|
||||
self.analyzer = analyzer
|
||||
self.semanticScorer = semanticScorer
|
||||
self.store = store
|
||||
self.persistentStore = persistentStore
|
||||
self.generativeProvider = generativeProvider
|
||||
self.configuration = configuration
|
||||
self.diagnostics = diagnostics
|
||||
}
|
||||
|
||||
public func stateUpdates() -> AsyncStream<RDAIIndexState> {
|
||||
let identifier = UUID()
|
||||
return AsyncStream { [weak self] continuation in
|
||||
guard let self else {
|
||||
continuation.finish()
|
||||
return
|
||||
}
|
||||
continuation.yield(self.state)
|
||||
self.stateContinuations[identifier] = continuation
|
||||
continuation.onTermination = { [weak self] _ in
|
||||
Task { @MainActor in self?.stateContinuations.removeValue(forKey: identifier) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func prepareIndex(options: RDAIIndexOptions) async throws {
|
||||
guard configuration.isAIEnabled else { throw RDAIError.disabled }
|
||||
let startedAt = Date()
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
var resources = try await contentProvider.aiResources().sorted { $0.order < $1.order }
|
||||
resources = prioritized(resources, preferred: options.priorityResource)
|
||||
let scopedResources = try await resourcesAllowed(by: options.scope, document: document, resources: resources)
|
||||
state = .indexing(completedResources: 0, totalResources: scopedResources.count)
|
||||
|
||||
do {
|
||||
for (index, resource) in scopedResources.enumerated() {
|
||||
try Task.checkCancellation()
|
||||
await waitIfPaused(completed: index, total: scopedResources.count)
|
||||
try Task.checkCancellation()
|
||||
let sourceSnapshot = try await contentProvider.aiResourceSnapshot(for: resource.identifier)
|
||||
let snapshot = limited(sourceSnapshot, by: options.scope)
|
||||
guard !snapshot.sourceText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
|
||||
state = .indexing(completedResources: index + 1, totalResources: scopedResources.count)
|
||||
continue
|
||||
}
|
||||
if let persistentStore,
|
||||
try await persistentStore.resourceHash(documentID: document.identifier, resourceID: resource.identifier) == snapshot.sourceHash {
|
||||
try await persistentStore.saveCheckpoint(documentID: document.identifier, resourceOrder: resource.order)
|
||||
state = .indexing(completedResources: index + 1, totalResources: scopedResources.count)
|
||||
continue
|
||||
}
|
||||
let analysis = await analyzer.analyze(document: document, snapshot: snapshot)
|
||||
if let persistentStore {
|
||||
try await persistentStore.replace(document: document, resource: resource, analysis: analysis, sourceHash: snapshot.sourceHash)
|
||||
try await persistentStore.saveCheckpoint(documentID: document.identifier, resourceOrder: resource.order)
|
||||
} else {
|
||||
await store.replace(document: document, resource: resource, analysis: analysis)
|
||||
}
|
||||
state = .indexing(completedResources: index + 1, totalResources: scopedResources.count)
|
||||
}
|
||||
state = .ready
|
||||
await record(stage: .indexing, startedAt: startedAt, count: scopedResources.count)
|
||||
} catch is CancellationError {
|
||||
state = .notStarted
|
||||
throw RDAIError.cancelled
|
||||
} catch let error as RDAIError {
|
||||
state = .failed(error)
|
||||
await record(stage: .indexing, startedAt: startedAt, errorCode: "rdai-error")
|
||||
throw error
|
||||
} catch {
|
||||
// Public errors must not expose paths, OCR text or SQLite details.
|
||||
let wrapped = RDAIError.indexingFailed("unexpected")
|
||||
state = .failed(wrapped)
|
||||
await record(stage: .indexing, startedAt: startedAt, errorCode: "indexing-failed")
|
||||
throw wrapped
|
||||
}
|
||||
}
|
||||
|
||||
public func retrieve(query: String, options: RDAIRetrievalOptions) async -> [RDAIRetrievalMatch] {
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
let allPassages = await storedPassages(for: document.identifier)
|
||||
let filtered = await filter(allPassages, scope: options.scope, documentIdentifier: document.identifier)
|
||||
var matches: [RDAIRetrievalMatch] = []
|
||||
for passage in filtered {
|
||||
let score = lexicalScore(query: query, text: passage.text)
|
||||
let semantic = await semanticScorer?.score(query: query, text: passage.text, languageCode: passage.languageCode)
|
||||
let finalScore = semantic.map { 0.6 * score + 0.4 * $0 } ?? score
|
||||
guard finalScore >= options.minimumScore else { continue }
|
||||
matches.append(RDAIRetrievalMatch(passage: passage, score: finalScore, lexicalScore: score, semanticScore: semantic))
|
||||
}
|
||||
return matches.sorted {
|
||||
if $0.score == $1.score { return $0.passage.order < $1.passage.order }
|
||||
return $0.score > $1.score
|
||||
}
|
||||
.prefix(options.maximumResults)
|
||||
.map { $0 }
|
||||
}
|
||||
|
||||
public func removeIndex() async {
|
||||
if let persistentStore { try? await persistentStore.remove(documentID: contentProvider.aiDocumentDescriptor().identifier) }
|
||||
else { await store.remove(documentIdentifier: contentProvider.aiDocumentDescriptor().identifier) }
|
||||
state = .notStarted
|
||||
}
|
||||
|
||||
public func pauseIndexing() {
|
||||
pauseRequested = true
|
||||
}
|
||||
|
||||
public func resumeIndexing() {
|
||||
pauseRequested = false
|
||||
resumeContinuation?.resume()
|
||||
resumeContinuation = nil
|
||||
}
|
||||
|
||||
public func removeAllAIData() async {
|
||||
if let persistentStore { try? await persistentStore.removeAll() }
|
||||
else { await store.remove(documentIdentifier: contentProvider.aiDocumentDescriptor().identifier) }
|
||||
state = .notStarted
|
||||
}
|
||||
|
||||
public func storageBytes() async -> Int64 {
|
||||
await persistentStore?.storageBytes() ?? 0
|
||||
}
|
||||
|
||||
public func showCitation(_ citation: RDAICitation) async throws {
|
||||
try await contentProvider.aiShowCitationHighlight(citation)
|
||||
}
|
||||
|
||||
public func clearCitationHighlight() {
|
||||
contentProvider.aiClearCitationHighlight()
|
||||
}
|
||||
|
||||
public func availability(for capability: RDAICapability, locale: Locale = .current) async -> RDAICapabilityAvailability {
|
||||
guard configuration.isAIEnabled else { return .unavailable(reason: .providerNotInstalled) }
|
||||
switch capability {
|
||||
case .languageAnalysis, .entityExtraction, .lexicalSearch: return .available
|
||||
case .semanticSearch:
|
||||
return semanticScorer == nil ? .degraded(reason: .embeddingAssetsUnavailable) : .available
|
||||
case .summarization, .questionAnswering, .characterRelationships:
|
||||
guard let generativeProvider else { return .degraded(reason: .providerNotInstalled) }
|
||||
return await generativeProvider.availability(for: capability, locale: locale)
|
||||
}
|
||||
}
|
||||
|
||||
public func summarize(scope: RDAIReadScope, length: RDAISummaryLength) async throws -> RDAISummary {
|
||||
guard configuration.isAIEnabled else { throw RDAIError.disabled }
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
let cacheKey = artifactKey(document: document, scope: scope, suffix: "summary|\(length.rawValue)")
|
||||
if configuration.storesGeneratedArtifacts,
|
||||
let persistentStore,
|
||||
let data = try? await persistentStore.artifact(documentID: document.identifier, key: cacheKey),
|
||||
let cached = try? JSONDecoder().decode(RDAISummary.self, from: data) { return cached }
|
||||
let passages = try await passages(for: scope, limit: length == .brief ? 2 : (length == .standard ? 4 : 6))
|
||||
let metadata = generationMetadata(provider: generativeProvider?.identifier ?? "extractive")
|
||||
if let generativeProvider,
|
||||
case .available = await generativeProvider.availability(for: .summarization, locale: .current),
|
||||
case let .summary(overview, statements) = try await generativeProvider.generate(.init(task: .summary(length), passages: passages, scope: scope)) {
|
||||
let validated = validatedStatements(statements, passages: passages)
|
||||
guard !validated.isEmpty else { throw RDAIError.insufficientEvidence }
|
||||
let result = RDAISummary(title: document.title, overview: overview, keyPoints: validated, citations: citations(for: passages), metadata: metadata)
|
||||
try? await saveArtifact(result, document: document, key: cacheKey)
|
||||
return result
|
||||
}
|
||||
let statements = passages.map { RDAISourcedStatement(text: $0.text, citationIdentifiers: [$0.id]) }
|
||||
let result = RDAISummary(title: document.title, overview: passages.first?.text ?? "暂无可总结的已读内容。", keyPoints: statements, citations: citations(for: passages), metadata: metadata)
|
||||
try? await saveArtifact(result, document: document, key: cacheKey)
|
||||
return result
|
||||
}
|
||||
|
||||
public func answer(question: String, scope: RDAIReadScope) async throws -> RDAIAnswer {
|
||||
guard configuration.isAIEnabled else { throw RDAIError.disabled }
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
let cacheKey = artifactKey(document: document, scope: scope, suffix: "answer|\(RDAIContentHasher.hash(question))")
|
||||
if configuration.storesGeneratedArtifacts,
|
||||
let persistentStore,
|
||||
let data = try? await persistentStore.artifact(documentID: document.identifier, key: cacheKey),
|
||||
let cached = try? JSONDecoder().decode(RDAIAnswer.self, from: data) { return cached }
|
||||
let matches = await retrieve(query: question, options: .init(maximumResults: configuration.maximumRetrievedPassages, scope: scope))
|
||||
let passages = matches.map(\.passage)
|
||||
let metadata = generationMetadata(provider: generativeProvider?.identifier ?? "extractive")
|
||||
guard !passages.isEmpty else { return RDAIAnswer(status: .insufficientEvidence, text: "在当前阅读范围内未找到足够证据。", statements: [], citations: [], metadata: metadata) }
|
||||
if let generativeProvider,
|
||||
case .available = await generativeProvider.availability(for: .questionAnswering, locale: .current),
|
||||
case let .answer(text, statements) = try await generativeProvider.generate(.init(task: .answer, userText: question, passages: passages, scope: scope)) {
|
||||
let validated = validatedStatements(statements, passages: passages)
|
||||
guard !validated.isEmpty else { return RDAIAnswer(status: .insufficientEvidence, text: "模型结果缺少可验证的原文证据。", statements: [], citations: [], metadata: metadata) }
|
||||
let result = RDAIAnswer(status: .answered, text: text, statements: validated, citations: citations(for: passages), metadata: metadata)
|
||||
try? await saveArtifact(result, document: document, key: cacheKey)
|
||||
return result
|
||||
}
|
||||
let statements = passages.map { RDAISourcedStatement(text: $0.text, citationIdentifiers: [$0.id]) }
|
||||
let result = RDAIAnswer(status: .answered, text: "以下内容与问题最相关：", statements: statements, citations: citations(for: passages), metadata: metadata)
|
||||
try? await saveArtifact(result, document: document, key: cacheKey)
|
||||
return result
|
||||
}
|
||||
|
||||
public func characters(scope: RDAIReadScope) async -> [RDAICharacter] {
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
let mentions = (await storedEntities(for: document.identifier)).filter { $0.kind == .person }
|
||||
let grouped = Dictionary(grouping: mentions, by: { canonicalCharacterName($0.normalizedName) })
|
||||
return grouped.compactMap { (name, values) -> RDAICharacter? in
|
||||
guard let first = values.first else { return nil }
|
||||
let evidence = values.prefix(3).map { mention in
|
||||
RDAICitation(passageIdentifier: mention.id, quote: mention.surfaceText, locator: mention.locator)
|
||||
}
|
||||
return RDAICharacter(id: RDAIContentHasher.hash(name), displayName: first.surfaceText, aliases: Array(Set(values.map(\.surfaceText))).sorted(), description: "在已读内容中出现 \(values.count) 次。", firstAppearance: evidence.first, evidence: evidence)
|
||||
}.sorted { $0.displayName < $1.displayName }
|
||||
}
|
||||
|
||||
/// Conservative local relationship extraction: entities must co-occur in
|
||||
/// one allowed passage. It never promotes co-occurrence to a fact.
|
||||
public func relationships(scope: RDAIReadScope) async -> [RDAIRelationship] {
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
let passages = await filter(await storedPassages(for: document.identifier), scope: scope, documentIdentifier: document.identifier)
|
||||
let mentions = (await storedEntities(for: document.identifier)).filter { $0.kind == .person }
|
||||
var evidenceByPair: [String: [RDAICitation]] = [:]
|
||||
for passage in passages {
|
||||
let names = Array(Set(mentions.filter {
|
||||
$0.locator.resourceIdentifier == passage.resourceIdentifier && $0.locator.textRange.intersects(passage.locator.textRange)
|
||||
}.map { canonicalCharacterName($0.normalizedName) })).sorted()
|
||||
guard names.count > 1 else { continue }
|
||||
for left in names.indices {
|
||||
for right in names.indices where right > left {
|
||||
let key = "\(names[left])|\(names[right])"
|
||||
evidenceByPair[key, default: []].append(citation(for: passage))
|
||||
}
|
||||
}
|
||||
}
|
||||
return evidenceByPair.compactMap { key, evidence in
|
||||
let names = key.split(separator: "|", maxSplits: 1).map(String.init)
|
||||
guard names.count == 2, !evidence.isEmpty else { return nil }
|
||||
return RDAIRelationship(id: RDAIContentHasher.hash(key), sourceCharacterIdentifier: RDAIContentHasher.hash(names[0]), targetCharacterIdentifier: RDAIContentHasher.hash(names[1]), label: "共同出现", status: .possible, evidence: Array(evidence.prefix(3)))
|
||||
}.sorted { $0.id < $1.id }
|
||||
}
|
||||
|
||||
private func passages(for scope: RDAIReadScope, limit: Int) async throws -> [RDAIPassage] {
|
||||
let document = contentProvider.aiDocumentDescriptor()
|
||||
let stored = await storedPassages(for: document.identifier)
|
||||
let all = await filter(stored, scope: scope, documentIdentifier: document.identifier)
|
||||
guard !all.isEmpty else { throw RDAIError.insufficientEvidence }
|
||||
return Array(all.sorted { $0.order < $1.order }.prefix(limit))
|
||||
}
|
||||
|
||||
private func citations(for passages: [RDAIPassage]) -> [RDAICitation] { passages.map(citation(for:)) }
|
||||
private func citation(for passage: RDAIPassage) -> RDAICitation { RDAICitation(passageIdentifier: passage.id, quote: passage.text, locator: passage.locator) }
|
||||
private func validatedStatements(_ statements: [RDAISourcedStatement], passages: [RDAIPassage]) -> [RDAISourcedStatement] {
|
||||
let identifiers = Set(passages.map(\.id))
|
||||
return statements.compactMap { statement in
|
||||
let citations = statement.citationIdentifiers.filter { identifiers.contains($0) }
|
||||
guard !citations.isEmpty, !statement.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil }
|
||||
return RDAISourcedStatement(id: statement.id, text: statement.text, citationIdentifiers: citations)
|
||||
}
|
||||
}
|
||||
private func canonicalCharacterName(_ value: String) -> String {
|
||||
let compact = value.unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) || (0x4E00...0x9FFF).contains($0.value) }
|
||||
var name = String(String.UnicodeScalarView(compact)).lowercased()
|
||||
// Only remove explicit honorific suffixes; ambiguous aliases remain
|
||||
// distinct so the UI does not silently merge two people.
|
||||
for suffix in ["先生", "女士", "小姐", "老师", "教授", "博士", "将军"] where name.hasSuffix(suffix) && name.count > suffix.count {
|
||||
name.removeLast(suffix.count)
|
||||
break
|
||||
}
|
||||
return name
|
||||
}
|
||||
private func generationMetadata(provider: String) -> RDAIGenerationMetadata { .init(providerIdentifier: provider, promptIdentifier: "rdai.evidence.v1", promptVersion: 1, scopeHash: RDAIContentHasher.hash(contentProvider.aiDocumentDescriptor().identifier.rawValue)) }
|
||||
private func artifactKey(document: RDAIDocumentDescriptor, scope: RDAIReadScope, suffix: String) -> String {
|
||||
let scopeData = (try? JSONEncoder().encode(scope)).map { String(data: $0, encoding: .utf8) } ?? ""
|
||||
return RDAIContentHasher.hash("\(document.contentRevision)|\(scopeData)|rdai.evidence.v1|\(generativeProvider?.identifier ?? "extractive")|\(suffix)")
|
||||
}
|
||||
private func saveArtifact<T: Encodable>(_ value: T, document: RDAIDocumentDescriptor, key: String) async throws {
|
||||
guard configuration.storesGeneratedArtifacts, let persistentStore else { return }
|
||||
try await persistentStore.saveArtifact(documentID: document.identifier, key: key, payload: JSONEncoder().encode(value))
|
||||
}
|
||||
private func record(stage: RDAIDiagnosticStage, startedAt: Date, errorCode: String? = nil, count: Int = 0) async {
|
||||
guard configuration.diagnosticsLevel != .disabled, let diagnostics else { return }
|
||||
await diagnostics.record(.init(documentHash: RDAIContentHasher.hash(contentProvider.aiDocumentDescriptor().identifier.rawValue), stage: stage, durationMilliseconds: Int(Date().timeIntervalSince(startedAt) * 1_000), errorCode: errorCode, count: count))
|
||||
}
|
||||
|
||||
private func prioritized(
|
||||
_ resources: [RDAIResourceDescriptor],
|
||||
preferred: RDAIResourceIdentifier?
|
||||
) -> [RDAIResourceDescriptor] {
|
||||
guard let preferred else { return resources }
|
||||
return resources.sorted { lhs, rhs in
|
||||
if lhs.identifier == preferred { return true }
|
||||
if rhs.identifier == preferred { return false }
|
||||
return lhs.order < rhs.order
|
||||
}
|
||||
}
|
||||
|
||||
private func waitIfPaused(completed: Int, total: Int) async {
|
||||
guard pauseRequested else { return }
|
||||
state = .paused(completedResources: completed, totalResources: total)
|
||||
await withTaskCancellationHandler(operation: {
|
||||
await withCheckedContinuation { continuation in
|
||||
resumeContinuation = continuation
|
||||
}
|
||||
}, onCancel: { [weak self] in
|
||||
Task { @MainActor in self?.resumeIndexing() }
|
||||
})
|
||||
state = .indexing(completedResources: completed, totalResources: total)
|
||||
}
|
||||
|
||||
private func resourcesAllowed(
|
||||
by scope: RDAIReadScope,
|
||||
document: RDAIDocumentDescriptor,
|
||||
resources: [RDAIResourceDescriptor]
|
||||
) async throws -> [RDAIResourceDescriptor] {
|
||||
guard !scope.includesWholeDocument else { return resources }
|
||||
guard let upperBound = scope.upperBound,
|
||||
upperBound.documentIdentifier == document.identifier,
|
||||
let boundary = resources.first(where: { $0.identifier == upperBound.resourceIdentifier }) else {
|
||||
return []
|
||||
}
|
||||
return resources.filter { $0.order <= boundary.order }
|
||||
}
|
||||
|
||||
private func limited(_ snapshot: RDAIResourceSnapshot, by scope: RDAIReadScope) -> RDAIResourceSnapshot {
|
||||
guard !scope.includesWholeDocument,
|
||||
let upperBound = scope.upperBound,
|
||||
upperBound.resourceIdentifier == snapshot.descriptor.identifier else {
|
||||
return snapshot
|
||||
}
|
||||
let length = min(max(upperBound.textRange.upperBound, 0), snapshot.sourceText.utf16.count)
|
||||
let source = (snapshot.sourceText as NSString).substring(to: length)
|
||||
let availableRange = RDAITextRange(location: 0, length: length)
|
||||
let runs = snapshot.locatorRuns.filter { $0.textRange.intersects(availableRange) }
|
||||
return RDAIResourceSnapshot(
|
||||
descriptor: snapshot.descriptor,
|
||||
sourceText: source,
|
||||
sourceHash: RDAIContentHasher.hash(source),
|
||||
locatorRuns: runs
|
||||
)
|
||||
}
|
||||
|
||||
private func filter(
|
||||
_ passages: [RDAIPassage],
|
||||
scope: RDAIReadScope,
|
||||
documentIdentifier: RDAIDocumentIdentifier
|
||||
) async -> [RDAIPassage] {
|
||||
guard !scope.includesWholeDocument else { return passages }
|
||||
guard let upperBound = scope.upperBound,
|
||||
upperBound.documentIdentifier == documentIdentifier,
|
||||
let boundaryOrder = await resourceOrder(for: upperBound.resourceIdentifier, documentIdentifier: documentIdentifier) else {
|
||||
return []
|
||||
}
|
||||
return await withTaskGroup(of: (RDAIPassage, Int?).self) { group in
|
||||
for passage in passages {
|
||||
group.addTask { [store] in
|
||||
let order = await self.resourceOrder(for: passage.resourceIdentifier, documentIdentifier: documentIdentifier)
|
||||
return (passage, order)
|
||||
}
|
||||
}
|
||||
var allowed: [RDAIPassage] = []
|
||||
for await (passage, order) in group {
|
||||
guard let order, order <= boundaryOrder else { continue }
|
||||
if passage.resourceIdentifier == upperBound.resourceIdentifier,
|
||||
passage.locator.textRange.location >= upperBound.textRange.upperBound {
|
||||
continue
|
||||
}
|
||||
allowed.append(passage)
|
||||
}
|
||||
return allowed
|
||||
}
|
||||
}
|
||||
|
||||
private func resourceOrder(for identifier: RDAIResourceIdentifier, documentIdentifier: RDAIDocumentIdentifier) async -> Int? {
|
||||
if let persistentStore { return try? await persistentStore.resourceOrder(documentID: documentIdentifier, resourceID: identifier) }
|
||||
return await store.resourceOrder(for: identifier, documentIdentifier: documentIdentifier)
|
||||
}
|
||||
|
||||
private func storedPassages(for identifier: RDAIDocumentIdentifier) async -> [RDAIPassage] {
|
||||
if let persistentStore { return (try? await persistentStore.passages(documentID: identifier)) ?? [] }
|
||||
return await store.passages(for: identifier)
|
||||
}
|
||||
|
||||
private func storedEntities(for identifier: RDAIDocumentIdentifier) async -> [RDAIEntityMention] {
|
||||
if let persistentStore { return (try? await persistentStore.entities(documentID: identifier)) ?? [] }
|
||||
return await store.entities(for: identifier)
|
||||
}
|
||||
|
||||
private func lexicalScore(query: String, text: String) -> Double {
|
||||
let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return 0 }
|
||||
let normalizedText = text.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current)
|
||||
let normalizedQuery = trimmed.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current)
|
||||
if normalizedText.localizedCaseInsensitiveContains(normalizedQuery) { return 1 }
|
||||
let terms = normalizedQuery.split(whereSeparator: { $0.isWhitespace || $0.isNewline }).map(String.init)
|
||||
guard !terms.isEmpty else { return 0 }
|
||||
let matches = terms.filter { normalizedText.localizedCaseInsensitiveContains($0) }.count
|
||||
return Double(matches) / Double(terms.count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import Foundation
|
||||
|
||||
enum RDAILocatorBuilder {
|
||||
static func makeLocator(
|
||||
document: RDAIDocumentDescriptor,
|
||||
snapshot: RDAIResourceSnapshot,
|
||||
range: RDAITextRange
|
||||
) -> RDAILocator? {
|
||||
let matchingRuns = snapshot.locatorRuns.filter { $0.textRange.intersects(range) }
|
||||
guard let first = matchingRuns.first else { return nil }
|
||||
let anchor = mergedAnchor(matchingRuns) ?? first.anchor
|
||||
return RDAILocator(
|
||||
documentIdentifier: document.identifier,
|
||||
resourceIdentifier: snapshot.descriptor.identifier,
|
||||
textRange: range,
|
||||
anchor: anchor,
|
||||
sourceHash: snapshot.sourceHash
|
||||
)
|
||||
}
|
||||
|
||||
private static func mergedAnchor(_ runs: [RDAILocatorRun]) -> RDAIAnchor? {
|
||||
guard !runs.isEmpty else { return nil }
|
||||
let anchors = runs.compactMap { run -> RDAIPDFAnchor? in
|
||||
if case .pdf(let anchor) = run.anchor { return anchor }
|
||||
return nil
|
||||
}
|
||||
guard anchors.count == runs.count,
|
||||
let first = anchors.first,
|
||||
anchors.allSatisfy({ $0.pageIndex == first.pageIndex && $0.textSource == first.textSource }) else {
|
||||
return nil
|
||||
}
|
||||
var rects: [RDAINormalizedRect] = []
|
||||
for rect in anchors.flatMap(\.rects) where !rects.contains(rect) {
|
||||
rects.append(rect)
|
||||
}
|
||||
let readingOrder = anchors.compactMap(\.readingOrder).min()
|
||||
return .pdf(RDAIPDFAnchor(
|
||||
pageIndex: first.pageIndex,
|
||||
rects: rects,
|
||||
textSource: first.textSource,
|
||||
readingOrder: readingOrder
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
import Foundation
|
||||
|
||||
public struct RDAIDocumentIdentifier: RawRepresentable, Codable, Hashable, Sendable {
|
||||
public let rawValue: String
|
||||
|
||||
public init(rawValue: String) {
|
||||
self.rawValue = rawValue
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIResourceIdentifier: RawRepresentable, Codable, Hashable, Sendable {
|
||||
public let rawValue: String
|
||||
|
||||
public init(rawValue: String) {
|
||||
self.rawValue = rawValue
|
||||
}
|
||||
}
|
||||
|
||||
/// UTF-16 offsets are shared by PDF text runs, EPUB search and speech progress.
|
||||
public struct RDAITextRange: Codable, Hashable, Sendable {
|
||||
public var location: Int
|
||||
public var length: Int
|
||||
|
||||
public init(location: Int, length: Int) {
|
||||
self.location = max(0, location)
|
||||
self.length = max(0, length)
|
||||
}
|
||||
|
||||
public var upperBound: Int { location + length }
|
||||
|
||||
public func intersects(_ other: RDAITextRange) -> Bool {
|
||||
location < other.upperBound && other.location < upperBound
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAINormalizedRect: Codable, Hashable, Sendable {
|
||||
public var x: Double
|
||||
public var y: Double
|
||||
public var width: Double
|
||||
public var height: Double
|
||||
|
||||
public init(x: Double, y: Double, width: Double, height: Double) {
|
||||
self.x = min(max(x, 0), 1)
|
||||
self.y = min(max(y, 0), 1)
|
||||
self.width = min(max(width, 0), 1)
|
||||
self.height = min(max(height, 0), 1)
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIPDFAnchor: Codable, Hashable, Sendable {
|
||||
public enum TextSource: String, Codable, Sendable {
|
||||
case native
|
||||
case ocr
|
||||
}
|
||||
|
||||
public var pageIndex: Int
|
||||
public var rects: [RDAINormalizedRect]
|
||||
public var textSource: TextSource
|
||||
public var readingOrder: Int?
|
||||
|
||||
public init(pageIndex: Int, rects: [RDAINormalizedRect], textSource: TextSource, readingOrder: Int? = nil) {
|
||||
self.pageIndex = max(0, pageIndex)
|
||||
self.rects = rects
|
||||
self.textSource = textSource
|
||||
self.readingOrder = readingOrder
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIEPUBAnchor: Codable, Hashable, Sendable {
|
||||
public var href: String
|
||||
public var cfi: String?
|
||||
public var rangeCFI: String?
|
||||
public var progression: Double?
|
||||
|
||||
public init(href: String, cfi: String? = nil, rangeCFI: String? = nil, progression: Double? = nil) {
|
||||
self.href = href
|
||||
self.cfi = cfi
|
||||
self.rangeCFI = rangeCFI
|
||||
self.progression = progression.map { min(max($0, 0), 1) }
|
||||
}
|
||||
}
|
||||
|
||||
public enum RDAIAnchor: Hashable, Sendable {
|
||||
case pdf(RDAIPDFAnchor)
|
||||
case epub(RDAIEPUBAnchor)
|
||||
}
|
||||
|
||||
extension RDAIAnchor: Codable {
|
||||
private enum CodingKeys: String, CodingKey { case type, pdf, epub }
|
||||
private enum Kind: String, Codable { case pdf, epub }
|
||||
|
||||
public init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
switch try container.decode(Kind.self, forKey: .type) {
|
||||
case .pdf:
|
||||
self = .pdf(try container.decode(RDAIPDFAnchor.self, forKey: .pdf))
|
||||
case .epub:
|
||||
self = .epub(try container.decode(RDAIEPUBAnchor.self, forKey: .epub))
|
||||
}
|
||||
}
|
||||
|
||||
public func encode(to encoder: Encoder) throws {
|
||||
var container = encoder.container(keyedBy: CodingKeys.self)
|
||||
switch self {
|
||||
case .pdf(let anchor):
|
||||
try container.encode(Kind.pdf, forKey: .type)
|
||||
try container.encode(anchor, forKey: .pdf)
|
||||
case .epub(let anchor):
|
||||
try container.encode(Kind.epub, forKey: .type)
|
||||
try container.encode(anchor, forKey: .epub)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAILocator: Codable, Hashable, Sendable {
|
||||
public var documentIdentifier: RDAIDocumentIdentifier
|
||||
public var resourceIdentifier: RDAIResourceIdentifier
|
||||
public var textRange: RDAITextRange
|
||||
public var anchor: RDAIAnchor
|
||||
public var sourceHash: String
|
||||
|
||||
public init(
|
||||
documentIdentifier: RDAIDocumentIdentifier,
|
||||
resourceIdentifier: RDAIResourceIdentifier,
|
||||
textRange: RDAITextRange,
|
||||
anchor: RDAIAnchor,
|
||||
sourceHash: String
|
||||
) {
|
||||
self.documentIdentifier = documentIdentifier
|
||||
self.resourceIdentifier = resourceIdentifier
|
||||
self.textRange = textRange
|
||||
self.anchor = anchor
|
||||
self.sourceHash = sourceHash
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAICitation: Codable, Hashable, Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let passageIdentifier: String
|
||||
public let quote: String
|
||||
public let locator: RDAILocator
|
||||
|
||||
public init(id: String = UUID().uuidString, passageIdentifier: String, quote: String, locator: RDAILocator) {
|
||||
self.id = id
|
||||
self.passageIdentifier = passageIdentifier
|
||||
self.quote = quote
|
||||
self.locator = locator
|
||||
}
|
||||
}
|
||||
|
||||
public enum RDAIDocumentFormat: String, Codable, Sendable {
|
||||
case pdf
|
||||
case epub
|
||||
}
|
||||
|
||||
public struct RDAIDocumentDescriptor: Codable, Equatable, Sendable {
|
||||
public let identifier: RDAIDocumentIdentifier
|
||||
public let title: String
|
||||
public let format: RDAIDocumentFormat
|
||||
public let contentRevision: String
|
||||
|
||||
public init(identifier: RDAIDocumentIdentifier, title: String, format: RDAIDocumentFormat, contentRevision: String) {
|
||||
self.identifier = identifier
|
||||
self.title = title
|
||||
self.format = format
|
||||
self.contentRevision = contentRevision
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIResourceDescriptor: Codable, Equatable, Sendable {
|
||||
public let identifier: RDAIResourceIdentifier
|
||||
public let title: String?
|
||||
public let order: Int
|
||||
public let estimatedUTF16Length: Int?
|
||||
|
||||
public init(identifier: RDAIResourceIdentifier, title: String? = nil, order: Int, estimatedUTF16Length: Int? = nil) {
|
||||
self.identifier = identifier
|
||||
self.title = title
|
||||
self.order = max(0, order)
|
||||
self.estimatedUTF16Length = estimatedUTF16Length
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAILocatorRun: Sendable {
|
||||
public let textRange: RDAITextRange
|
||||
public let anchor: RDAIAnchor
|
||||
|
||||
public init(textRange: RDAITextRange, anchor: RDAIAnchor) {
|
||||
self.textRange = textRange
|
||||
self.anchor = anchor
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIResourceSnapshot: Sendable {
|
||||
public let descriptor: RDAIResourceDescriptor
|
||||
public let sourceText: String
|
||||
public let sourceHash: String
|
||||
public let locatorRuns: [RDAILocatorRun]
|
||||
|
||||
public init(descriptor: RDAIResourceDescriptor, sourceText: String, sourceHash: String, locatorRuns: [RDAILocatorRun]) {
|
||||
self.descriptor = descriptor
|
||||
self.sourceText = sourceText
|
||||
self.sourceHash = sourceHash
|
||||
self.locatorRuns = locatorRuns
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
public protocol RDAIContentProvider: AnyObject {
|
||||
func aiDocumentDescriptor() -> RDAIDocumentDescriptor
|
||||
func aiResources() async throws -> [RDAIResourceDescriptor]
|
||||
func aiResourceSnapshot(for identifier: RDAIResourceIdentifier) async throws -> RDAIResourceSnapshot
|
||||
func aiNavigate(to locator: RDAILocator, animated: Bool) async throws
|
||||
func aiShowCitationHighlight(_ citation: RDAICitation) async throws
|
||||
func aiClearCitationHighlight()
|
||||
}
|
||||
|
||||
public struct RDAIReadScope: Codable, Equatable, Sendable {
|
||||
public let upperBound: RDAILocator?
|
||||
public let includesWholeDocument: Bool
|
||||
|
||||
public init(upperBound: RDAILocator? = nil, includesWholeDocument: Bool = false) {
|
||||
self.upperBound = upperBound
|
||||
self.includesWholeDocument = includesWholeDocument
|
||||
}
|
||||
|
||||
public static let wholeDocument = RDAIReadScope(includesWholeDocument: true)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
public protocol RDAIReadScopeProviding: AnyObject {
|
||||
func aiCurrentReadScope() -> RDAIReadScope
|
||||
}
|
||||
|
||||
public enum RDAIPassageKind: String, Codable, Sendable {
|
||||
case paragraph
|
||||
case unknown
|
||||
}
|
||||
|
||||
public struct RDAIPassage: Codable, Equatable, Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let documentIdentifier: RDAIDocumentIdentifier
|
||||
public let resourceIdentifier: RDAIResourceIdentifier
|
||||
public let text: String
|
||||
public let languageCode: String?
|
||||
public let kind: RDAIPassageKind
|
||||
public let locator: RDAILocator
|
||||
public let contentHash: String
|
||||
public let order: Int
|
||||
|
||||
public init(
|
||||
id: String,
|
||||
documentIdentifier: RDAIDocumentIdentifier,
|
||||
resourceIdentifier: RDAIResourceIdentifier,
|
||||
text: String,
|
||||
languageCode: String?,
|
||||
kind: RDAIPassageKind = .paragraph,
|
||||
locator: RDAILocator,
|
||||
contentHash: String,
|
||||
order: Int
|
||||
) {
|
||||
self.id = id
|
||||
self.documentIdentifier = documentIdentifier
|
||||
self.resourceIdentifier = resourceIdentifier
|
||||
self.text = text
|
||||
self.languageCode = languageCode
|
||||
self.kind = kind
|
||||
self.locator = locator
|
||||
self.contentHash = contentHash
|
||||
self.order = order
|
||||
}
|
||||
}
|
||||
|
||||
public enum RDAIEntityKind: String, Codable, Sendable {
|
||||
case person
|
||||
case place
|
||||
case organization
|
||||
case other
|
||||
}
|
||||
|
||||
public struct RDAIEntityMention: Codable, Equatable, Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let normalizedName: String
|
||||
public let surfaceText: String
|
||||
public let kind: RDAIEntityKind
|
||||
public let confidence: Double
|
||||
public let locator: RDAILocator
|
||||
|
||||
public init(id: String, normalizedName: String, surfaceText: String, kind: RDAIEntityKind, confidence: Double, locator: RDAILocator) {
|
||||
self.id = id
|
||||
self.normalizedName = normalizedName
|
||||
self.surfaceText = surfaceText
|
||||
self.kind = kind
|
||||
self.confidence = min(max(confidence, 0), 1)
|
||||
self.locator = locator
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAIAnalysisResult: Sendable {
|
||||
public let passages: [RDAIPassage]
|
||||
public let entityMentions: [RDAIEntityMention]
|
||||
|
||||
public init(passages: [RDAIPassage], entityMentions: [RDAIEntityMention]) {
|
||||
self.passages = passages
|
||||
self.entityMentions = entityMentions
|
||||
}
|
||||
}
|
||||
|
||||
public protocol RDAIAnalyzing: Sendable {
|
||||
func analyze(document: RDAIDocumentDescriptor, snapshot: RDAIResourceSnapshot) async -> RDAIAnalysisResult
|
||||
}
|
||||
|
||||
public protocol RDAISemanticScoring: Sendable {
|
||||
func score(query: String, text: String, languageCode: String?) async -> Double?
|
||||
}
|
||||
|
||||
public enum RDAIError: Error, Equatable, Sendable {
|
||||
case disabled
|
||||
case invalidDocument
|
||||
case resourceUnavailable(RDAIResourceIdentifier)
|
||||
case staleCitation
|
||||
case indexingFailed(String)
|
||||
case modelUnavailable(RDAIUnavailableReason)
|
||||
case insufficientEvidence
|
||||
case storageFailure(String)
|
||||
case cancelled
|
||||
}
|
||||
|
||||
public enum RDAICapability: String, Codable, Sendable {
|
||||
case languageAnalysis, entityExtraction, lexicalSearch, semanticSearch
|
||||
case summarization, questionAnswering, characterRelationships
|
||||
}
|
||||
|
||||
public enum RDAIUnavailableReason: Equatable, Sendable {
|
||||
case operatingSystemUnsupported, deviceNotEligible, appleIntelligenceNotEnabled
|
||||
case modelNotReady, languageUnsupported(String?), embeddingAssetsUnavailable
|
||||
case providerNotInstalled, unknown(String)
|
||||
}
|
||||
|
||||
public enum RDAICapabilityAvailability: Equatable, Sendable {
|
||||
case available
|
||||
case degraded(reason: RDAIUnavailableReason)
|
||||
case unavailable(reason: RDAIUnavailableReason)
|
||||
}
|
||||
|
||||
public protocol RDAICapabilityProviding: Sendable {
|
||||
func availability(for capability: RDAICapability, locale: Locale?) async -> RDAICapabilityAvailability
|
||||
}
|
||||
|
||||
public enum RDAISummaryLength: String, Codable, Sendable { case brief, standard, detailed }
|
||||
public enum RDAIAnswerStatus: String, Codable, Sendable { case answered, insufficientEvidence, unsupportedLanguage, unavailable }
|
||||
public enum RDAIRelationshipStatus: String, Codable, Sendable { case confirmed, possible, conflicting }
|
||||
|
||||
public struct RDAIGenerationMetadata: Codable, Sendable {
|
||||
public let providerIdentifier: String
|
||||
public let modelVersion: String?
|
||||
public let promptIdentifier: String
|
||||
public let promptVersion: Int
|
||||
public let generatedAt: Date
|
||||
public let scopeHash: String
|
||||
|
||||
public init(providerIdentifier: String, modelVersion: String? = nil, promptIdentifier: String, promptVersion: Int, generatedAt: Date = .init(), scopeHash: String) {
|
||||
self.providerIdentifier = providerIdentifier
|
||||
self.modelVersion = modelVersion
|
||||
self.promptIdentifier = promptIdentifier
|
||||
self.promptVersion = promptVersion
|
||||
self.generatedAt = generatedAt
|
||||
self.scopeHash = scopeHash
|
||||
}
|
||||
}
|
||||
|
||||
public struct RDAISourcedStatement: Codable, Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let text: String
|
||||
public let citationIdentifiers: [String]
|
||||
public init(id: String = UUID().uuidString, text: String, citationIdentifiers: [String]) { self.id = id; self.text = text; self.citationIdentifiers = citationIdentifiers }
|
||||
}
|
||||
|
||||
public struct RDAISummary: Codable, Sendable {
|
||||
public let title: String
|
||||
public let overview: String
|
||||
public let keyPoints: [RDAISourcedStatement]
|
||||
public let citations: [RDAICitation]
|
||||
public let metadata: RDAIGenerationMetadata
|
||||
}
|
||||
|
||||
public struct RDAIAnswer: Codable, Sendable {
|
||||
public let status: RDAIAnswerStatus
|
||||
public let text: String
|
||||
public let statements: [RDAISourcedStatement]
|
||||
public let citations: [RDAICitation]
|
||||
public let metadata: RDAIGenerationMetadata
|
||||
}
|
||||
|
||||
public struct RDAICharacter: Codable, Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let displayName: String
|
||||
public let aliases: [String]
|
||||
public let description: String
|
||||
public let firstAppearance: RDAICitation?
|
||||
public let evidence: [RDAICitation]
|
||||
}
|
||||
|
||||
public struct RDAIRelationship: Codable, Sendable, Identifiable {
|
||||
public let id: String
|
||||
public let sourceCharacterIdentifier: String
|
||||
public let targetCharacterIdentifier: String
|
||||
public let label: String
|
||||
public let status: RDAIRelationshipStatus
|
||||
public let evidence: [RDAICitation]
|
||||
}
|
||||
|
||||
public enum RDAIGenerationTask: Sendable { case summary(RDAISummaryLength), answer, characters, relationships }
|
||||
|
||||
public struct RDAIGenerationRequest: Sendable {
|
||||
public let task: RDAIGenerationTask
|
||||
public let userText: String?
|
||||
public let passages: [RDAIPassage]
|
||||
public let locale: Locale
|
||||
public let scope: RDAIReadScope
|
||||
public init(task: RDAIGenerationTask, userText: String? = nil, passages: [RDAIPassage], locale: Locale = .current, scope: RDAIReadScope) { self.task = task; self.userText = userText; self.passages = passages; self.locale = locale; self.scope = scope }
|
||||
}
|
||||
|
||||
public enum RDAIGeneratedArtifact: Sendable {
|
||||
case summary(overview: String, statements: [RDAISourcedStatement])
|
||||
case answer(text: String, statements: [RDAISourcedStatement])
|
||||
case characters([RDAICharacter])
|
||||
case relationships([RDAIRelationship])
|
||||
}
|
||||
|
||||
public protocol RDAIGenerativeProvider: RDAICapabilityProviding {
|
||||
var identifier: String { get }
|
||||
func generate(_ request: RDAIGenerationRequest) async throws -> RDAIGeneratedArtifact
|
||||
}
|
||||
|
||||
public enum RDAISpoilerPolicy: String, Codable, Sendable { case readContentOnly, wholeDocument }
|
||||
public enum RDAIDiagnosticsLevel: String, Codable, Sendable { case disabled, metadataOnly }
|
||||
|
||||
public struct RDAIReaderConfiguration: Sendable {
|
||||
public var isAIEnabled: Bool
|
||||
public var isLocalOnly: Bool
|
||||
public var spoilerPolicy: RDAISpoilerPolicy
|
||||
public var maximumRetrievedPassages: Int
|
||||
public var storesGeneratedArtifacts: Bool
|
||||
public var diagnosticsLevel: RDAIDiagnosticsLevel
|
||||
public init(isAIEnabled: Bool = true, isLocalOnly: Bool = true, spoilerPolicy: RDAISpoilerPolicy = .readContentOnly, maximumRetrievedPassages: Int = 4, storesGeneratedArtifacts: Bool = true, diagnosticsLevel: RDAIDiagnosticsLevel = .metadataOnly) { self.isAIEnabled = isAIEnabled; self.isLocalOnly = isLocalOnly; self.spoilerPolicy = spoilerPolicy; self.maximumRetrievedPassages = max(1, maximumRetrievedPassages); self.storesGeneratedArtifacts = storesGeneratedArtifacts; self.diagnosticsLevel = diagnosticsLevel }
|
||||
public static let `default` = RDAIReaderConfiguration()
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import Foundation
|
||||
import SQLite3
|
||||
|
||||
private let rdaiSQLiteTransient = unsafeBitCast(-1, to: sqlite3_destructor_type.self)
|
||||
|
||||
/// Durable local store. Each resource replacement is atomic, so interrupted
|
||||
/// indexing can safely resume from the first resource without a checkpoint.
|
||||
public actor RDAISQLiteIndexStore {
|
||||
private var database: OpaquePointer?
|
||||
private let encoder = JSONEncoder()
|
||||
private let decoder = JSONDecoder()
|
||||
|
||||
public init(directory: URL? = nil) throws {
|
||||
let base = directory ?? FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first!
|
||||
try FileManager.default.createDirectory(at: base, withIntermediateDirectories: true)
|
||||
let url = base.appendingPathComponent("RDAIReaderView.sqlite")
|
||||
guard sqlite3_open_v2(url.path, &database, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
|
||||
throw RDAIError.storageFailure("open")
|
||||
}
|
||||
try Self.configure(database: database)
|
||||
}
|
||||
|
||||
deinit { sqlite3_close(database) }
|
||||
|
||||
public func replace(document: RDAIDocumentDescriptor, resource: RDAIResourceDescriptor, analysis: RDAIAnalysisResult, sourceHash: String) throws {
|
||||
try transaction {
|
||||
try execute("INSERT INTO documents(id,title,format,revision) VALUES(?,?,?,?) ON CONFLICT(id) DO UPDATE SET title=excluded.title,format=excluded.format,revision=excluded.revision", [document.identifier.rawValue, document.title, document.format.rawValue, document.contentRevision])
|
||||
try execute("DELETE FROM resources WHERE document_id=? AND resource_id=?", [document.identifier.rawValue, resource.identifier.rawValue])
|
||||
try execute("INSERT INTO resources(document_id,resource_id,source_hash,payload) VALUES(?,?,?,?)", [document.identifier.rawValue, resource.identifier.rawValue, sourceHash, try json(resource)])
|
||||
try execute("DELETE FROM passages WHERE document_id=? AND resource_id=?", [document.identifier.rawValue, resource.identifier.rawValue])
|
||||
try execute("DELETE FROM passage_fts WHERE document_id=? AND resource_id=?", [document.identifier.rawValue, resource.identifier.rawValue])
|
||||
try execute("DELETE FROM entity_mentions WHERE document_id=? AND resource_id=?", [document.identifier.rawValue, resource.identifier.rawValue])
|
||||
for passage in analysis.passages {
|
||||
try execute("INSERT INTO passages(id,document_id,resource_id,content_hash,payload) VALUES(?,?,?,?,?)", [passage.id, document.identifier.rawValue, resource.identifier.rawValue, passage.contentHash, try json(passage)])
|
||||
try execute("INSERT INTO passage_fts(id,document_id,resource_id,text) VALUES(?,?,?,?)", [passage.id, document.identifier.rawValue, resource.identifier.rawValue, passage.text])
|
||||
}
|
||||
for mention in analysis.entityMentions {
|
||||
try execute("INSERT INTO entity_mentions(id,document_id,resource_id,normalized_name,payload) VALUES(?,?,?,?,?)", [mention.id, document.identifier.rawValue, resource.identifier.rawValue, mention.normalizedName, try json(mention)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func resourceHash(documentID: RDAIDocumentIdentifier, resourceID: RDAIResourceIdentifier) throws -> String? {
|
||||
try query("SELECT source_hash FROM resources WHERE document_id=? AND resource_id=?", [documentID.rawValue, resourceID.rawValue]).first
|
||||
}
|
||||
|
||||
public func passages(documentID: RDAIDocumentIdentifier) throws -> [RDAIPassage] {
|
||||
try query("SELECT payload FROM passages WHERE document_id=? ORDER BY rowid", [documentID.rawValue]).compactMap { try? decode(RDAIPassage.self, $0) }
|
||||
}
|
||||
|
||||
public func entities(documentID: RDAIDocumentIdentifier) throws -> [RDAIEntityMention] {
|
||||
try query("SELECT payload FROM entity_mentions WHERE document_id=? ORDER BY rowid", [documentID.rawValue]).compactMap { try? decode(RDAIEntityMention.self, $0) }
|
||||
}
|
||||
|
||||
public func resourceOrder(documentID: RDAIDocumentIdentifier, resourceID: RDAIResourceIdentifier) throws -> Int? {
|
||||
guard let payload = try query("SELECT payload FROM resources WHERE document_id=? AND resource_id=?", [documentID.rawValue, resourceID.rawValue]).first,
|
||||
let resource = try? decode(RDAIResourceDescriptor.self, payload) else { return nil }
|
||||
return resource.order
|
||||
}
|
||||
|
||||
public func remove(documentID: RDAIDocumentIdentifier) throws {
|
||||
try transaction {
|
||||
try execute("DELETE FROM passage_fts WHERE document_id=?", [documentID.rawValue])
|
||||
try execute("DELETE FROM documents WHERE id=?", [documentID.rawValue])
|
||||
}
|
||||
}
|
||||
public func removeAll() throws {
|
||||
try transaction {
|
||||
try execute("DELETE FROM documents")
|
||||
try execute("DELETE FROM passage_fts")
|
||||
}
|
||||
}
|
||||
|
||||
public func saveCheckpoint(documentID: RDAIDocumentIdentifier, resourceOrder: Int) throws {
|
||||
try execute("INSERT INTO index_jobs(document_id,completed_order,updated_at) VALUES(?,?,?) ON CONFLICT(document_id) DO UPDATE SET completed_order=excluded.completed_order,updated_at=excluded.updated_at", [documentID.rawValue, String(resourceOrder), String(Date().timeIntervalSince1970)])
|
||||
}
|
||||
|
||||
public func checkpoint(documentID: RDAIDocumentIdentifier) throws -> Int? {
|
||||
try query("SELECT completed_order FROM index_jobs WHERE document_id=?", [documentID.rawValue]).first.flatMap(Int.init)
|
||||
}
|
||||
|
||||
public func saveArtifact(documentID: RDAIDocumentIdentifier, key: String, payload: Data) throws {
|
||||
try execute("INSERT INTO artifacts(document_id,cache_key,payload,created_at) VALUES(?,?,?,?) ON CONFLICT(document_id,cache_key) DO UPDATE SET payload=excluded.payload,created_at=excluded.created_at", [documentID.rawValue, key, payload, String(Date().timeIntervalSince1970)])
|
||||
}
|
||||
|
||||
public func artifact(documentID: RDAIDocumentIdentifier, key: String) throws -> Data? {
|
||||
try queryData("SELECT payload FROM artifacts WHERE document_id=? AND cache_key=?", [documentID.rawValue, key]).first
|
||||
}
|
||||
|
||||
public func storageBytes() -> Int64 {
|
||||
guard let database,
|
||||
let path = sqlite3_db_filename(database, "main"),
|
||||
let attributes = try? FileManager.default.attributesOfItem(atPath: String(cString: path)),
|
||||
let size = attributes[.size] as? NSNumber else { return 0 }
|
||||
return size.int64Value
|
||||
}
|
||||
|
||||
private static func configure(database: OpaquePointer?) throws {
|
||||
try executeScript(database: database, sql: """
|
||||
CREATE TABLE IF NOT EXISTS schema_metadata(version INTEGER NOT NULL);
|
||||
INSERT INTO schema_metadata(version) SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM schema_metadata);
|
||||
CREATE TABLE IF NOT EXISTS documents(id TEXT PRIMARY KEY,title TEXT NOT NULL,format TEXT NOT NULL,revision TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS resources(document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,resource_id TEXT NOT NULL,source_hash TEXT NOT NULL,payload BLOB NOT NULL,PRIMARY KEY(document_id,resource_id));
|
||||
CREATE TABLE IF NOT EXISTS passages(id TEXT PRIMARY KEY,document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,resource_id TEXT NOT NULL,content_hash TEXT NOT NULL,payload BLOB NOT NULL);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS passage_fts USING fts5(id UNINDEXED,document_id UNINDEXED,resource_id UNINDEXED,text);
|
||||
CREATE TABLE IF NOT EXISTS entity_mentions(id TEXT PRIMARY KEY,document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,resource_id TEXT NOT NULL,normalized_name TEXT NOT NULL,payload BLOB NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS index_jobs(document_id TEXT PRIMARY KEY REFERENCES documents(id) ON DELETE CASCADE,completed_order INTEGER NOT NULL,updated_at REAL NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS artifacts(document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,cache_key TEXT NOT NULL,payload BLOB NOT NULL,created_at REAL NOT NULL,PRIMARY KEY(document_id,cache_key));
|
||||
""")
|
||||
}
|
||||
|
||||
private func transaction(_ body: () throws -> Void) throws { try execute("BEGIN IMMEDIATE"); do { try body(); try execute("COMMIT") } catch { try? execute("ROLLBACK"); throw error } }
|
||||
private static func executeScript(database: OpaquePointer?, sql: String) throws {
|
||||
var errorMessage: UnsafeMutablePointer<Int8>?
|
||||
guard sqlite3_exec(database, sql, nil, nil, &errorMessage) == SQLITE_OK else {
|
||||
defer { sqlite3_free(errorMessage) }
|
||||
throw RDAIError.storageFailure("migration")
|
||||
}
|
||||
}
|
||||
private func json<T: Encodable>(_ value: T) throws -> Data { try encoder.encode(value) }
|
||||
private func decode<T: Decodable>(_ type: T.Type, _ value: String) throws -> T { try decoder.decode(T.self, from: Data(value.utf8)) }
|
||||
private func queryData(_ sql: String, _ bindings: [String]) throws -> [Data] { var statement: OpaquePointer?; guard sqlite3_prepare_v2(database, sql, -1, &statement, nil) == SQLITE_OK else { throw RDAIError.storageFailure("prepare") }; defer { sqlite3_finalize(statement) }; for (index, value) in bindings.enumerated() { sqlite3_bind_text(statement, Int32(index + 1), value, -1, rdaiSQLiteTransient) }; var rows: [Data] = []; while sqlite3_step(statement) == SQLITE_ROW { let count = sqlite3_column_bytes(statement, 0); if let value = sqlite3_column_blob(statement, 0), count > 0 { rows.append(Data(bytes: value, count: Int(count))) } }; return rows }
|
||||
private func query(_ sql: String, _ bindings: [String]) throws -> [String] { var statement: OpaquePointer?; guard sqlite3_prepare_v2(database, sql, -1, &statement, nil) == SQLITE_OK else { throw RDAIError.storageFailure("prepare") }; defer { sqlite3_finalize(statement) }; for (index, value) in bindings.enumerated() { sqlite3_bind_text(statement, Int32(index + 1), value, -1, rdaiSQLiteTransient) }; var rows: [String] = []; while sqlite3_step(statement) == SQLITE_ROW { if let value = sqlite3_column_text(statement, 0) { rows.append(String(cString: value)) } }; return rows }
|
||||
private func execute(_ sql: String, _ bindings: [Any] = []) throws { var statement: OpaquePointer?; guard sqlite3_prepare_v2(database, sql, -1, &statement, nil) == SQLITE_OK else { throw RDAIError.storageFailure("prepare") }; defer { sqlite3_finalize(statement) }; for (index, value) in bindings.enumerated() { if let text = value as? String { sqlite3_bind_text(statement, Int32(index + 1), text, -1, rdaiSQLiteTransient) } else if let data = value as? Data { _ = data.withUnsafeBytes { sqlite3_bind_blob(statement, Int32(index + 1), $0.baseAddress, Int32(data.count), rdaiSQLiteTransient) } } }; guard sqlite3_step(statement) == SQLITE_DONE else { throw RDAIError.storageFailure("execute") } }
|
||||
}
|
||||
Reference in New Issue
Block a user