From e0593a4eced4af7be38a32a13449d78debb29d60 Mon Sep 17 00:00:00 2001 From: ori0o0p Date: Mon, 16 Feb 2026 03:26:09 +0900 Subject: [PATCH 1/3] feat(telemetry): aggregate execution outcome metrics --- Sources/Zero/Models/ZeroError.swift | 59 +++++++++++++++ Sources/Zero/Services/ExecutionService.swift | 74 ++++++++++++++++++- .../Zero/Services/ExecutionTelemetry.swift | 29 ++++++++ Tests/ZeroTests/ExecutionServiceTests.swift | 35 +++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 Sources/Zero/Services/ExecutionTelemetry.swift diff --git a/Sources/Zero/Models/ZeroError.swift b/Sources/Zero/Models/ZeroError.swift index 759b0fd..013820c 100644 --- a/Sources/Zero/Models/ZeroError.swift +++ b/Sources/Zero/Models/ZeroError.swift @@ -167,6 +167,65 @@ enum ZeroError: Error, Equatable { return true } } + + var telemetryCode: String { + switch self { + case .dockerNotInstalled: + return "docker_not_installed" + case .containerCreationFailed: + return "container_creation_failed" + case .containerExecutionFailed: + return "container_execution_failed" + case .containerNotFound: + return "container_not_found" + case .imageNotFound: + return "image_not_found" + case .gitNotInstalled: + return "git_not_installed" + case .gitCloneFailed: + return "git_clone_failed" + case .gitAuthenticationFailed: + return "git_authentication_failed" + case .invalidRepositoryURL: + return "invalid_repository_url" + case .githubAPIFailed: + return "github_api_failed" + case .githubAuthenticationFailed: + return "github_authentication_failed" + case .githubRateLimited: + return "github_rate_limited" + case .buildConfigurationFailed: + return "build_configuration_failed" + case .jdkNotFound: + return "jdk_not_found" + case .invalidJDKImage: + return "invalid_jdk_image" + case .sessionNotFound: + return "session_not_found" + case .sessionCreationFailed: + return "session_creation_failed" + case .sessionAlreadyExists: + return "session_already_exists" + case .fileNotFound: + return "file_not_found" + case .fileReadFailed: + return "file_read_failed" + case .fileWriteFailed: + return "file_write_failed" + case .keychainSaveFailed: + return "keychain_save_failed" + case .keychainLoadFailed: + return "keychain_load_failed" + case .keychainDeleteFailed: + return "keychain_delete_failed" + case .runtimeCommandFailed: + return "runtime_command_failed" + case .unknown: + return "unknown" + case .notImplemented: + return "not_implemented" + } + } } // MARK: - Result Extension diff --git a/Sources/Zero/Services/ExecutionService.swift b/Sources/Zero/Services/ExecutionService.swift index db9ebe1..798f93c 100644 --- a/Sources/Zero/Services/ExecutionService.swift +++ b/Sources/Zero/Services/ExecutionService.swift @@ -14,22 +14,33 @@ class ExecutionService: ObservableObject { let runProfileService: RunProfileService @Published var status: ExecutionStatus = .idle @Published var output: String = "" + @Published private(set) var telemetrySummary: ExecutionTelemetrySummary = .empty + + var telemetryEnabled = false + private var cancellationRequested = false private let installTimeoutSeconds: TimeInterval = 20 private let installMaxAttempts = 3 private let installRetryDelayNanoseconds: UInt64 = 300_000_000 + private let nowProvider: () -> Date + private var telemetryTotalDurationSeconds: TimeInterval = 0 + private var telemetryErrorCounts: [String: Int] = [:] init( dockerService: DockerServiceProtocol, buildConfigService: BuildConfigurationService = FileBasedBuildConfigurationService(), - runProfileService: RunProfileService = FileBasedRunProfileService() + runProfileService: RunProfileService = FileBasedRunProfileService(), + nowProvider: @escaping () -> Date = Date.init ) { self.dockerService = dockerService self.buildConfigService = buildConfigService self.runProfileService = runProfileService + self.nowProvider = nowProvider } func run(container: String, command: String) async { + let startedAt = nowProvider() + await MainActor.run { self.status = .running self.cancellationRequested = false @@ -52,8 +63,10 @@ class ExecutionService: ObservableObject { if self.cancellationRequested { self.status = .failed("Execution cancelled") self.output += "\n⏹️ Execution cancelled by user" + self.recordTelemetryIfEnabled(success: false, errorCode: "execution_cancelled", startedAt: startedAt, endedAt: self.nowProvider()) } else { self.status = .success + self.recordTelemetryIfEnabled(success: true, errorCode: nil, startedAt: startedAt, endedAt: self.nowProvider()) } } } catch { @@ -61,10 +74,17 @@ class ExecutionService: ObservableObject { if self.cancellationRequested { self.status = .failed("Execution cancelled") self.output += "\n⏹️ Execution cancelled by user" + self.recordTelemetryIfEnabled(success: false, errorCode: "execution_cancelled", startedAt: startedAt, endedAt: self.nowProvider()) } else { let userMessage = self.userMessage(for: error) self.status = .failed(userMessage) self.output += "\n❌ Error: \(userMessage)" + self.recordTelemetryIfEnabled( + success: false, + errorCode: self.telemetryErrorCode(for: error), + startedAt: startedAt, + endedAt: self.nowProvider() + ) } } } @@ -152,6 +172,58 @@ class ExecutionService: ObservableObject { .replacingOccurrences(of: "\\", with: "\\\\") .replacingOccurrences(of: "\"", with: "\\\"") } + + private func recordTelemetryIfEnabled(success: Bool, errorCode: String?, startedAt: Date, endedAt: Date) { + guard telemetryEnabled else { return } + + let elapsed = max(0, endedAt.timeIntervalSince(startedAt)) + telemetryTotalDurationSeconds += elapsed + + let totalRuns = telemetrySummary.totalRuns + 1 + let successfulRuns = telemetrySummary.successfulRuns + (success ? 1 : 0) + let failedRuns = telemetrySummary.failedRuns + (success ? 0 : 1) + + if let errorCode { + telemetryErrorCounts[errorCode, default: 0] += 1 + } + + let topErrorCodes = telemetryErrorCounts + .map { TelemetryErrorMetric(code: $0.key, count: $0.value) } + .sorted { lhs, rhs in + if lhs.count == rhs.count { + return lhs.code < rhs.code + } + return lhs.count > rhs.count + } + .prefix(3) + + let averageDurationSeconds = telemetryTotalDurationSeconds / Double(totalRuns) + + telemetrySummary = ExecutionTelemetrySummary( + totalRuns: totalRuns, + successfulRuns: successfulRuns, + failedRuns: failedRuns, + averageDurationSeconds: averageDurationSeconds, + topErrorCodes: Array(topErrorCodes) + ) + } + + private func telemetryErrorCode(for error: Error) -> String { + if let zeroError = error as? ZeroError { + return zeroError.telemetryCode + } + + if let urlError = error as? URLError { + return "url_error_\(urlError.errorCode)" + } + + if case let CommandRunnerError.commandFailed(_, _, exitCode, _) = error { + return "command_failed_\(exitCode)" + } + + let nsError = error as NSError + return "\(nsError.domain)#\(nsError.code)" + } func createJavaContainer(name: String) async throws -> String { let config = try buildConfigService.load() diff --git a/Sources/Zero/Services/ExecutionTelemetry.swift b/Sources/Zero/Services/ExecutionTelemetry.swift new file mode 100644 index 0000000..5c2ada1 --- /dev/null +++ b/Sources/Zero/Services/ExecutionTelemetry.swift @@ -0,0 +1,29 @@ +import Foundation + +struct TelemetryErrorMetric: Equatable, Identifiable { + let code: String + let count: Int + + var id: String { code } +} + +struct ExecutionTelemetrySummary: Equatable { + let totalRuns: Int + let successfulRuns: Int + let failedRuns: Int + let averageDurationSeconds: TimeInterval + let topErrorCodes: [TelemetryErrorMetric] + + static let empty = ExecutionTelemetrySummary( + totalRuns: 0, + successfulRuns: 0, + failedRuns: 0, + averageDurationSeconds: 0, + topErrorCodes: [] + ) + + var successRate: Double { + guard totalRuns > 0 else { return 0 } + return Double(successfulRuns) / Double(totalRuns) + } +} diff --git a/Tests/ZeroTests/ExecutionServiceTests.swift b/Tests/ZeroTests/ExecutionServiceTests.swift index bab896d..3ab4b13 100644 --- a/Tests/ZeroTests/ExecutionServiceTests.swift +++ b/Tests/ZeroTests/ExecutionServiceTests.swift @@ -282,6 +282,41 @@ final class ExecutionServiceTests: XCTestCase { XCTAssertTrue(installScript.contains("timeout 20 sh -lc")) } + func testRunRecordsTelemetrySummaryWhenEnabled() async { + // Given + service.telemetryEnabled = true + mockDocker.commandOutput = "ok" + + // When + await service.run(container: "test-container", command: "echo ok") + + mockDocker.executionError = ZeroError.runtimeCommandFailed( + userMessage: "Docker shell command failed.", + debugDetails: "runtime command failed" + ) + await service.run(container: "test-container", command: "echo fail") + + // Then + XCTAssertEqual(service.telemetrySummary.totalRuns, 2) + XCTAssertEqual(service.telemetrySummary.successfulRuns, 1) + XCTAssertEqual(service.telemetrySummary.failedRuns, 1) + XCTAssertEqual(service.telemetrySummary.topErrorCodes.first?.code, "runtime_command_failed") + } + + func testRunDoesNotRecordTelemetryWhenDisabled() async { + // Given + service.telemetryEnabled = false + mockDocker.commandOutput = "ok" + + // When + await service.run(container: "test-container", command: "echo ok") + + // Then + XCTAssertEqual(service.telemetrySummary.totalRuns, 0) + XCTAssertEqual(service.telemetrySummary.successfulRuns, 0) + XCTAssertEqual(service.telemetrySummary.failedRuns, 0) + } + func testSaveAndLoadRunProfileCommand() throws { // Given let repositoryURL = URL(string: "https://github.com/zero-ide/Zero.git")! From 90d678239b6483c7be2764aa8b0380e711ed2628 Mon Sep 17 00:00:00 2001 From: ori0o0p Date: Mon, 16 Feb 2026 03:26:14 +0900 Subject: [PATCH 2/3] feat(app-state): persist telemetry opt-in preference --- Sources/Zero/Constants.swift | 1 + Sources/Zero/Views/AppState.swift | 19 +++++++++++++++++++ Tests/ZeroTests/AppStateTests.swift | 20 ++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/Sources/Zero/Constants.swift b/Sources/Zero/Constants.swift index b6fd540..993594f 100644 --- a/Sources/Zero/Constants.swift +++ b/Sources/Zero/Constants.swift @@ -24,5 +24,6 @@ enum Constants { enum Preferences { static let selectedOrgLogin = "com.zero.ide.last_selected_org_login" + static let telemetryOptIn = "com.zero.ide.telemetry_opt_in" } } diff --git a/Sources/Zero/Views/AppState.swift b/Sources/Zero/Views/AppState.swift index bffbea2..55f5a18 100644 --- a/Sources/Zero/Views/AppState.swift +++ b/Sources/Zero/Views/AppState.swift @@ -22,6 +22,13 @@ class AppState: ObservableObject { persistSelectedOrgContextIfNeeded() } } + + @Published var telemetryOptIn: Bool = false { + didSet { + persistTelemetryOptInIfNeeded() + executionService.telemetryEnabled = telemetryOptIn + } + } // 페이지 크기 (테스트 시 조정 가능) var pageSize: Int = Constants.GitHub.pageSize @@ -70,6 +77,7 @@ class AppState: ObservableObject { private var pendingOAuthCodeVerifier: String? private var pendingOAuthRedirectURI: String? private var shouldPersistSelectedOrgContext = true + private var shouldPersistTelemetryOptIn = true var pendingOAuthStateForTesting: String? { pendingOAuthState @@ -94,6 +102,12 @@ class AppState: ObservableObject { self.persistedSessionDeleter = { session in try manager.deleteSession(session) } + + let storedTelemetryOptIn = UserDefaults.standard.bool(forKey: Constants.Preferences.telemetryOptIn) + shouldPersistTelemetryOptIn = false + telemetryOptIn = storedTelemetryOptIn + shouldPersistTelemetryOptIn = true + executionService.telemetryEnabled = storedTelemetryOptIn checkLoginStatus() } @@ -416,6 +430,11 @@ class AppState: ObservableObject { shouldPersistSelectedOrgContext = true } + private func persistTelemetryOptInIfNeeded() { + guard shouldPersistTelemetryOptIn else { return } + UserDefaults.standard.set(telemetryOptIn, forKey: Constants.Preferences.telemetryOptIn) + } + private func reconcileSelectedOrgContext() { if let currentOrg = selectedOrg, let matchingCurrentOrg = organizations.first(where: { $0.login == currentOrg.login }) { diff --git a/Tests/ZeroTests/AppStateTests.swift b/Tests/ZeroTests/AppStateTests.swift index e7c83c1..5dfca93 100644 --- a/Tests/ZeroTests/AppStateTests.swift +++ b/Tests/ZeroTests/AppStateTests.swift @@ -11,11 +11,13 @@ class AppStateTests: XCTestCase { // Clear keychain before test to ensure clean state try? KeychainHelper.standard.delete(service: "com.zero.ide", account: "github_token") UserDefaults.standard.removeObject(forKey: Constants.Preferences.selectedOrgLogin) + UserDefaults.standard.removeObject(forKey: Constants.Preferences.telemetryOptIn) appState = AppState() } override func tearDown() { UserDefaults.standard.removeObject(forKey: Constants.Preferences.selectedOrgLogin) + UserDefaults.standard.removeObject(forKey: Constants.Preferences.telemetryOptIn) appState = nil super.tearDown() } @@ -495,6 +497,24 @@ class AppStateTests: XCTestCase { // Then XCTAssertEqual(selectedName, "selected-repo") } + + func testTelemetryOptInPersistsAcrossAppStateInstances() { + // Given + appState.telemetryOptIn = true + + // When + let reloadedState = AppState() + + // Then + XCTAssertTrue(reloadedState.telemetryOptIn) + + // When + reloadedState.telemetryOptIn = false + let reloadedAgain = AppState() + + // Then + XCTAssertFalse(reloadedAgain.telemetryOptIn) + } } // MARK: - Mocks From db610f5577c0582cfbb8f2d76b8a6bdee863d37a Mon Sep 17 00:00:00 2001 From: ori0o0p Date: Mon, 16 Feb 2026 03:26:21 +0900 Subject: [PATCH 3/3] feat(diagnostics): add lightweight telemetry panel --- Sources/Zero/Views/DiagnosticsView.swift | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Sources/Zero/Views/DiagnosticsView.swift b/Sources/Zero/Views/DiagnosticsView.swift index 7674f26..2e54ab6 100644 --- a/Sources/Zero/Views/DiagnosticsView.swift +++ b/Sources/Zero/Views/DiagnosticsView.swift @@ -52,6 +52,33 @@ struct DiagnosticsView: View { .foregroundStyle(exportMessage.hasPrefix("Failed") ? Color.red : Color.secondary) } + VStack(alignment: .leading, spacing: 10) { + Toggle("Enable lightweight telemetry", isOn: $appState.telemetryOptIn) + + let summary = appState.executionService.telemetrySummary + Text("Success Rate: \(formatPercent(summary.successRate))") + .font(.caption) + .foregroundStyle(.secondary) + Text("Average Runtime: \(formatDuration(summary.averageDurationSeconds))") + .font(.caption) + .foregroundStyle(.secondary) + + if summary.topErrorCodes.isEmpty { + Text("Top Error Codes: none") + .font(.caption) + .foregroundStyle(.secondary) + } else { + ForEach(summary.topErrorCodes) { metric in + Text("Error: \(metric.code) (\(metric.count))") + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + .padding(12) + .background(Color(nsColor: .controlBackgroundColor)) + .cornerRadius(8) + if let snapshot { VStack(alignment: .leading, spacing: 10) { Text("Docker") @@ -178,6 +205,14 @@ struct DiagnosticsView: View { } } } + + private func formatPercent(_ value: Double) -> String { + String(format: "%.1f%%", value * 100) + } + + private func formatDuration(_ value: TimeInterval) -> String { + String(format: "%.2fs", value) + } } private struct DiagnosticsStatusRow: View {