From 2040bc17b5e0d5d44488e758c99eb8e739d43c92 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Mon, 8 Jun 2026 23:03:20 -0700 Subject: [PATCH 1/4] feat(apple-ai): add support for Apple Foundation Models integration --- .../FirebaseAIExample/ContentView.swift | 6 + .../AppleFoundationModels/Models/Models.swift | 95 +++++ .../Screens/AppleAIScreen.swift | 346 ++++++++++++++++++ .../Tools/FindLocalPlacesTool.swift | 91 +++++ .../ViewModels/AppleAIViewModel.swift | 211 +++++++++++ .../Shared/Models/Sample.swift | 6 + .../Shared/Util/MLErrorHelper.swift | 72 ++++ .../Shared/Views/ErrorDetailsView.swift | 13 +- 8 files changed, 838 insertions(+), 2 deletions(-) create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 62808c579..103b4d69c 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -112,6 +112,12 @@ struct ContentView: View { GroundingScreen(backendType: selectedBackend, sample: sample) case "LiveScreen": LiveScreen(backendType: selectedBackend, sample: sample) + case "AppleAIScreen": + if #available(iOS 27.0, *) { + AppleAIScreen() + } else { + Text("Apple Intelligence is only available on iOS 27 or newer.") + } default: EmptyView() } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift new file mode 100644 index 000000000..543a05ccf --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +@available(iOS 26.0, *) +@Generable +public struct Itinerary: Equatable, Codable { + @Guide(description: "An exciting name for the trip.") + public let title: String + + @Guide(description: "The name of the destination.") + public let destinationName: String + + @Guide(description: "A brief, catchy description of the trip.") + public let description: String + + @Guide(description: "An explanation of why this plan fits the user's request.") + public let rationale: String + + @Guide(description: "A list of day plans.") + public let days: [DayPlan] +} + +@available(iOS 26.0, *) +@Generable +public struct DayPlan: Equatable, Codable { + @Guide(description: "A unique title for this day.") + public let title: String + + public let subtitle: String + + @Guide(description: "A list of activities planned for this day.") + public let activities: [Activity] +} + +@available(iOS 26.0, *) +@Generable +public struct Activity: Equatable, Codable { + public let type: ActivityKind + public let title: String + public let description: String + public let latitude: Double? + public let longitude: Double? +} + +@available(iOS 26.0, *) +@Generable +public enum ActivityKind: String, Codable { + case sightseeing + case foodAndDining + case shopping + case hotelAndLodging + + public var displayName: String { + switch self { + case .sightseeing: return "Sightseeing" + case .foodAndDining: return "Food & Dining" + case .shopping: return "Shopping" + case .hotelAndLodging: return "Hotel & Lodging" + } + } +} + +@available(iOS 26.0, *) +@Generable +public struct IdentifiedObject: Equatable, Codable { + @Guide(description: "The name of the primary object or landmark detected.") + public let name: String + + @Guide(description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing).") + public let category: String + + @Guide(description: "A short, 2-sentence description of the object and interesting facts.") + public let description: String +} + +@available(iOS 26.0, *) +@Generable +public struct TextSummary: Equatable, Codable { + @Guide(description: "A list of exactly 2 key summary points.") + public let summaryPoints: [String] +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift new file mode 100644 index 000000000..4673e5123 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -0,0 +1,346 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI +import PhotosUI +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +struct AppleAIScreen: View { + @StateObject private var viewModel = AppleAIViewModel() + @State private var selectedTab = 0 + @State private var photosPickerItem: PhotosPickerItem? = nil + @State private var presentErrorDetails = false + + var body: some View { + ZStack { + VStack(spacing: 0) { + // Segmented Control + Picker("Feature", selection: $selectedTab) { + Text("Hybrid AI").tag(0) + Text("Planner").tag(1) + Text("Vision ID").tag(2) + } + .pickerStyle(.segmented) + .padding() + .background(Color(.systemBackground)) + + Divider() + + // Content Views + ScrollView { + VStack(spacing: 20) { + if selectedTab == 0 { + hybridAIView + } else if selectedTab == 1 { + plannerView + } else { + visionIDView + } + } + .padding() + } + } + + if viewModel.inProgress { + ProgressOverlay() + } + } + .background(Color(.systemGroupedBackground)) + .navigationTitle("Apple Intelligence") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + if viewModel.inProgress { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Stop") { + viewModel.stopActiveTask() + } + } + } + } + .sheet(isPresented: $presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .onChange(of: viewModel.error != nil) { oldValue, newValue in + if newValue { + presentErrorDetails = true + } + } + } + + // MARK: - Subviews + + // Feature 1: Hybrid AI + private var hybridAIView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 8) { + Text("Input Text") + .font(.headline) + TextEditor(text: $viewModel.inputText) + .frame(height: 120) + .padding(4) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(8) + } + + Button(action: { + Task { + await viewModel.runSummarization() + } + }) { + HStack { + Spacer() + Image(systemName: "sparkles") + Text("Summarize with Apple SDK") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let summary = viewModel.outputSummary { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Summary Points") + .font(.headline) + Spacer() + + // Badge indicating where it was executed + HStack(spacing: 4) { + Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") + Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") + } + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) + .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) + .cornerRadius(8) + } + + VStack(alignment: .leading, spacing: 8) { + ForEach(summary.summaryPoints, id: \.self) { point in + HStack(alignment: .top, spacing: 6) { + Text("•") + .bold() + Text(point) + .font(.subheadline) + } + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity.combined(with: .slide)) + } + } + } + + // Feature 2: Planner View + private var plannerView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(spacing: 12) { + HStack { + Text("Destination") + .frame(width: 100, alignment: .leading) + TextField("e.g. Paris, Tokyo", text: $viewModel.destination) + .textFieldStyle(.roundedBorder) + } + HStack { + Text("Interests") + .frame(width: 100, alignment: .leading) + TextField("e.g. art, cafes, parks", text: $viewModel.interests) + .textFieldStyle(.roundedBorder) + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + + Button(action: { + Task { + await viewModel.generateItinerary() + } + }) { + HStack { + Spacer() + Image(systemName: "map.fill") + Text("Create Itinerary Plan") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let itinerary = viewModel.itinerary { + VStack(alignment: .leading, spacing: 12) { + Text(itinerary.title ?? "Generating plan...") + .font(.title3) + .bold() + + if let desc = itinerary.description { + Text(desc) + .font(.subheadline) + .foregroundColor(.secondary) + } + + if let rationale = itinerary.rationale { + Text("Rationale: \(rationale)") + .font(.caption) + .padding() + .background(Color.blue.opacity(0.1)) + .cornerRadius(8) + } + + // Day Plans + if let days = itinerary.days { + ForEach(days, id: \.title) { day in + VStack(alignment: .leading, spacing: 8) { + Text(day.title ?? "Day Plan") + .font(.headline) + .padding(.top, 4) + + if let activities = day.activities { + ForEach(activities, id: \.title) { activity in + HStack(alignment: .top, spacing: 12) { + Image(systemName: getSymbol(for: activity.type)) + .foregroundColor(.blue) + .frame(width: 24, height: 24) + .background(Color.blue.opacity(0.1)) + .cornerRadius(6) + + VStack(alignment: .leading, spacing: 2) { + Text(activity.title ?? "Activity") + .font(.subheadline) + .bold() + if let desc = activity.description { + Text(desc) + .font(.caption) + .foregroundColor(.secondary) + } + if let lat = activity.latitude, let lon = activity.longitude { + Text(String(format: "Location: %.4f, %.4f", lat, lon)) + .font(.system(.caption, design: .monospaced)) + .foregroundColor(.gray) + } + } + } + .padding(.vertical, 4) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + } + + + } + } + } + } + + // Feature 3: Vision ID View + private var visionIDView: some View { + VStack(alignment: .leading, spacing: 16) { + Text("Select or Snap a Photo to Identify") + .font(.headline) + + PhotosPicker(selection: $photosPickerItem, matching: .images) { + VStack(spacing: 12) { + if let image = viewModel.selectedImage { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxHeight: 200) + .cornerRadius(12) + } else { + VStack(spacing: 8) { + Image(systemName: "photo.badge.plus") + .font(.system(size: 40)) + Text("Select an Image") + } + .frame(maxWidth: .infinity) + .frame(height: 180) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(12) + } + } + } + .onChange(of: photosPickerItem) { oldItem, newItem in + Task { + if let data = try? await newItem?.loadTransferable(type: Data.self), + let image = UIImage(data: data) { + viewModel.selectedImage = image + await viewModel.identifySelectedImage() + } + } + } + + if let identified = viewModel.identifiedObject { + VStack(alignment: .leading, spacing: 12) { + Text("Identification Result") + .font(.headline) + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Name:") + .bold() + Text(identified.name) + } + HStack { + Text("Category:") + .bold() + Text(identified.category) + } + VStack(alignment: .leading, spacing: 4) { + Text("Description:") + .bold() + Text(identified.description) + .foregroundColor(.secondary) + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity) + } + } + } + + // Helper to pick icons for activity kinds + private func getSymbol(for kind: ActivityKind?) -> String { + guard let kind = kind else { return "info.circle" } + switch kind { + case .sightseeing: return "binoculars.fill" + case .foodAndDining: return "fork.knife" + case .shopping: return "bag.fill" + case .hotelAndLodging: return "bed.double.fill" + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift new file mode 100644 index 000000000..716aec456 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -0,0 +1,91 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels +import FirebaseAILogic + +@available(iOS 26.0, *) +@Generable +public struct PlacesResponse: Codable { + public let places: [String] +} + +@available(iOS 26.0, *) +@Generable +public struct ToolResult: Codable { + public let summary: String +} + +@available(iOS 26.0, *) +public final class FindLocalPlacesTool { + public let name = "findLocalPlaces" + public let description = "Finds points of interest and local businesses matching a query for a specified destination." + + public init() {} + + @available(iOS 26.0, *) + @Generable + public struct Arguments { + @Guide(description: "The destination city/place to look up for (e.g. 'Paris', 'New York').") + public let destination: String + + @Guide(description: "The category or query to search (e.g. 'hotels', 'Italian restaurants', 'museums').") + public let category: String + } + + public func call(arguments: Arguments) async throws -> ToolResult { + print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") + + // Setup Firebase AI with Google Search grounding + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let session = ai.generativeModelSession( + model: "gemini-3.1-flash-lite", + tools: [Tool.googleSearch()] + ) + + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Search to make sure the places are real and currently open." + + do { + let response = try await session.respond( + to: prompt, + generating: PlacesResponse.self + ) + + let results = response.content.places + print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") + + return ToolResult(summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))") + } catch { + print("FindLocalPlacesTool error: \(error.localizedDescription)") + throw error + } + } +} + +// Struct wrapper to conform to FoundationModels.Tool AND ToolRepresentable +@available(iOS 26.0, *) +public struct FindLocalPlacesToolWrapper: FoundationModels.Tool, ToolRepresentable, @unchecked Sendable { + let tool: FindLocalPlacesTool + + public var name: String { tool.name } + public var description: String { tool.description } + + public typealias Arguments = FindLocalPlacesTool.Arguments + public typealias Output = ToolResult + + public func call(arguments: Arguments) async throws -> ToolResult { + try await tool.call(arguments: arguments) + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift new file mode 100644 index 000000000..31d6b38f6 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -0,0 +1,211 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +@MainActor +public final class AppleAIViewModel: ObservableObject { + // Shared state + @Published public var inProgress = false + @Published public var error: Error? + + // Feature 1: Hybrid Summary/Translation + @Published public var inputText: String = "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." + @Published public var outputSummary: TextSummary? + @Published public var isUsingLocalModel: Bool = false + + // Feature 2: Smart Planner + @Published public var destination: String = "San Francisco" + @Published public var interests: String = "tech history, good coffee, bay views" + @Published public var itinerary: Itinerary.PartiallyGenerated? + + // Feature 3: Visual Identifier + @Published public var selectedImage: UIImage? + @Published public var identifiedObject: IdentifiedObject? + + private var activeTask: Task? + + public init() {} + + public func stopActiveTask() { + activeTask?.cancel() + activeTask = nil + inProgress = false + } + + // MARK: - Feature 1: Hybrid Summary/Translation + public func runSummarization() async { + stopActiveTask() + inProgress = true + error = nil + outputSummary = nil + + activeTask = Task { + let availability = SystemLanguageModel.default.availability + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + // Try local model first if it reports available + if availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + inProgress = false + return + } catch { + // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) + if error.isMLAssetUnavailable { + print("Local ML assets unavailable. Falling back to cloud model...") + } else { + print("Local model failed: \(error.localizedDescription). Falling back to cloud model...") + } + } + } + + // Fallback to cloud model + isUsingLocalModel = false + do { + let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + } catch { + if !Task.isCancelled { + // If even the cloud model via LanguageModelSession fails (likely due to missing guardrail assets), + // we show a descriptive error message. + if let mlMessage = error.mlAssetErrorMessage { + print("ML Asset Error detected in cloud fallback: \(mlMessage)") + // We can either set the error here or try one last direct fallback. + // Let's set a custom error that wraps the message. + self.error = NSError(domain: "FirebaseAIExample", + code: 1, + userInfo: [NSLocalizedDescriptionKey: mlMessage]) + } else { + self.error = error + } + } + } + inProgress = false + } + } + + // MARK: - Feature 2: Smart Planner + public func generateItinerary() async { + stopActiveTask() + inProgress = true + error = nil + itinerary = nil + + activeTask = Task { + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + let model = ai.geminiLanguageModel( + name: "gemini-3.5-flash" + ) + + let session = LanguageModelSession( + model: model, + tools: [localPlacesTool], + instructions: Instructions { + "Your job is to create a structured 1-day itinerary for the user." + "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." + } + ) + + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations." + + let stream = session.streamResponse( + to: promptText, + generating: Itinerary.self + ) + + do { + for try await response in stream { + if Task.isCancelled { break } + self.itinerary = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + inProgress = false + } + } + + // MARK: - Feature 3: Visual Identifier + public func identifySelectedImage() async { + guard let image = selectedImage else { return } + stopActiveTask() + inProgress = true + error = nil + identifiedObject = nil + + activeTask = Task { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + + guard let cgImage = image.cgImage else { + inProgress = false + return + } + + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "You are a visual object identifier." + } + ) + + do { + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + Attachment(cgImage) + } + + if !Task.isCancelled { + self.identifiedObject = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + inProgress = false + } + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift index 384f6f4cd..4f0129345 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift @@ -309,6 +309,12 @@ extension Sample { ), tip: InlineTip(text: "Try asking the model to change the background color"), ), + Sample( + title: "Apple Foundation Models", + description: "Demonstrates how to integrate Firebase AI Logic (Gemini) with Apple's Foundation Models framework (Apple Intelligence).", + useCases: [.text, .image, .functionCalling], + navRoute: "AppleAIScreen" + ), ] public static var sample = samples[0] diff --git a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift new file mode 100644 index 000000000..bfaf543f7 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +extension Error { + /// Returns true if this error indicates that required ML assets (like safety guardrails) + /// are missing from the device or simulator. + public var isMLAssetUnavailable: Bool { + // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable + if #available(iOS 26.0, *) { + if let genError = self as? LanguageModelSession.GenerationError { + if case .assetsUnavailable = genError { + return true + } + } + } + + let nsError = self as NSError + + // 2. Check for the specific SensitiveContentAnalysisML error (Code 15) + if nsError.domain == "com.apple.SensitiveContentAnalysisML" && nsError.code == 15 { + return true + } + + // 3. Check for underlying GenerativeFunctionsFoundation error (Code 1020000) + // This is often found in the UserInfo of higher-level errors. + if nsError.domain == "com.apple.GenerativeFunctionsFoundation.GenerativeError" && nsError.code == 1020000 { + return true + } + + // 4. Recursive check for underlying errors + if let underlying = nsError.userInfo[NSUnderlyingErrorKey] as? Error { + return underlying.isMLAssetUnavailable + } + + return false + } + + /// Returns a user-friendly message for ML asset errors, particularly for simulators. + public var mlAssetErrorMessage: String? { + guard isMLAssetUnavailable else { return nil } + + #if targetEnvironment(simulator) + return """ + Apple Intelligence assets are missing in this simulator. + + To fix this: + 1. Open Settings in the simulator. + 2. Go to Apple Intelligence & Siri. + 3. Toggle Apple Intelligence ON. + 4. Wait for assets to download (check the status in Settings). + + Alternatively, use a physical device with Apple Intelligence support. + """ + #else + return "Apple Intelligence assets are not yet ready on this device. Please ensure Apple Intelligence is enabled in Settings and that model assets have finished downloading." + #endif + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift index cd1e3f60e..7ba2b1788 100644 --- a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift +++ b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift @@ -148,11 +148,20 @@ struct ErrorDetailsView: View { default: Section("Error Type") { - Text("Some other error") + if error.isMLAssetUnavailable { + Text("Apple Intelligence Assets Missing") + .foregroundColor(.orange) + } else { + Text("Some other error") + } } Section("Details") { - SubtitleFormRow(title: "Error description", value: error.localizedDescription) + if let mlMessage = error.mlAssetErrorMessage { + SubtitleFormRow(title: "How to fix", value: mlMessage) + } else { + SubtitleFormRow(title: "Error description", value: error.localizedDescription) + } } } } From 8c2d9cf45d90cae81b23b69cdb0701d36d0d33a8 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Tue, 9 Jun 2026 06:41:04 -0700 Subject: [PATCH 2/4] refactor(apple-ai): clean up tasks and session handling in AppleAIViewModel --- .../ViewModels/AppleAIViewModel.swift | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 31d6b38f6..8bc81bd83 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -57,8 +57,13 @@ public final class AppleAIViewModel: ObservableObject { outputSummary = nil activeTask = Task { + defer { self.inProgress = false } + + let instructions = Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + let availability = SystemLanguageModel.default.availability - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) // Try local model first if it reports available if availability == .available { @@ -66,15 +71,12 @@ public final class AppleAIViewModel: ObservableObject { do { let session = LanguageModelSession( model: SystemLanguageModel.default, - instructions: Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } + instructions: instructions ) let response = try await session.respond(to: inputText, generating: TextSummary.self) if !Task.isCancelled { self.outputSummary = response.content } - inProgress = false return } catch { // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) @@ -89,12 +91,11 @@ public final class AppleAIViewModel: ObservableObject { // Fallback to cloud model isUsingLocalModel = false do { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") let session = LanguageModelSession( model: model, - instructions: Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } + instructions: instructions ) let response = try await session.respond(to: inputText, generating: TextSummary.self) if !Task.isCancelled { @@ -116,7 +117,6 @@ public final class AppleAIViewModel: ObservableObject { } } } - inProgress = false } } @@ -128,6 +128,8 @@ public final class AppleAIViewModel: ObservableObject { itinerary = nil activeTask = Task { + defer { self.inProgress = false } + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) @@ -161,7 +163,6 @@ public final class AppleAIViewModel: ObservableObject { self.error = error } } - inProgress = false } } @@ -174,14 +175,13 @@ public final class AppleAIViewModel: ObservableObject { identifiedObject = nil activeTask = Task { + defer { self.inProgress = false } + + guard let cgImage = image.cgImage else { return } + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - guard let cgImage = image.cgImage else { - inProgress = false - return - } - let session = LanguageModelSession( model: model, instructions: Instructions { @@ -205,7 +205,6 @@ public final class AppleAIViewModel: ObservableObject { self.error = error } } - inProgress = false } } } From 5bbf426af91da791a5fa51940341facc7a9e36e2 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Tue, 9 Jun 2026 06:45:52 -0700 Subject: [PATCH 3/4] feat(apple-ai): add support for Google Maps grounding attributions in planner --- .../AppleFoundationModels/Models/Models.swift | 10 +++++++ .../Screens/AppleAIScreen.swift | 26 ++++++++++++++++++- .../Tools/FindLocalPlacesTool.swift | 21 ++++++++++++--- .../ViewModels/AppleAIViewModel.swift | 2 +- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift index 543a05ccf..821cef536 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -32,6 +32,16 @@ public struct Itinerary: Equatable, Codable { @Guide(description: "A list of day plans.") public let days: [DayPlan] + + @Guide(description: "Any source attributions or links for the recommended places.") + public let attributions: [Attribution]? +} + +@available(iOS 26.0, *) +@Generable +public struct Attribution: Equatable, Codable { + public let title: String + public let url: String } @available(iOS 26.0, *) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift index 4673e5123..e942051f4 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -257,7 +257,31 @@ struct AppleAIScreen: View { } } - + if let attributions = itinerary.attributions, !attributions.isEmpty { + VStack(alignment: .leading, spacing: 8) { + Text("Sources & Maps Links") + .font(.headline) + + ForEach(attributions, id: \.url) { attribution in + if let urlString = attribution.url, let url = URL(string: urlString), let title = attribution.title { + Link(destination: url) { + HStack { + Image(systemName: "mappin.and.ellipse") + Text(title) + .underline() + Spacer() + } + .font(.caption) + .foregroundColor(.blue) + } + .padding(.vertical, 2) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } } } } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift index 716aec456..e2efbc92b 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -26,6 +26,7 @@ public struct PlacesResponse: Codable { @Generable public struct ToolResult: Codable { public let summary: String + public let attributions: [Attribution] } @available(iOS 26.0, *) @@ -48,14 +49,14 @@ public final class FindLocalPlacesTool { public func call(arguments: Arguments) async throws -> ToolResult { print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") - // Setup Firebase AI with Google Search grounding + // Setup Firebase AI with Google Maps grounding let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let session = ai.generativeModelSession( model: "gemini-3.1-flash-lite", - tools: [Tool.googleSearch()] + tools: [Tool.googleMaps()] ) - let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Search to make sure the places are real and currently open." + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Maps to make sure the places are real and currently open." do { let response = try await session.respond( @@ -63,10 +64,22 @@ public final class FindLocalPlacesTool { generating: PlacesResponse.self ) + var attributions: [Attribution] = [] + if let metadata = response.rawResponse.candidates.first?.groundingMetadata { + for chunk in metadata.groundingChunks { + if let maps = chunk.maps, let title = maps.title, let url = maps.url?.absoluteString { + attributions.append(Attribution(title: title, url: url)) + } + } + } + let results = response.content.places print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") - return ToolResult(summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))") + return ToolResult( + summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))", + attributions: attributions + ) } catch { print("FindLocalPlacesTool error: \(error.localizedDescription)") throw error diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 8bc81bd83..00b71d41c 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -146,7 +146,7 @@ public final class AppleAIViewModel: ObservableObject { } ) - let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations." + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." let stream = session.streamResponse( to: promptText, From 2f7bf2541e1229019381b8cf0ded8a7d3e64f42f Mon Sep 17 00:00:00 2001 From: Paul Beusterien Date: Sat, 27 Jun 2026 17:16:13 -0700 Subject: [PATCH 4/4] Replace ConversationKit, enable Vertex AI --- .../ViewModels/AppleAIViewModel.swift | 2 +- .../Features/Chat/Models/ChatMessage.swift | 11 +- .../Features/Chat/Screens/ChatScreen.swift | 1 - .../Chat/ViewModels/ChatViewModel.swift | 3 +- .../Features/Chat/Views/MessageView.swift | 3 +- .../Screens/FunctionCallingScreen.swift | 1 - .../ViewModels/FunctionCallingViewModel.swift | 1 - .../GenerateContentFromTemplateScreen.swift | 5 +- .../Grounding/Screens/GroundingScreen.swift | 6 - .../ViewModels/GroundingViewModel.swift | 1 - .../Models/MultimodalAttachment.swift | 3 +- .../Multimodal/Screens/MultimodalScreen.swift | 1 - .../ViewModels/MultimodalViewModel.swift | 1 - .../NanoBananaFromTemplateScreen.swift | 5 +- .../NanoBanana/NanoBananaScreen.swift | 5 +- .../Shared/Models/Sample.swift | 2 +- .../Shared/Views/ConversationView.swift | 171 ++++++++++++++++++ .../Shared/Views/MessageComposerView.swift | 97 ++++++++++ 18 files changed, 286 insertions(+), 33 deletions(-) create mode 100644 firebaseai/FirebaseAIExample/Shared/Views/ConversationView.swift create mode 100644 firebaseai/FirebaseAIExample/Shared/Views/MessageComposerView.swift diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 00b71d41c..807bec489 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -92,7 +92,7 @@ public final class AppleAIViewModel: ObservableObject { isUsingLocalModel = false do { let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) - let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") + let model = ai.geminiLanguageModel(name: "gemini-2.5-flash-lite") let session = LanguageModelSession( model: model, instructions: instructions diff --git a/firebaseai/FirebaseAIExample/Features/Chat/Models/ChatMessage.swift b/firebaseai/FirebaseAIExample/Features/Chat/Models/ChatMessage.swift index b181ca19d..42c9bcc79 100644 --- a/firebaseai/FirebaseAIExample/Features/Chat/Models/ChatMessage.swift +++ b/firebaseai/FirebaseAIExample/Features/Chat/Models/ChatMessage.swift @@ -18,10 +18,14 @@ import FirebaseAI #endif import Foundation -import ConversationKit import UIKit -public struct ChatMessage: Message { +public enum Participant: Equatable, Hashable { + case user + case other +} + +public struct ChatMessage: Identifiable, Equatable, Hashable { public let id: UUID = .init() public var content: String? public let participant: Participant @@ -30,7 +34,7 @@ public struct ChatMessage: Message { public var groundingMetadata: GroundingMetadata? public var attachments: [MultimodalAttachment] = [] public var image: UIImage? - // required by the Message protocol, but not used in this app + // Not used in this app, but kept for compatibility public var imageURL: String? public init(content: String? = nil, imageURL: String? = nil, participant: Participant, @@ -45,7 +49,6 @@ public struct ChatMessage: Message { self.image = image } - // Protocol-required initializer public init(content: String?, imageURL: String? = nil, participant: Participant) { self.content = content self.imageURL = imageURL diff --git a/firebaseai/FirebaseAIExample/Features/Chat/Screens/ChatScreen.swift b/firebaseai/FirebaseAIExample/Features/Chat/Screens/ChatScreen.swift index ac405951f..68b914df7 100644 --- a/firebaseai/FirebaseAIExample/Features/Chat/Screens/ChatScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/Chat/Screens/ChatScreen.swift @@ -18,7 +18,6 @@ import FirebaseAI #endif import SwiftUI -import ConversationKit struct ChatScreen: View { let backendType: BackendOption diff --git a/firebaseai/FirebaseAIExample/Features/Chat/ViewModels/ChatViewModel.swift b/firebaseai/FirebaseAIExample/Features/Chat/ViewModels/ChatViewModel.swift index 80ef69515..89f95dda6 100644 --- a/firebaseai/FirebaseAIExample/Features/Chat/ViewModels/ChatViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/Chat/ViewModels/ChatViewModel.swift @@ -20,7 +20,6 @@ import Foundation import UIKit import Combine -import ConversationKit @MainActor class ChatViewModel: ObservableObject { @@ -57,7 +56,7 @@ class ChatViewModel: ObservableObject { : FirebaseAI.firebaseAI(backend: .vertexAI()) model = firebaseService.generativeModel( - modelName: sample?.modelName ?? "gemini-3.1-flash-lite", + modelName: sample?.modelName ?? "gemini-2.5-flash-lite", generationConfig: sample?.generationConfig, systemInstruction: sample?.systemInstruction ) diff --git a/firebaseai/FirebaseAIExample/Features/Chat/Views/MessageView.swift b/firebaseai/FirebaseAIExample/Features/Chat/Views/MessageView.swift index 9b06b4d90..0c56415e9 100644 --- a/firebaseai/FirebaseAIExample/Features/Chat/Views/MessageView.swift +++ b/firebaseai/FirebaseAIExample/Features/Chat/Views/MessageView.swift @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import ConversationKit import MarkdownUI import SwiftUI #if canImport(FirebaseAILogic) @@ -144,7 +143,7 @@ struct MessageView: View { corners: [ .topLeft, .topRight, - message.participant == .other ? .bottomRight : .bottomLeft, + message.participant == .other ? UIRectCorner.bottomRight : UIRectCorner.bottomLeft, ]) if message.participant == .other { Spacer() diff --git a/firebaseai/FirebaseAIExample/Features/FunctionCalling/Screens/FunctionCallingScreen.swift b/firebaseai/FirebaseAIExample/Features/FunctionCalling/Screens/FunctionCallingScreen.swift index 97d26081e..f0afedbab 100644 --- a/firebaseai/FirebaseAIExample/Features/FunctionCalling/Screens/FunctionCallingScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/FunctionCalling/Screens/FunctionCallingScreen.swift @@ -18,7 +18,6 @@ import FirebaseAI #endif import SwiftUI -import ConversationKit struct FunctionCallingScreen: View { let backendType: BackendOption diff --git a/firebaseai/FirebaseAIExample/Features/FunctionCalling/ViewModels/FunctionCallingViewModel.swift b/firebaseai/FirebaseAIExample/Features/FunctionCalling/ViewModels/FunctionCallingViewModel.swift index ee7cead73..5968931c3 100644 --- a/firebaseai/FirebaseAIExample/Features/FunctionCalling/ViewModels/FunctionCallingViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/FunctionCalling/ViewModels/FunctionCallingViewModel.swift @@ -20,7 +20,6 @@ import Foundation import UIKit import Combine -import ConversationKit @MainActor class FunctionCallingViewModel: ObservableObject { diff --git a/firebaseai/FirebaseAIExample/Features/GenerativeAIText/Screens/GenerateContentFromTemplateScreen.swift b/firebaseai/FirebaseAIExample/Features/GenerativeAIText/Screens/GenerateContentFromTemplateScreen.swift index 539cf95f7..484573842 100644 --- a/firebaseai/FirebaseAIExample/Features/GenerativeAIText/Screens/GenerateContentFromTemplateScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/GenerativeAIText/Screens/GenerateContentFromTemplateScreen.swift @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import ConversationKit import MarkdownUI import SwiftUI #if canImport(FirebaseAILogic) @@ -44,10 +43,10 @@ struct GenerateContentFromTemplateScreen: View { ScrollView { VStack { MessageComposerView(message: $viewModel.userInput) - .padding(.bottom, 10) - .focused($focusedField, equals: .message) .disableAttachments() .onSubmitAction { sendOrStop() } + .padding(.bottom, 10) + .focused($focusedField, equals: .message) if viewModel.error != nil { HStack { diff --git a/firebaseai/FirebaseAIExample/Features/Grounding/Screens/GroundingScreen.swift b/firebaseai/FirebaseAIExample/Features/Grounding/Screens/GroundingScreen.swift index 77bc414da..4ea827399 100644 --- a/firebaseai/FirebaseAIExample/Features/Grounding/Screens/GroundingScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/Grounding/Screens/GroundingScreen.swift @@ -12,13 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if canImport(FirebaseAILogic) - import FirebaseAILogic -#else - import FirebaseAI -#endif import SwiftUI -import ConversationKit struct GroundingScreen: View { let backendType: BackendOption diff --git a/firebaseai/FirebaseAIExample/Features/Grounding/ViewModels/GroundingViewModel.swift b/firebaseai/FirebaseAIExample/Features/Grounding/ViewModels/GroundingViewModel.swift index 704ddbe24..4548b8afa 100644 --- a/firebaseai/FirebaseAIExample/Features/Grounding/ViewModels/GroundingViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/Grounding/ViewModels/GroundingViewModel.swift @@ -20,7 +20,6 @@ import Foundation import Combine import UIKit -import ConversationKit @MainActor class GroundingViewModel: ObservableObject { diff --git a/firebaseai/FirebaseAIExample/Features/Multimodal/Models/MultimodalAttachment.swift b/firebaseai/FirebaseAIExample/Features/Multimodal/Models/MultimodalAttachment.swift index c4502a567..62bd7098a 100644 --- a/firebaseai/FirebaseAIExample/Features/Multimodal/Models/MultimodalAttachment.swift +++ b/firebaseai/FirebaseAIExample/Features/Multimodal/Models/MultimodalAttachment.swift @@ -20,7 +20,6 @@ import PhotosUI #else import FirebaseAI #endif -import ConversationKit public enum MultimodalAttachmentError: LocalizedError { case unsupportedFileType(extension: String) @@ -43,7 +42,7 @@ public enum MultimodalAttachmentError: LocalizedError { } // MultimodalAttachment is a struct used for transporting data between ViewModels and AttachmentPreviewCard -public struct MultimodalAttachment: Attachment, Equatable { +public struct MultimodalAttachment: Identifiable, Equatable, Hashable { public let id = UUID() public let mimeType: String public let data: Data? diff --git a/firebaseai/FirebaseAIExample/Features/Multimodal/Screens/MultimodalScreen.swift b/firebaseai/FirebaseAIExample/Features/Multimodal/Screens/MultimodalScreen.swift index c3623cc77..a20ea4bbc 100644 --- a/firebaseai/FirebaseAIExample/Features/Multimodal/Screens/MultimodalScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/Multimodal/Screens/MultimodalScreen.swift @@ -19,7 +19,6 @@ #endif import SwiftUI import PhotosUI -import ConversationKit struct MultimodalScreen: View { let backendType: BackendOption diff --git a/firebaseai/FirebaseAIExample/Features/Multimodal/ViewModels/MultimodalViewModel.swift b/firebaseai/FirebaseAIExample/Features/Multimodal/ViewModels/MultimodalViewModel.swift index a0b3b8ce5..4716d7b90 100644 --- a/firebaseai/FirebaseAIExample/Features/Multimodal/ViewModels/MultimodalViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/Multimodal/ViewModels/MultimodalViewModel.swift @@ -23,7 +23,6 @@ import PhotosUI import SwiftUI import AVFoundation import Combine -import ConversationKit @MainActor class MultimodalViewModel: ObservableObject { diff --git a/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaFromTemplateScreen.swift b/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaFromTemplateScreen.swift index 19b3057b6..ed00dbdb8 100644 --- a/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaFromTemplateScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaFromTemplateScreen.swift @@ -18,7 +18,6 @@ import SwiftUI #else import FirebaseAI #endif -import ConversationKit struct NanoBananaFromTemplateScreen: View { let backendType: BackendOption @@ -43,10 +42,10 @@ struct NanoBananaFromTemplateScreen: View { ScrollView { VStack { MessageComposerView(message: $viewModel.userInput) - .padding(.bottom, 10) - .focused($focusedField, equals: .message) .disableAttachments() .onSubmitAction { sendOrStop() } + .padding(.bottom, 10) + .focused($focusedField, equals: .message) if viewModel.error != nil { HStack { diff --git a/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaScreen.swift b/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaScreen.swift index 75aabecac..4189ecd12 100644 --- a/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/NanoBanana/NanoBananaScreen.swift @@ -18,7 +18,6 @@ import SwiftUI #else import FirebaseAI #endif -import ConversationKit struct NanoBananaScreen: View { let backendType: BackendOption @@ -46,10 +45,10 @@ struct NanoBananaScreen: View { ScrollView { VStack { MessageComposerView(message: $userPrompt) - .padding(.bottom, 10) - .focused($focusedField, equals: .message) .disableAttachments() .onSubmitAction { sendOrStop() } + .padding(.bottom, 10) + .focused($focusedField, equals: .message) if viewModel.error != nil { HStack { diff --git a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift index 4f0129345..7c6742919 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift @@ -39,7 +39,7 @@ public struct Sample: Identifiable { description: String, useCases: [UseCase], navRoute: String, - modelName: String = "gemini-3.1-flash-lite", + modelName: String = "gemini-2.5-flash-lite", chatHistory: [ModelContent]? = nil, initialPrompt: String? = nil, systemInstruction: ModelContent? = nil, diff --git a/firebaseai/FirebaseAIExample/Shared/Views/ConversationView.swift b/firebaseai/FirebaseAIExample/Shared/Views/ConversationView.swift new file mode 100644 index 000000000..23c39084f --- /dev/null +++ b/firebaseai/FirebaseAIExample/Shared/Views/ConversationView.swift @@ -0,0 +1,171 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +struct ConversationView: View { + @Binding var messages: [ChatMessage] + @Binding var attachments: [MultimodalAttachment] + var userPrompt: String? + var content: (ChatMessage) -> Content + + @State private var inputText: String = "" + @FocusState private var isFocused: Bool + + private var hasAttachments: Bool = true + private var attachmentActionsClosure: (() -> AnyView)? = nil + private var onSendMessageClosure: ((ChatMessage) async -> Void)? = nil + private var onErrorClosure: ((Error) -> Void)? = nil + + init(messages: Binding<[ChatMessage]>, + attachments: Binding<[MultimodalAttachment]>, + userPrompt: String? = "", + @ViewBuilder content: @escaping (ChatMessage) -> Content) { + self._messages = messages + self._attachments = attachments + self.userPrompt = userPrompt + self.content = content + } + + init(messages: Binding<[ChatMessage]>, + userPrompt: String? = "", + @ViewBuilder content: @escaping (ChatMessage) -> Content) { + self._messages = messages + self._attachments = .constant([]) + self.userPrompt = userPrompt + self.content = content + self.hasAttachments = false + } + + var body: some View { + VStack(spacing: 0) { + ScrollViewReader { proxy in + ScrollView { + LazyVStack(spacing: 16) { + ForEach(messages) { message in + content(message) + .id(message.id) + } + } + .padding(.vertical) + } + .scrollDismissesKeyboard(.interactively) + .onChange(of: messages) { _, newMessages in + if let lastId = newMessages.last?.id { + withAnimation { + proxy.scrollTo(lastId, anchor: .bottom) + } + } + } + .onAppear { + if let lastId = messages.last?.id { + proxy.scrollTo(lastId, anchor: .bottom) + } + if let userPrompt = userPrompt, !userPrompt.isEmpty { + inputText = userPrompt + } + } + } + + Divider() + + // Composer area + HStack(spacing: 12) { + if hasAttachments, let actions = attachmentActionsClosure { + Menu { + actions() + } label: { + Image(systemName: "plus.circle.fill") + .font(.system(size: 24)) + .foregroundColor(.blue) + } + .padding(.leading) + } + + MessageComposerView(message: $inputText, attachments: $attachments) + .disableAttachments(!hasAttachments) + .onSubmitAction { + submitMessage() + } + .focused($isFocused) + .padding(.vertical, 8) + } + .background(Color(.systemBackground)) + } + .environment(\.presentErrorAction) { error in + if let onError = onErrorClosure { + onError(error) + } + } + } + + private func submitMessage() { + let trimmedText = inputText.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedText.isEmpty || !attachments.isEmpty else { return } + + let messageToSend = ChatMessage( + content: trimmedText, + participant: .user, + attachments: attachments + ) + + inputText = "" + attachments = [] + isFocused = false + + Task { + if let onSendMessage = onSendMessageClosure { + await onSendMessage(messageToSend) + } + } + } +} + +extension ConversationView { + func disableAttachments() -> Self { + var copy = self + copy.hasAttachments = false + return copy + } + + func attachmentActions(@ViewBuilder _ actions: @escaping () -> V) -> Self { + var copy = self + copy.attachmentActionsClosure = { AnyView(actions()) } + return copy + } + + func onSendMessage(_ perform: @escaping (ChatMessage) async -> Void) -> Self { + var copy = self + copy.onSendMessageClosure = perform + return copy + } + + func onError(_ perform: @escaping (Error) -> Void) -> Self { + var copy = self + copy.onErrorClosure = perform + return copy + } +} + +// Custom EnvironmentKey to support presenting errors from bubbles +struct PresentErrorActionKey: EnvironmentKey { + static let defaultValue: ((Error) -> Void)? = nil +} + +extension EnvironmentValues { + var presentErrorAction: ((Error) -> Void)? { + get { self[PresentErrorActionKey.self] } + set { self[PresentErrorActionKey.self] = newValue } + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Views/MessageComposerView.swift b/firebaseai/FirebaseAIExample/Shared/Views/MessageComposerView.swift new file mode 100644 index 000000000..46936ab4f --- /dev/null +++ b/firebaseai/FirebaseAIExample/Shared/Views/MessageComposerView.swift @@ -0,0 +1,97 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +struct MessageComposerView: View { + @Binding var message: String + @Binding var attachments: [MultimodalAttachment] + + private var hasAttachments: Bool = true + private var onSubmitClosure: (() -> Void)? = nil + + init(message: Binding, attachments: Binding<[MultimodalAttachment]>) { + self._message = message + self._attachments = attachments + } + + init(message: Binding) { + self._message = message + self._attachments = .constant([]) + self.hasAttachments = false + } + + var body: some View { + VStack(spacing: 8) { + // Draft attachments preview + if hasAttachments && !attachments.isEmpty { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(attachments) { attachment in + ZStack(alignment: .topTrailing) { + AttachmentPreviewCard(attachment: attachment) + .frame(width: 140, height: 60) + + Button(action: { + attachments.removeAll { $0.id == attachment.id } + }) { + Image(systemName: "xmark.circle.fill") + .foregroundColor(.gray) + .background(Color.white.clipShape(Circle())) + } + .offset(x: 4, y: -4) + } + } + } + .padding(.horizontal) + } + } + + HStack(spacing: 12) { + TextField("Type a message...", text: $message, axis: .vertical) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(Color(.systemGray6)) + .cornerRadius(18) + .lineLimit(1...5) + + Button(action: { + if let onSubmit = onSubmitClosure { + onSubmit() + } + }) { + Image(systemName: "arrow.up.circle.fill") + .font(.system(size: 28)) + .foregroundColor(message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && attachments.isEmpty ? .gray : .blue) + } + .disabled(message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && attachments.isEmpty) + } + .padding(.horizontal) + } + } +} + +extension MessageComposerView { + func disableAttachments(_ disable: Bool = true) -> Self { + var copy = self + copy.hasAttachments = !disable + return copy + } + + func onSubmitAction(_ perform: @escaping () -> Void) -> Self { + var copy = self + copy.onSubmitClosure = perform + return copy + } +}