From 04efb3a3d39c40d5b56bb0bbd444fb1452c0aa81 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Mon, 8 Jun 2026 23:03:20 -0700 Subject: [PATCH 01/10] feat(apple-ai): add support for Apple Foundation Models integration --- .../project.pbxproj | 53 ++- .../FirebaseAIExample/ContentView.swift | 6 + .../AppleFoundationModels/Models/Models.swift | 95 +++++ .../Screens/AppleAIScreen.swift | 346 ++++++++++++++++++ .../Tools/FindLocalPlacesTool.swift | 91 +++++ .../ViewModels/AppleAIViewModel.swift | 211 +++++++++++ .../Shared/Models/Sample.swift | 6 + .../Shared/Util/MLErrorHelper.swift | 72 ++++ .../Shared/Views/ErrorDetailsView.swift | 13 +- 9 files changed, 874 insertions(+), 19 deletions(-) create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index a1fcedc92..7124e3c93 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -9,7 +9,10 @@ /* Begin PBXBuildFile section */ 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */ = {isa = PBXBuildFile; productRef = 88151ADB2EC9345700775CFB /* MarkdownUI */; }; 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D8F2EC8AA920080D023 /* ConversationKit */; }; - 88779D932EC8AC460080D023 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D922EC8AC460080D023 /* FirebaseAILogic */; }; + 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5C2FD7C19F003702C7 /* FirebaseAI */; }; + 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */; }; + 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D602FD7C19F003702C7 /* FirebaseAppCheck */; }; + 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D622FD7C19F003702C7 /* FirebaseCore */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -29,9 +32,12 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */, 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */, - 88779D932EC8AC460080D023 /* FirebaseAILogic in Frameworks */, 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */, + 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */, + 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */, + 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -75,8 +81,11 @@ name = FirebaseAIExample; packageProductDependencies = ( 88779D8F2EC8AA920080D023 /* ConversationKit */, - 88779D922EC8AC460080D023 /* FirebaseAILogic */, 88151ADB2EC9345700775CFB /* MarkdownUI */, + 88862D5C2FD7C19F003702C7 /* FirebaseAI */, + 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */, + 88862D602FD7C19F003702C7 /* FirebaseAppCheck */, + 88862D622FD7C19F003702C7 /* FirebaseCore */, ); productName = FirebaseAIExample; productReference = 88779D352EC8A9CF0080D023 /* FirebaseAIExample.app */; @@ -108,8 +117,8 @@ minimizedProjectReferenceProxies = 1; packageReferences = ( 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */, - 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, + 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */, ); preferredProjectObjectVersion = 77; productRefGroup = 88779D362EC8A9CF0080D023 /* Products */; @@ -268,7 +277,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = YGAZHQXHH4; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -302,7 +311,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = YGAZHQXHH4; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -352,6 +361,13 @@ }; /* End XCConfigurationList section */ +/* Begin XCLocalSwiftPackageReference section */ + 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../../firebase-ios-sdk"; + }; +/* End XCLocalSwiftPackageReference section */ + /* Begin XCRemoteSwiftPackageReference section */ 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { isa = XCRemoteSwiftPackageReference; @@ -364,17 +380,9 @@ 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/peterfriese/ConversationKit"; - requirement = { - kind = exactVersion; - version = 0.0.4; - }; - }; - 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/firebase/firebase-ios-sdk"; requirement = { kind = upToNextMajorVersion; - minimumVersion = 12.15.0; + minimumVersion = 0.0.4; }; }; /* End XCRemoteSwiftPackageReference section */ @@ -390,11 +398,22 @@ package = 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */; productName = ConversationKit; }; - 88779D922EC8AC460080D023 /* FirebaseAILogic */ = { + 88862D5C2FD7C19F003702C7 /* FirebaseAI */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseAI; + }; + 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */ = { isa = XCSwiftPackageProductDependency; - package = 88779D912EC8AC460080D023 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; productName = FirebaseAILogic; }; + 88862D602FD7C19F003702C7 /* FirebaseAppCheck */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseAppCheck; + }; + 88862D622FD7C19F003702C7 /* FirebaseCore */ = { + isa = XCSwiftPackageProductDependency; + productName = FirebaseCore; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 88779D2D2EC8A9CF0080D023 /* Project object */; diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 62808c579..103b4d69c 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -112,6 +112,12 @@ struct ContentView: View { GroundingScreen(backendType: selectedBackend, sample: sample) case "LiveScreen": LiveScreen(backendType: selectedBackend, sample: sample) + case "AppleAIScreen": + if #available(iOS 27.0, *) { + AppleAIScreen() + } else { + Text("Apple Intelligence is only available on iOS 27 or newer.") + } default: EmptyView() } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift new file mode 100644 index 000000000..543a05ccf --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -0,0 +1,95 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +@available(iOS 26.0, *) +@Generable +public struct Itinerary: Equatable, Codable { + @Guide(description: "An exciting name for the trip.") + public let title: String + + @Guide(description: "The name of the destination.") + public let destinationName: String + + @Guide(description: "A brief, catchy description of the trip.") + public let description: String + + @Guide(description: "An explanation of why this plan fits the user's request.") + public let rationale: String + + @Guide(description: "A list of day plans.") + public let days: [DayPlan] +} + +@available(iOS 26.0, *) +@Generable +public struct DayPlan: Equatable, Codable { + @Guide(description: "A unique title for this day.") + public let title: String + + public let subtitle: String + + @Guide(description: "A list of activities planned for this day.") + public let activities: [Activity] +} + +@available(iOS 26.0, *) +@Generable +public struct Activity: Equatable, Codable { + public let type: ActivityKind + public let title: String + public let description: String + public let latitude: Double? + public let longitude: Double? +} + +@available(iOS 26.0, *) +@Generable +public enum ActivityKind: String, Codable { + case sightseeing + case foodAndDining + case shopping + case hotelAndLodging + + public var displayName: String { + switch self { + case .sightseeing: return "Sightseeing" + case .foodAndDining: return "Food & Dining" + case .shopping: return "Shopping" + case .hotelAndLodging: return "Hotel & Lodging" + } + } +} + +@available(iOS 26.0, *) +@Generable +public struct IdentifiedObject: Equatable, Codable { + @Guide(description: "The name of the primary object or landmark detected.") + public let name: String + + @Guide(description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing).") + public let category: String + + @Guide(description: "A short, 2-sentence description of the object and interesting facts.") + public let description: String +} + +@available(iOS 26.0, *) +@Generable +public struct TextSummary: Equatable, Codable { + @Guide(description: "A list of exactly 2 key summary points.") + public let summaryPoints: [String] +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift new file mode 100644 index 000000000..4673e5123 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -0,0 +1,346 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI +import PhotosUI +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +struct AppleAIScreen: View { + @StateObject private var viewModel = AppleAIViewModel() + @State private var selectedTab = 0 + @State private var photosPickerItem: PhotosPickerItem? = nil + @State private var presentErrorDetails = false + + var body: some View { + ZStack { + VStack(spacing: 0) { + // Segmented Control + Picker("Feature", selection: $selectedTab) { + Text("Hybrid AI").tag(0) + Text("Planner").tag(1) + Text("Vision ID").tag(2) + } + .pickerStyle(.segmented) + .padding() + .background(Color(.systemBackground)) + + Divider() + + // Content Views + ScrollView { + VStack(spacing: 20) { + if selectedTab == 0 { + hybridAIView + } else if selectedTab == 1 { + plannerView + } else { + visionIDView + } + } + .padding() + } + } + + if viewModel.inProgress { + ProgressOverlay() + } + } + .background(Color(.systemGroupedBackground)) + .navigationTitle("Apple Intelligence") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + if viewModel.inProgress { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Stop") { + viewModel.stopActiveTask() + } + } + } + } + .sheet(isPresented: $presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .onChange(of: viewModel.error != nil) { oldValue, newValue in + if newValue { + presentErrorDetails = true + } + } + } + + // MARK: - Subviews + + // Feature 1: Hybrid AI + private var hybridAIView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 8) { + Text("Input Text") + .font(.headline) + TextEditor(text: $viewModel.inputText) + .frame(height: 120) + .padding(4) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(8) + } + + Button(action: { + Task { + await viewModel.runSummarization() + } + }) { + HStack { + Spacer() + Image(systemName: "sparkles") + Text("Summarize with Apple SDK") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let summary = viewModel.outputSummary { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Summary Points") + .font(.headline) + Spacer() + + // Badge indicating where it was executed + HStack(spacing: 4) { + Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") + Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") + } + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) + .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) + .cornerRadius(8) + } + + VStack(alignment: .leading, spacing: 8) { + ForEach(summary.summaryPoints, id: \.self) { point in + HStack(alignment: .top, spacing: 6) { + Text("•") + .bold() + Text(point) + .font(.subheadline) + } + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity.combined(with: .slide)) + } + } + } + + // Feature 2: Planner View + private var plannerView: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(spacing: 12) { + HStack { + Text("Destination") + .frame(width: 100, alignment: .leading) + TextField("e.g. Paris, Tokyo", text: $viewModel.destination) + .textFieldStyle(.roundedBorder) + } + HStack { + Text("Interests") + .frame(width: 100, alignment: .leading) + TextField("e.g. art, cafes, parks", text: $viewModel.interests) + .textFieldStyle(.roundedBorder) + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + + Button(action: { + Task { + await viewModel.generateItinerary() + } + }) { + HStack { + Spacer() + Image(systemName: "map.fill") + Text("Create Itinerary Plan") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let itinerary = viewModel.itinerary { + VStack(alignment: .leading, spacing: 12) { + Text(itinerary.title ?? "Generating plan...") + .font(.title3) + .bold() + + if let desc = itinerary.description { + Text(desc) + .font(.subheadline) + .foregroundColor(.secondary) + } + + if let rationale = itinerary.rationale { + Text("Rationale: \(rationale)") + .font(.caption) + .padding() + .background(Color.blue.opacity(0.1)) + .cornerRadius(8) + } + + // Day Plans + if let days = itinerary.days { + ForEach(days, id: \.title) { day in + VStack(alignment: .leading, spacing: 8) { + Text(day.title ?? "Day Plan") + .font(.headline) + .padding(.top, 4) + + if let activities = day.activities { + ForEach(activities, id: \.title) { activity in + HStack(alignment: .top, spacing: 12) { + Image(systemName: getSymbol(for: activity.type)) + .foregroundColor(.blue) + .frame(width: 24, height: 24) + .background(Color.blue.opacity(0.1)) + .cornerRadius(6) + + VStack(alignment: .leading, spacing: 2) { + Text(activity.title ?? "Activity") + .font(.subheadline) + .bold() + if let desc = activity.description { + Text(desc) + .font(.caption) + .foregroundColor(.secondary) + } + if let lat = activity.latitude, let lon = activity.longitude { + Text(String(format: "Location: %.4f, %.4f", lat, lon)) + .font(.system(.caption, design: .monospaced)) + .foregroundColor(.gray) + } + } + } + .padding(.vertical, 4) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + } + + + } + } + } + } + + // Feature 3: Vision ID View + private var visionIDView: some View { + VStack(alignment: .leading, spacing: 16) { + Text("Select or Snap a Photo to Identify") + .font(.headline) + + PhotosPicker(selection: $photosPickerItem, matching: .images) { + VStack(spacing: 12) { + if let image = viewModel.selectedImage { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxHeight: 200) + .cornerRadius(12) + } else { + VStack(spacing: 8) { + Image(systemName: "photo.badge.plus") + .font(.system(size: 40)) + Text("Select an Image") + } + .frame(maxWidth: .infinity) + .frame(height: 180) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(12) + } + } + } + .onChange(of: photosPickerItem) { oldItem, newItem in + Task { + if let data = try? await newItem?.loadTransferable(type: Data.self), + let image = UIImage(data: data) { + viewModel.selectedImage = image + await viewModel.identifySelectedImage() + } + } + } + + if let identified = viewModel.identifiedObject { + VStack(alignment: .leading, spacing: 12) { + Text("Identification Result") + .font(.headline) + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Name:") + .bold() + Text(identified.name) + } + HStack { + Text("Category:") + .bold() + Text(identified.category) + } + VStack(alignment: .leading, spacing: 4) { + Text("Description:") + .bold() + Text(identified.description) + .foregroundColor(.secondary) + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity) + } + } + } + + // Helper to pick icons for activity kinds + private func getSymbol(for kind: ActivityKind?) -> String { + guard let kind = kind else { return "info.circle" } + switch kind { + case .sightseeing: return "binoculars.fill" + case .foodAndDining: return "fork.knife" + case .shopping: return "bag.fill" + case .hotelAndLodging: return "bed.double.fill" + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift new file mode 100644 index 000000000..716aec456 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -0,0 +1,91 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels +import FirebaseAILogic + +@available(iOS 26.0, *) +@Generable +public struct PlacesResponse: Codable { + public let places: [String] +} + +@available(iOS 26.0, *) +@Generable +public struct ToolResult: Codable { + public let summary: String +} + +@available(iOS 26.0, *) +public final class FindLocalPlacesTool { + public let name = "findLocalPlaces" + public let description = "Finds points of interest and local businesses matching a query for a specified destination." + + public init() {} + + @available(iOS 26.0, *) + @Generable + public struct Arguments { + @Guide(description: "The destination city/place to look up for (e.g. 'Paris', 'New York').") + public let destination: String + + @Guide(description: "The category or query to search (e.g. 'hotels', 'Italian restaurants', 'museums').") + public let category: String + } + + public func call(arguments: Arguments) async throws -> ToolResult { + print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") + + // Setup Firebase AI with Google Search grounding + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let session = ai.generativeModelSession( + model: "gemini-3.1-flash-lite", + tools: [Tool.googleSearch()] + ) + + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Search to make sure the places are real and currently open." + + do { + let response = try await session.respond( + to: prompt, + generating: PlacesResponse.self + ) + + let results = response.content.places + print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") + + return ToolResult(summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))") + } catch { + print("FindLocalPlacesTool error: \(error.localizedDescription)") + throw error + } + } +} + +// Struct wrapper to conform to FoundationModels.Tool AND ToolRepresentable +@available(iOS 26.0, *) +public struct FindLocalPlacesToolWrapper: FoundationModels.Tool, ToolRepresentable, @unchecked Sendable { + let tool: FindLocalPlacesTool + + public var name: String { tool.name } + public var description: String { tool.description } + + public typealias Arguments = FindLocalPlacesTool.Arguments + public typealias Output = ToolResult + + public func call(arguments: Arguments) async throws -> ToolResult { + try await tool.call(arguments: arguments) + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift new file mode 100644 index 000000000..31d6b38f6 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -0,0 +1,211 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +@MainActor +public final class AppleAIViewModel: ObservableObject { + // Shared state + @Published public var inProgress = false + @Published public var error: Error? + + // Feature 1: Hybrid Summary/Translation + @Published public var inputText: String = "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." + @Published public var outputSummary: TextSummary? + @Published public var isUsingLocalModel: Bool = false + + // Feature 2: Smart Planner + @Published public var destination: String = "San Francisco" + @Published public var interests: String = "tech history, good coffee, bay views" + @Published public var itinerary: Itinerary.PartiallyGenerated? + + // Feature 3: Visual Identifier + @Published public var selectedImage: UIImage? + @Published public var identifiedObject: IdentifiedObject? + + private var activeTask: Task? + + public init() {} + + public func stopActiveTask() { + activeTask?.cancel() + activeTask = nil + inProgress = false + } + + // MARK: - Feature 1: Hybrid Summary/Translation + public func runSummarization() async { + stopActiveTask() + inProgress = true + error = nil + outputSummary = nil + + activeTask = Task { + let availability = SystemLanguageModel.default.availability + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + // Try local model first if it reports available + if availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + inProgress = false + return + } catch { + // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) + if error.isMLAssetUnavailable { + print("Local ML assets unavailable. Falling back to cloud model...") + } else { + print("Local model failed: \(error.localizedDescription). Falling back to cloud model...") + } + } + } + + // Fallback to cloud model + isUsingLocalModel = false + do { + let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + ) + let response = try await session.respond(to: inputText, generating: TextSummary.self) + if !Task.isCancelled { + self.outputSummary = response.content + } + } catch { + if !Task.isCancelled { + // If even the cloud model via LanguageModelSession fails (likely due to missing guardrail assets), + // we show a descriptive error message. + if let mlMessage = error.mlAssetErrorMessage { + print("ML Asset Error detected in cloud fallback: \(mlMessage)") + // We can either set the error here or try one last direct fallback. + // Let's set a custom error that wraps the message. + self.error = NSError(domain: "FirebaseAIExample", + code: 1, + userInfo: [NSLocalizedDescriptionKey: mlMessage]) + } else { + self.error = error + } + } + } + inProgress = false + } + } + + // MARK: - Feature 2: Smart Planner + public func generateItinerary() async { + stopActiveTask() + inProgress = true + error = nil + itinerary = nil + + activeTask = Task { + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + + let model = ai.geminiLanguageModel( + name: "gemini-3.5-flash" + ) + + let session = LanguageModelSession( + model: model, + tools: [localPlacesTool], + instructions: Instructions { + "Your job is to create a structured 1-day itinerary for the user." + "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." + } + ) + + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations." + + let stream = session.streamResponse( + to: promptText, + generating: Itinerary.self + ) + + do { + for try await response in stream { + if Task.isCancelled { break } + self.itinerary = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + inProgress = false + } + } + + // MARK: - Feature 3: Visual Identifier + public func identifySelectedImage() async { + guard let image = selectedImage else { return } + stopActiveTask() + inProgress = true + error = nil + identifiedObject = nil + + activeTask = Task { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + + guard let cgImage = image.cgImage else { + inProgress = false + return + } + + let session = LanguageModelSession( + model: model, + instructions: Instructions { + "You are a visual object identifier." + } + ) + + do { + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + Attachment(cgImage) + } + + if !Task.isCancelled { + self.identifiedObject = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + inProgress = false + } + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift index 384f6f4cd..4f0129345 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift @@ -309,6 +309,12 @@ extension Sample { ), tip: InlineTip(text: "Try asking the model to change the background color"), ), + Sample( + title: "Apple Foundation Models", + description: "Demonstrates how to integrate Firebase AI Logic (Gemini) with Apple's Foundation Models framework (Apple Intelligence).", + useCases: [.text, .image, .functionCalling], + navRoute: "AppleAIScreen" + ), ] public static var sample = samples[0] diff --git a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift new file mode 100644 index 000000000..bfaf543f7 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import FoundationModels + +extension Error { + /// Returns true if this error indicates that required ML assets (like safety guardrails) + /// are missing from the device or simulator. + public var isMLAssetUnavailable: Bool { + // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable + if #available(iOS 26.0, *) { + if let genError = self as? LanguageModelSession.GenerationError { + if case .assetsUnavailable = genError { + return true + } + } + } + + let nsError = self as NSError + + // 2. Check for the specific SensitiveContentAnalysisML error (Code 15) + if nsError.domain == "com.apple.SensitiveContentAnalysisML" && nsError.code == 15 { + return true + } + + // 3. Check for underlying GenerativeFunctionsFoundation error (Code 1020000) + // This is often found in the UserInfo of higher-level errors. + if nsError.domain == "com.apple.GenerativeFunctionsFoundation.GenerativeError" && nsError.code == 1020000 { + return true + } + + // 4. Recursive check for underlying errors + if let underlying = nsError.userInfo[NSUnderlyingErrorKey] as? Error { + return underlying.isMLAssetUnavailable + } + + return false + } + + /// Returns a user-friendly message for ML asset errors, particularly for simulators. + public var mlAssetErrorMessage: String? { + guard isMLAssetUnavailable else { return nil } + + #if targetEnvironment(simulator) + return """ + Apple Intelligence assets are missing in this simulator. + + To fix this: + 1. Open Settings in the simulator. + 2. Go to Apple Intelligence & Siri. + 3. Toggle Apple Intelligence ON. + 4. Wait for assets to download (check the status in Settings). + + Alternatively, use a physical device with Apple Intelligence support. + """ + #else + return "Apple Intelligence assets are not yet ready on this device. Please ensure Apple Intelligence is enabled in Settings and that model assets have finished downloading." + #endif + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift index cd1e3f60e..7ba2b1788 100644 --- a/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift +++ b/firebaseai/FirebaseAIExample/Shared/Views/ErrorDetailsView.swift @@ -148,11 +148,20 @@ struct ErrorDetailsView: View { default: Section("Error Type") { - Text("Some other error") + if error.isMLAssetUnavailable { + Text("Apple Intelligence Assets Missing") + .foregroundColor(.orange) + } else { + Text("Some other error") + } } Section("Details") { - SubtitleFormRow(title: "Error description", value: error.localizedDescription) + if let mlMessage = error.mlAssetErrorMessage { + SubtitleFormRow(title: "How to fix", value: mlMessage) + } else { + SubtitleFormRow(title: "Error description", value: error.localizedDescription) + } } } } From 9ad90c813d143aa9ee4caa53f462500e8b59b397 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Tue, 9 Jun 2026 06:41:04 -0700 Subject: [PATCH 02/10] refactor(apple-ai): clean up tasks and session handling in AppleAIViewModel --- .../ViewModels/AppleAIViewModel.swift | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 31d6b38f6..8bc81bd83 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -57,8 +57,13 @@ public final class AppleAIViewModel: ObservableObject { outputSummary = nil activeTask = Task { + defer { self.inProgress = false } + + let instructions = Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + let availability = SystemLanguageModel.default.availability - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) // Try local model first if it reports available if availability == .available { @@ -66,15 +71,12 @@ public final class AppleAIViewModel: ObservableObject { do { let session = LanguageModelSession( model: SystemLanguageModel.default, - instructions: Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } + instructions: instructions ) let response = try await session.respond(to: inputText, generating: TextSummary.self) if !Task.isCancelled { self.outputSummary = response.content } - inProgress = false return } catch { // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) @@ -89,12 +91,11 @@ public final class AppleAIViewModel: ObservableObject { // Fallback to cloud model isUsingLocalModel = false do { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") let session = LanguageModelSession( model: model, - instructions: Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } + instructions: instructions ) let response = try await session.respond(to: inputText, generating: TextSummary.self) if !Task.isCancelled { @@ -116,7 +117,6 @@ public final class AppleAIViewModel: ObservableObject { } } } - inProgress = false } } @@ -128,6 +128,8 @@ public final class AppleAIViewModel: ObservableObject { itinerary = nil activeTask = Task { + defer { self.inProgress = false } + let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) @@ -161,7 +163,6 @@ public final class AppleAIViewModel: ObservableObject { self.error = error } } - inProgress = false } } @@ -174,14 +175,13 @@ public final class AppleAIViewModel: ObservableObject { identifiedObject = nil activeTask = Task { + defer { self.inProgress = false } + + guard let cgImage = image.cgImage else { return } + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - guard let cgImage = image.cgImage else { - inProgress = false - return - } - let session = LanguageModelSession( model: model, instructions: Instructions { @@ -205,7 +205,6 @@ public final class AppleAIViewModel: ObservableObject { self.error = error } } - inProgress = false } } } From 7cd57d2fd8b3f3470421106f300a3345e5b467a9 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Tue, 9 Jun 2026 06:45:52 -0700 Subject: [PATCH 03/10] feat(apple-ai): add support for Google Maps grounding attributions in planner --- .../AppleFoundationModels/Models/Models.swift | 10 +++++++ .../Screens/AppleAIScreen.swift | 26 ++++++++++++++++++- .../Tools/FindLocalPlacesTool.swift | 21 ++++++++++++--- .../ViewModels/AppleAIViewModel.swift | 2 +- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift index 543a05ccf..821cef536 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -32,6 +32,16 @@ public struct Itinerary: Equatable, Codable { @Guide(description: "A list of day plans.") public let days: [DayPlan] + + @Guide(description: "Any source attributions or links for the recommended places.") + public let attributions: [Attribution]? +} + +@available(iOS 26.0, *) +@Generable +public struct Attribution: Equatable, Codable { + public let title: String + public let url: String } @available(iOS 26.0, *) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift index 4673e5123..e942051f4 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -257,7 +257,31 @@ struct AppleAIScreen: View { } } - + if let attributions = itinerary.attributions, !attributions.isEmpty { + VStack(alignment: .leading, spacing: 8) { + Text("Sources & Maps Links") + .font(.headline) + + ForEach(attributions, id: \.url) { attribution in + if let urlString = attribution.url, let url = URL(string: urlString), let title = attribution.title { + Link(destination: url) { + HStack { + Image(systemName: "mappin.and.ellipse") + Text(title) + .underline() + Spacer() + } + .font(.caption) + .foregroundColor(.blue) + } + .padding(.vertical, 2) + } + } + } + .padding() + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } } } } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift index 716aec456..e2efbc92b 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift @@ -26,6 +26,7 @@ public struct PlacesResponse: Codable { @Generable public struct ToolResult: Codable { public let summary: String + public let attributions: [Attribution] } @available(iOS 26.0, *) @@ -48,14 +49,14 @@ public final class FindLocalPlacesTool { public func call(arguments: Arguments) async throws -> ToolResult { print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") - // Setup Firebase AI with Google Search grounding + // Setup Firebase AI with Google Maps grounding let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) let session = ai.generativeModelSession( model: "gemini-3.1-flash-lite", - tools: [Tool.googleSearch()] + tools: [Tool.googleMaps()] ) - let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Search to make sure the places are real and currently open." + let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Maps to make sure the places are real and currently open." do { let response = try await session.respond( @@ -63,10 +64,22 @@ public final class FindLocalPlacesTool { generating: PlacesResponse.self ) + var attributions: [Attribution] = [] + if let metadata = response.rawResponse.candidates.first?.groundingMetadata { + for chunk in metadata.groundingChunks { + if let maps = chunk.maps, let title = maps.title, let url = maps.url?.absoluteString { + attributions.append(Attribution(title: title, url: url)) + } + } + } + let results = response.content.places print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") - return ToolResult(summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))") + return ToolResult( + summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))", + attributions: attributions + ) } catch { print("FindLocalPlacesTool error: \(error.localizedDescription)") throw error diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 8bc81bd83..00b71d41c 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -146,7 +146,7 @@ public final class AppleAIViewModel: ObservableObject { } ) - let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations." + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." let stream = session.streamResponse( to: promptText, From cfb49d850481b66a9ff0e58d459511dffcb65ceb Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Sat, 27 Jun 2026 16:16:36 +0200 Subject: [PATCH 04/10] feat(apple-foundation-models): add model indicator to all tabs --- .../project.pbxproj | 72 ++++++++++--------- .../Screens/AppleAIScreen.swift | 43 ++++++----- .../ViewModels/AppleAIViewModel.swift | 2 + 3 files changed, 68 insertions(+), 49 deletions(-) diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index 7124e3c93..43106ee49 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -7,12 +7,14 @@ objects = { /* Begin PBXBuildFile section */ - 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */ = {isa = PBXBuildFile; productRef = 88151ADB2EC9345700775CFB /* MarkdownUI */; }; - 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 88779D8F2EC8AA920080D023 /* ConversationKit */; }; 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5C2FD7C19F003702C7 /* FirebaseAI */; }; 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */; }; 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D602FD7C19F003702C7 /* FirebaseAppCheck */; }; 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D622FD7C19F003702C7 /* FirebaseCore */; }; + 88AAB45E2FF0077E001B143E /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB45D2FF0077E001B143E /* FirebaseAI */; }; + 88AAB4602FF0077E001B143E /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB45F2FF0077E001B143E /* FirebaseAppCheck */; }; + 88AAB4622FF0077E001B143E /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB4612FF0077E001B143E /* FirebaseCore */; }; + 88AAB4652FF007DD001B143E /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB4642FF007DD001B143E /* ConversationKit */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -32,11 +34,13 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 88AAB45E2FF0077E001B143E /* FirebaseAI in Frameworks */, 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */, - 88151ADC2EC9345700775CFB /* MarkdownUI in Frameworks */, - 88779D902EC8AA920080D023 /* ConversationKit in Frameworks */, + 88AAB4652FF007DD001B143E /* ConversationKit in Frameworks */, 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */, 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */, + 88AAB4622FF0077E001B143E /* FirebaseCore in Frameworks */, + 88AAB4602FF0077E001B143E /* FirebaseAppCheck in Frameworks */, 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -80,12 +84,14 @@ ); name = FirebaseAIExample; packageProductDependencies = ( - 88779D8F2EC8AA920080D023 /* ConversationKit */, - 88151ADB2EC9345700775CFB /* MarkdownUI */, 88862D5C2FD7C19F003702C7 /* FirebaseAI */, 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */, 88862D602FD7C19F003702C7 /* FirebaseAppCheck */, 88862D622FD7C19F003702C7 /* FirebaseCore */, + 88AAB45D2FF0077E001B143E /* FirebaseAI */, + 88AAB45F2FF0077E001B143E /* FirebaseAppCheck */, + 88AAB4612FF0077E001B143E /* FirebaseCore */, + 88AAB4642FF007DD001B143E /* ConversationKit */, ); productName = FirebaseAIExample; productReference = 88779D352EC8A9CF0080D023 /* FirebaseAIExample.app */; @@ -116,9 +122,8 @@ mainGroup = 88779D2C2EC8A9CF0080D023; minimizedProjectReferenceProxies = 1; packageReferences = ( - 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */, - 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, - 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */, + 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, + 88AAB4632FF007DD001B143E /* XCLocalSwiftPackageReference "../../../../ConversationKit/ConversationKit" */, ); preferredProjectObjectVersion = 77; productRefGroup = 88779D362EC8A9CF0080D023 /* Products */; @@ -362,42 +367,24 @@ /* End XCConfigurationList section */ /* Begin XCLocalSwiftPackageReference section */ - 88862D5B2FD7C19F003702C7 /* XCLocalSwiftPackageReference "../../firebase-ios-sdk" */ = { + 88AAB4632FF007DD001B143E /* XCLocalSwiftPackageReference "../../../../ConversationKit/ConversationKit" */ = { isa = XCLocalSwiftPackageReference; - relativePath = "../../firebase-ios-sdk"; + relativePath = ../../../../ConversationKit/ConversationKit; }; /* End XCLocalSwiftPackageReference section */ /* Begin XCRemoteSwiftPackageReference section */ - 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { + 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */ = { isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/gonzalezreal/swift-markdown-ui"; + repositoryURL = "https://github.com/firebase/firebase-ios-sdk/"; requirement = { - kind = upToNextMajorVersion; - minimumVersion = 2.4.1; - }; - }; - 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/peterfriese/ConversationKit"; - requirement = { - kind = upToNextMajorVersion; - minimumVersion = 0.0.4; + branch = "wwdc26-preview"; + kind = branch; }; }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - 88151ADB2EC9345700775CFB /* MarkdownUI */ = { - isa = XCSwiftPackageProductDependency; - package = 88151ADA2EC9345700775CFB /* XCRemoteSwiftPackageReference "swift-markdown-ui" */; - productName = MarkdownUI; - }; - 88779D8F2EC8AA920080D023 /* ConversationKit */ = { - isa = XCSwiftPackageProductDependency; - package = 88779D8E2EC8AA920080D023 /* XCRemoteSwiftPackageReference "ConversationKit" */; - productName = ConversationKit; - }; 88862D5C2FD7C19F003702C7 /* FirebaseAI */ = { isa = XCSwiftPackageProductDependency; productName = FirebaseAI; @@ -414,6 +401,25 @@ isa = XCSwiftPackageProductDependency; productName = FirebaseCore; }; + 88AAB45D2FF0077E001B143E /* FirebaseAI */ = { + isa = XCSwiftPackageProductDependency; + package = 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; + productName = FirebaseAI; + }; + 88AAB45F2FF0077E001B143E /* FirebaseAppCheck */ = { + isa = XCSwiftPackageProductDependency; + package = 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; + productName = FirebaseAppCheck; + }; + 88AAB4612FF0077E001B143E /* FirebaseCore */ = { + isa = XCSwiftPackageProductDependency; + package = 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; + productName = FirebaseCore; + }; + 88AAB4642FF007DD001B143E /* ConversationKit */ = { + isa = XCSwiftPackageProductDependency; + productName = ConversationKit; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 88779D2D2EC8A9CF0080D023 /* Project object */; diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift index e942051f4..5078854e7 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift @@ -84,6 +84,19 @@ struct AppleAIScreen: View { // MARK: - Subviews + private var modelIndicatorView: some View { + HStack(spacing: 4) { + Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") + Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") + } + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) + .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) + .cornerRadius(8) + } + // Feature 1: Hybrid AI private var hybridAIView: some View { VStack(alignment: .leading, spacing: 16) { @@ -122,17 +135,7 @@ struct AppleAIScreen: View { .font(.headline) Spacer() - // Badge indicating where it was executed - HStack(spacing: 4) { - Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") - Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") - } - .font(.caption) - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) - .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) - .cornerRadius(8) + modelIndicatorView } VStack(alignment: .leading, spacing: 8) { @@ -196,9 +199,13 @@ struct AppleAIScreen: View { if let itinerary = viewModel.itinerary { VStack(alignment: .leading, spacing: 12) { - Text(itinerary.title ?? "Generating plan...") - .font(.title3) - .bold() + HStack { + Text(itinerary.title ?? "Generating plan...") + .font(.title3) + .bold() + Spacer() + modelIndicatorView + } if let desc = itinerary.description { Text(desc) @@ -326,8 +333,12 @@ struct AppleAIScreen: View { if let identified = viewModel.identifiedObject { VStack(alignment: .leading, spacing: 12) { - Text("Identification Result") - .font(.headline) + HStack { + Text("Identification Result") + .font(.headline) + Spacer() + modelIndicatorView + } VStack(alignment: .leading, spacing: 8) { HStack { diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 00b71d41c..53a94835b 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -126,6 +126,7 @@ public final class AppleAIViewModel: ObservableObject { inProgress = true error = nil itinerary = nil + isUsingLocalModel = false activeTask = Task { defer { self.inProgress = false } @@ -173,6 +174,7 @@ public final class AppleAIViewModel: ObservableObject { inProgress = true error = nil identifiedObject = nil + isUsingLocalModel = false activeTask = Task { defer { self.inProgress = false } From 784a95ae8f199fdd3537ded37d9b56d238cab91e Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Sat, 27 Jun 2026 16:57:28 +0200 Subject: [PATCH 05/10] feat(apple-foundation-models): implement local model fallback for planner and vision id --- .../ViewModels/AppleAIViewModel.swift | 112 ++++++++++++++---- 1 file changed, 87 insertions(+), 25 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index 53a94835b..fb426e10f 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -132,29 +132,59 @@ public final class AppleAIViewModel: ObservableObject { defer { self.inProgress = false } let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let instructions = Instructions { + "Your job is to create a structured 1-day itinerary for the user." + "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." + } + let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." - let model = ai.geminiLanguageModel( - name: "gemini-3.5-flash" - ) + let availability = SystemLanguageModel.default.availability - let session = LanguageModelSession( - model: model, - tools: [localPlacesTool], - instructions: Instructions { - "Your job is to create a structured 1-day itinerary for the user." - "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." + // Try local model first if it reports available + if availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + tools: [localPlacesTool], + instructions: instructions + ) + let stream = session.streamResponse( + to: promptText, + generating: Itinerary.self + ) + for try await response in stream { + if Task.isCancelled { break } + self.itinerary = response.content + print(response.content) + } + return + } catch { + // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) + if error.isMLAssetUnavailable { + print("Local ML assets unavailable for Planner. Falling back to cloud model...") + } else { + print("Local model failed for Planner: \(error.localizedDescription). Falling back to cloud model...") + } } - ) - - let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." - - let stream = session.streamResponse( - to: promptText, - generating: Itinerary.self - ) + } + // Fallback to cloud model + isUsingLocalModel = false do { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel( + name: "gemini-3.5-flash" + ) + let session = LanguageModelSession( + model: model, + tools: [localPlacesTool], + instructions: instructions + ) + let stream = session.streamResponse( + to: promptText, + generating: Itinerary.self + ) for try await response in stream { if Task.isCancelled { break } self.itinerary = response.content @@ -181,17 +211,49 @@ public final class AppleAIViewModel: ObservableObject { guard let cgImage = image.cgImage else { return } - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) - let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + let instructions = Instructions { + "You are a visual object identifier." + } + + let availability = SystemLanguageModel.default.availability - let session = LanguageModelSession( - model: model, - instructions: Instructions { - "You are a visual object identifier." + // Try local model first if it reports available + if availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: instructions + ) + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + Attachment(cgImage) + } + if !Task.isCancelled { + self.identifiedObject = response.content + } + return + } catch { + // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) + if error.isMLAssetUnavailable { + print("Local ML assets unavailable for Vision ID. Falling back to cloud model...") + } else { + print("Local model failed for Vision ID: \(error.localizedDescription). Falling back to cloud model...") + } } - ) + } + // Fallback to cloud model + isUsingLocalModel = false do { + let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + let session = LanguageModelSession( + model: model, + instructions: instructions + ) let response = try await session.respond( generating: IdentifiedObject.self ) { From 0de49f0a370fdcf0d16a8125b14275f64270bc75 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Sat, 27 Jun 2026 17:37:24 +0200 Subject: [PATCH 06/10] fix(apple-foundation-models): ensure image attachment is sent in cloud fallback path --- .../ViewModels/AppleAIViewModel.swift | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift index fb426e10f..6f4d527ae 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift @@ -39,6 +39,8 @@ public final class AppleAIViewModel: ObservableObject { @Published public var selectedImage: UIImage? @Published public var identifiedObject: IdentifiedObject? + @Published public var forceCloudModel: Bool = false + private var activeTask: Task? public init() {} @@ -65,8 +67,8 @@ public final class AppleAIViewModel: ObservableObject { let availability = SystemLanguageModel.default.availability - // Try local model first if it reports available - if availability == .available { + // Try local model first if it reports available and not forced to cloud + if !forceCloudModel && availability == .available { isUsingLocalModel = true do { let session = LanguageModelSession( @@ -140,8 +142,8 @@ public final class AppleAIViewModel: ObservableObject { let availability = SystemLanguageModel.default.availability - // Try local model first if it reports available - if availability == .available { + // Try local model first if it reports available and not forced to cloud + if !forceCloudModel && availability == .available { isUsingLocalModel = true do { let session = LanguageModelSession( @@ -209,7 +211,8 @@ public final class AppleAIViewModel: ObservableObject { activeTask = Task { defer { self.inProgress = false } - guard let cgImage = image.cgImage else { return } + guard let cgImage = image.cgImage, + let imageData = image.jpegData(compressionQuality: 0.8) else { return } let instructions = Instructions { "You are a visual object identifier." @@ -217,8 +220,8 @@ public final class AppleAIViewModel: ObservableObject { let availability = SystemLanguageModel.default.availability - // Try local model first if it reports available - if availability == .available { + // Try local model first if it reports available and not forced to cloud + if !forceCloudModel && availability == .available { isUsingLocalModel = true do { let session = LanguageModelSession( @@ -258,7 +261,7 @@ public final class AppleAIViewModel: ObservableObject { generating: IdentifiedObject.self ) { "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." - Attachment(cgImage) + InlineDataPart(data: imageData, mimeType: "image/jpeg") } if !Task.isCancelled { From 48bafcb6235cc43bc270d6bb2eb6e2ddcdc4681f Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Thu, 2 Jul 2026 22:56:23 +0200 Subject: [PATCH 07/10] Refactor Foundation Models: split ViewModels, inject dynamic backend, remove Smart Planner, and modernize UI --- .../FirebaseAIExample/ContentView.swift | 31 +- .../AppleFoundationModels/Models/Models.swift | 89 +-- .../Screens/AppleAIScreen.swift | 381 ----------- .../Screens/HybridAIScreen.swift | 30 + .../Screens/VisionIDScreen.swift | 32 + .../Tools/FindLocalPlacesTool.swift | 104 --- .../ViewModels/AppleAIViewModel.swift | 277 -------- .../FoundationModelsBaseViewModel.swift | 58 ++ .../ViewModels/HybridAIViewModel.swift | 100 +++ .../ViewModels/VisionIDViewModel.swift | 104 +++ .../Views/FoundationModelsContainer.swift | 72 ++ .../Views/HybridAIView.swift | 80 +++ .../Views/ModelIndicatorView.swift | 32 + .../Views/VisionIDView.swift | 99 +++ .../Shared/Models/BackendOption.swift | 4 +- .../Shared/Models/Sample.swift | 21 +- .../Shared/Util/MLErrorHelper.swift | 101 +-- firebaseai/FirebaseDocs.md | 625 ++++++++++++++++++ 18 files changed, 1339 insertions(+), 901 deletions(-) delete mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/HybridAIScreen.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/VisionIDScreen.swift delete mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift delete mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/FoundationModelsContainer.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/ModelIndicatorView.swift create mode 100644 firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift create mode 100644 firebaseai/FirebaseDocs.md diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 103b4d69c..5ec4af3cd 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -76,7 +76,24 @@ struct ContentView: View { .padding(.horizontal) LazyVGrid(columns: columns, spacing: 20) { - ForEach(filteredSamples) { sample in + ForEach(filteredSamples.filter { !$0.isFoundationModel }) { sample in + NavigationLink(destination: destinationView(for: sample)) { + SampleCardView(sample: sample) + } + .buttonStyle(PlainButtonStyle()) + } + } + .padding(.horizontal) + } + + // Foundation Models + VStack(alignment: .leading) { + Text("Foundation Models") + .font(.system(size: 20, weight: .bold)) + .padding(.horizontal) + + LazyVGrid(columns: columns, spacing: 20) { + ForEach(filteredSamples.filter { $0.isFoundationModel }) { sample in NavigationLink(destination: destinationView(for: sample)) { SampleCardView(sample: sample) } @@ -112,11 +129,17 @@ struct ContentView: View { GroundingScreen(backendType: selectedBackend, sample: sample) case "LiveScreen": LiveScreen(backendType: selectedBackend, sample: sample) - case "AppleAIScreen": + case "FoundationModelsHybrid": + if #available(iOS 27.0, *) { + HybridAIScreen(backendType: selectedBackend) + } else { + Text("Foundation Models are only available on iOS 27 or newer.") + } + case "FoundationModelsVisionID": if #available(iOS 27.0, *) { - AppleAIScreen() + VisionIDScreen(backendType: selectedBackend) } else { - Text("Apple Intelligence is only available on iOS 27 or newer.") + Text("Foundation Models are only available on iOS 27 or newer.") } default: EmptyView() diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift index 821cef536..7db83a2ec 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -17,89 +17,22 @@ import FoundationModels @available(iOS 26.0, *) @Generable -public struct Itinerary: Equatable, Codable { - @Guide(description: "An exciting name for the trip.") - public let title: String - - @Guide(description: "The name of the destination.") - public let destinationName: String - - @Guide(description: "A brief, catchy description of the trip.") - public let description: String - - @Guide(description: "An explanation of why this plan fits the user's request.") - public let rationale: String - - @Guide(description: "A list of day plans.") - public let days: [DayPlan] - - @Guide(description: "Any source attributions or links for the recommended places.") - public let attributions: [Attribution]? -} - -@available(iOS 26.0, *) -@Generable -public struct Attribution: Equatable, Codable { - public let title: String - public let url: String -} - -@available(iOS 26.0, *) -@Generable -public struct DayPlan: Equatable, Codable { - @Guide(description: "A unique title for this day.") - public let title: String - - public let subtitle: String - - @Guide(description: "A list of activities planned for this day.") - public let activities: [Activity] -} - -@available(iOS 26.0, *) -@Generable -public struct Activity: Equatable, Codable { - public let type: ActivityKind - public let title: String - public let description: String - public let latitude: Double? - public let longitude: Double? -} +public struct IdentifiedObject: Equatable, Codable { + @Guide(description: "The name of the primary object or landmark detected.") + public let name: String -@available(iOS 26.0, *) -@Generable -public enum ActivityKind: String, Codable { - case sightseeing - case foodAndDining - case shopping - case hotelAndLodging - - public var displayName: String { - switch self { - case .sightseeing: return "Sightseeing" - case .foodAndDining: return "Food & Dining" - case .shopping: return "Shopping" - case .hotelAndLodging: return "Hotel & Lodging" - } - } -} + @Guide( + description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing)." + ) + public let category: String -@available(iOS 26.0, *) -@Generable -public struct IdentifiedObject: Equatable, Codable { - @Guide(description: "The name of the primary object or landmark detected.") - public let name: String - - @Guide(description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing).") - public let category: String - - @Guide(description: "A short, 2-sentence description of the object and interesting facts.") - public let description: String + @Guide(description: "A short, 2-sentence description of the object and interesting facts.") + public let description: String } @available(iOS 26.0, *) @Generable public struct TextSummary: Equatable, Codable { - @Guide(description: "A list of exactly 2 key summary points.") - public let summaryPoints: [String] + @Guide(description: "A list of exactly 2 key summary points.") + public let summaryPoints: [String] } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift deleted file mode 100644 index 5078854e7..000000000 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/AppleAIScreen.swift +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import SwiftUI -import PhotosUI -import FoundationModels -import FirebaseAILogic - -@available(iOS 27.0, *) -struct AppleAIScreen: View { - @StateObject private var viewModel = AppleAIViewModel() - @State private var selectedTab = 0 - @State private var photosPickerItem: PhotosPickerItem? = nil - @State private var presentErrorDetails = false - - var body: some View { - ZStack { - VStack(spacing: 0) { - // Segmented Control - Picker("Feature", selection: $selectedTab) { - Text("Hybrid AI").tag(0) - Text("Planner").tag(1) - Text("Vision ID").tag(2) - } - .pickerStyle(.segmented) - .padding() - .background(Color(.systemBackground)) - - Divider() - - // Content Views - ScrollView { - VStack(spacing: 20) { - if selectedTab == 0 { - hybridAIView - } else if selectedTab == 1 { - plannerView - } else { - visionIDView - } - } - .padding() - } - } - - if viewModel.inProgress { - ProgressOverlay() - } - } - .background(Color(.systemGroupedBackground)) - .navigationTitle("Apple Intelligence") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - if viewModel.inProgress { - ToolbarItem(placement: .navigationBarTrailing) { - Button("Stop") { - viewModel.stopActiveTask() - } - } - } - } - .sheet(isPresented: $presentErrorDetails) { - if let error = viewModel.error { - ErrorDetailsView(error: error) - } - } - .onChange(of: viewModel.error != nil) { oldValue, newValue in - if newValue { - presentErrorDetails = true - } - } - } - - // MARK: - Subviews - - private var modelIndicatorView: some View { - HStack(spacing: 4) { - Image(systemName: viewModel.isUsingLocalModel ? "iphone" : "cloud.fill") - Text(viewModel.isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") - } - .font(.caption) - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background(viewModel.isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) - .foregroundColor(viewModel.isUsingLocalModel ? .green : .purple) - .cornerRadius(8) - } - - // Feature 1: Hybrid AI - private var hybridAIView: some View { - VStack(alignment: .leading, spacing: 16) { - VStack(alignment: .leading, spacing: 8) { - Text("Input Text") - .font(.headline) - TextEditor(text: $viewModel.inputText) - .frame(height: 120) - .padding(4) - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(8) - } - - Button(action: { - Task { - await viewModel.runSummarization() - } - }) { - HStack { - Spacer() - Image(systemName: "sparkles") - Text("Summarize with Apple SDK") - Spacer() - } - .padding() - .foregroundColor(.white) - .background(Color.blue) - .cornerRadius(10) - } - .disabled(viewModel.inProgress) - - if let summary = viewModel.outputSummary { - VStack(alignment: .leading, spacing: 12) { - HStack { - Text("Summary Points") - .font(.headline) - Spacer() - - modelIndicatorView - } - - VStack(alignment: .leading, spacing: 8) { - ForEach(summary.summaryPoints, id: \.self) { point in - HStack(alignment: .top, spacing: 6) { - Text("•") - .bold() - Text(point) - .font(.subheadline) - } - } - } - .padding() - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(10) - } - .transition(.opacity.combined(with: .slide)) - } - } - } - - // Feature 2: Planner View - private var plannerView: some View { - VStack(alignment: .leading, spacing: 16) { - VStack(spacing: 12) { - HStack { - Text("Destination") - .frame(width: 100, alignment: .leading) - TextField("e.g. Paris, Tokyo", text: $viewModel.destination) - .textFieldStyle(.roundedBorder) - } - HStack { - Text("Interests") - .frame(width: 100, alignment: .leading) - TextField("e.g. art, cafes, parks", text: $viewModel.interests) - .textFieldStyle(.roundedBorder) - } - } - .padding() - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(10) - - Button(action: { - Task { - await viewModel.generateItinerary() - } - }) { - HStack { - Spacer() - Image(systemName: "map.fill") - Text("Create Itinerary Plan") - Spacer() - } - .padding() - .foregroundColor(.white) - .background(Color.blue) - .cornerRadius(10) - } - .disabled(viewModel.inProgress) - - if let itinerary = viewModel.itinerary { - VStack(alignment: .leading, spacing: 12) { - HStack { - Text(itinerary.title ?? "Generating plan...") - .font(.title3) - .bold() - Spacer() - modelIndicatorView - } - - if let desc = itinerary.description { - Text(desc) - .font(.subheadline) - .foregroundColor(.secondary) - } - - if let rationale = itinerary.rationale { - Text("Rationale: \(rationale)") - .font(.caption) - .padding() - .background(Color.blue.opacity(0.1)) - .cornerRadius(8) - } - - // Day Plans - if let days = itinerary.days { - ForEach(days, id: \.title) { day in - VStack(alignment: .leading, spacing: 8) { - Text(day.title ?? "Day Plan") - .font(.headline) - .padding(.top, 4) - - if let activities = day.activities { - ForEach(activities, id: \.title) { activity in - HStack(alignment: .top, spacing: 12) { - Image(systemName: getSymbol(for: activity.type)) - .foregroundColor(.blue) - .frame(width: 24, height: 24) - .background(Color.blue.opacity(0.1)) - .cornerRadius(6) - - VStack(alignment: .leading, spacing: 2) { - Text(activity.title ?? "Activity") - .font(.subheadline) - .bold() - if let desc = activity.description { - Text(desc) - .font(.caption) - .foregroundColor(.secondary) - } - if let lat = activity.latitude, let lon = activity.longitude { - Text(String(format: "Location: %.4f, %.4f", lat, lon)) - .font(.system(.caption, design: .monospaced)) - .foregroundColor(.gray) - } - } - } - .padding(.vertical, 4) - } - } - } - .padding() - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(10) - } - } - - if let attributions = itinerary.attributions, !attributions.isEmpty { - VStack(alignment: .leading, spacing: 8) { - Text("Sources & Maps Links") - .font(.headline) - - ForEach(attributions, id: \.url) { attribution in - if let urlString = attribution.url, let url = URL(string: urlString), let title = attribution.title { - Link(destination: url) { - HStack { - Image(systemName: "mappin.and.ellipse") - Text(title) - .underline() - Spacer() - } - .font(.caption) - .foregroundColor(.blue) - } - .padding(.vertical, 2) - } - } - } - .padding() - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(10) - } - } - } - } - } - - // Feature 3: Vision ID View - private var visionIDView: some View { - VStack(alignment: .leading, spacing: 16) { - Text("Select or Snap a Photo to Identify") - .font(.headline) - - PhotosPicker(selection: $photosPickerItem, matching: .images) { - VStack(spacing: 12) { - if let image = viewModel.selectedImage { - Image(uiImage: image) - .resizable() - .aspectRatio(contentMode: .fit) - .frame(maxHeight: 200) - .cornerRadius(12) - } else { - VStack(spacing: 8) { - Image(systemName: "photo.badge.plus") - .font(.system(size: 40)) - Text("Select an Image") - } - .frame(maxWidth: .infinity) - .frame(height: 180) - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(12) - } - } - } - .onChange(of: photosPickerItem) { oldItem, newItem in - Task { - if let data = try? await newItem?.loadTransferable(type: Data.self), - let image = UIImage(data: data) { - viewModel.selectedImage = image - await viewModel.identifySelectedImage() - } - } - } - - if let identified = viewModel.identifiedObject { - VStack(alignment: .leading, spacing: 12) { - HStack { - Text("Identification Result") - .font(.headline) - Spacer() - modelIndicatorView - } - - VStack(alignment: .leading, spacing: 8) { - HStack { - Text("Name:") - .bold() - Text(identified.name) - } - HStack { - Text("Category:") - .bold() - Text(identified.category) - } - VStack(alignment: .leading, spacing: 4) { - Text("Description:") - .bold() - Text(identified.description) - .foregroundColor(.secondary) - } - } - .padding() - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color(.secondarySystemGroupedBackground)) - .cornerRadius(10) - } - .transition(.opacity) - } - } - } - - // Helper to pick icons for activity kinds - private func getSymbol(for kind: ActivityKind?) -> String { - guard let kind = kind else { return "info.circle" } - switch kind { - case .sightseeing: return "binoculars.fill" - case .foodAndDining: return "fork.knife" - case .shopping: return "bag.fill" - case .hotelAndLodging: return "bed.double.fill" - } - } -} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/HybridAIScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/HybridAIScreen.swift new file mode 100644 index 000000000..45b4fa64f --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/HybridAIScreen.swift @@ -0,0 +1,30 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +@available(iOS 27.0, *) +struct HybridAIScreen: View { + @StateObject private var viewModel: HybridAIViewModel + + init(backendType: BackendOption) { + _viewModel = StateObject(wrappedValue: HybridAIViewModel(backendType: backendType)) + } + + var body: some View { + FoundationModelsContainer(viewModel: viewModel, title: "Hybrid AI") { vm in + HybridAIView(viewModel: vm) + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/VisionIDScreen.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/VisionIDScreen.swift new file mode 100644 index 000000000..83b133465 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Screens/VisionIDScreen.swift @@ -0,0 +1,32 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI +import PhotosUI + +@available(iOS 27.0, *) +struct VisionIDScreen: View { + @StateObject private var viewModel: VisionIDViewModel + @State private var photosPickerItem: PhotosPickerItem? = nil + + init(backendType: BackendOption) { + _viewModel = StateObject(wrappedValue: VisionIDViewModel(backendType: backendType)) + } + + var body: some View { + FoundationModelsContainer(viewModel: viewModel, title: "Vision ID") { vm in + VisionIDView(viewModel: vm, photosPickerItem: $photosPickerItem) + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift deleted file mode 100644 index e2efbc92b..000000000 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Tools/FindLocalPlacesTool.swift +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import Foundation -import FoundationModels -import FirebaseAILogic - -@available(iOS 26.0, *) -@Generable -public struct PlacesResponse: Codable { - public let places: [String] -} - -@available(iOS 26.0, *) -@Generable -public struct ToolResult: Codable { - public let summary: String - public let attributions: [Attribution] -} - -@available(iOS 26.0, *) -public final class FindLocalPlacesTool { - public let name = "findLocalPlaces" - public let description = "Finds points of interest and local businesses matching a query for a specified destination." - - public init() {} - - @available(iOS 26.0, *) - @Generable - public struct Arguments { - @Guide(description: "The destination city/place to look up for (e.g. 'Paris', 'New York').") - public let destination: String - - @Guide(description: "The category or query to search (e.g. 'hotels', 'Italian restaurants', 'museums').") - public let category: String - } - - public func call(arguments: Arguments) async throws -> ToolResult { - print("FindLocalPlacesTool: call called with destination: \(arguments.destination), category: \(arguments.category)") - - // Setup Firebase AI with Google Maps grounding - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) - let session = ai.generativeModelSession( - model: "gemini-3.1-flash-lite", - tools: [Tool.googleMaps()] - ) - - let prompt = "Find 3 real popular \(arguments.category) in \(arguments.destination). Use Google Maps to make sure the places are real and currently open." - - do { - let response = try await session.respond( - to: prompt, - generating: PlacesResponse.self - ) - - var attributions: [Attribution] = [] - if let metadata = response.rawResponse.candidates.first?.groundingMetadata { - for chunk in metadata.groundingChunks { - if let maps = chunk.maps, let title = maps.title, let url = maps.url?.absoluteString { - attributions.append(Attribution(title: title, url: url)) - } - } - } - - let results = response.content.places - print("FindLocalPlacesTool: found places: \(results.joined(separator: ", "))") - - return ToolResult( - summary: "Here are some popular \(arguments.category) in \(arguments.destination): \(results.joined(separator: ", "))", - attributions: attributions - ) - } catch { - print("FindLocalPlacesTool error: \(error.localizedDescription)") - throw error - } - } -} - -// Struct wrapper to conform to FoundationModels.Tool AND ToolRepresentable -@available(iOS 26.0, *) -public struct FindLocalPlacesToolWrapper: FoundationModels.Tool, ToolRepresentable, @unchecked Sendable { - let tool: FindLocalPlacesTool - - public var name: String { tool.name } - public var description: String { tool.description } - - public typealias Arguments = FindLocalPlacesTool.Arguments - public typealias Output = ToolResult - - public func call(arguments: Arguments) async throws -> ToolResult { - try await tool.call(arguments: arguments) - } -} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift deleted file mode 100644 index 6f4d527ae..000000000 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/AppleAIViewModel.swift +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import Foundation -import SwiftUI -import Combine -import FoundationModels -import FirebaseAILogic - -@available(iOS 27.0, *) -@MainActor -public final class AppleAIViewModel: ObservableObject { - // Shared state - @Published public var inProgress = false - @Published public var error: Error? - - // Feature 1: Hybrid Summary/Translation - @Published public var inputText: String = "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." - @Published public var outputSummary: TextSummary? - @Published public var isUsingLocalModel: Bool = false - - // Feature 2: Smart Planner - @Published public var destination: String = "San Francisco" - @Published public var interests: String = "tech history, good coffee, bay views" - @Published public var itinerary: Itinerary.PartiallyGenerated? - - // Feature 3: Visual Identifier - @Published public var selectedImage: UIImage? - @Published public var identifiedObject: IdentifiedObject? - - @Published public var forceCloudModel: Bool = false - - private var activeTask: Task? - - public init() {} - - public func stopActiveTask() { - activeTask?.cancel() - activeTask = nil - inProgress = false - } - - // MARK: - Feature 1: Hybrid Summary/Translation - public func runSummarization() async { - stopActiveTask() - inProgress = true - error = nil - outputSummary = nil - - activeTask = Task { - defer { self.inProgress = false } - - let instructions = Instructions { - "Your job is to summarize the provided text in exactly 2 bullet points." - } - - let availability = SystemLanguageModel.default.availability - - // Try local model first if it reports available and not forced to cloud - if !forceCloudModel && availability == .available { - isUsingLocalModel = true - do { - let session = LanguageModelSession( - model: SystemLanguageModel.default, - instructions: instructions - ) - let response = try await session.respond(to: inputText, generating: TextSummary.self) - if !Task.isCancelled { - self.outputSummary = response.content - } - return - } catch { - // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) - if error.isMLAssetUnavailable { - print("Local ML assets unavailable. Falling back to cloud model...") - } else { - print("Local model failed: \(error.localizedDescription). Falling back to cloud model...") - } - } - } - - // Fallback to cloud model - isUsingLocalModel = false - do { - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) - let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") - let session = LanguageModelSession( - model: model, - instructions: instructions - ) - let response = try await session.respond(to: inputText, generating: TextSummary.self) - if !Task.isCancelled { - self.outputSummary = response.content - } - } catch { - if !Task.isCancelled { - // If even the cloud model via LanguageModelSession fails (likely due to missing guardrail assets), - // we show a descriptive error message. - if let mlMessage = error.mlAssetErrorMessage { - print("ML Asset Error detected in cloud fallback: \(mlMessage)") - // We can either set the error here or try one last direct fallback. - // Let's set a custom error that wraps the message. - self.error = NSError(domain: "FirebaseAIExample", - code: 1, - userInfo: [NSLocalizedDescriptionKey: mlMessage]) - } else { - self.error = error - } - } - } - } - } - - // MARK: - Feature 2: Smart Planner - public func generateItinerary() async { - stopActiveTask() - inProgress = true - error = nil - itinerary = nil - isUsingLocalModel = false - - activeTask = Task { - defer { self.inProgress = false } - - let localPlacesTool = FindLocalPlacesToolWrapper(tool: FindLocalPlacesTool()) - let instructions = Instructions { - "Your job is to create a structured 1-day itinerary for the user." - "Ensure you use the findLocalPlaces tool to search for real places and recommendations in the destination city." - } - let promptText = "Generate a structured 1-day itinerary for \(destination) focused on \(interests). Include exactly 3 activities (morning, afternoon, evening) with real recommendations. For any place returned by the findLocalPlaces tool, populate the 'attributions' array in the response with the exact name (title) and Google Maps URL (url) returned by the tool." - - let availability = SystemLanguageModel.default.availability - - // Try local model first if it reports available and not forced to cloud - if !forceCloudModel && availability == .available { - isUsingLocalModel = true - do { - let session = LanguageModelSession( - model: SystemLanguageModel.default, - tools: [localPlacesTool], - instructions: instructions - ) - let stream = session.streamResponse( - to: promptText, - generating: Itinerary.self - ) - for try await response in stream { - if Task.isCancelled { break } - self.itinerary = response.content - print(response.content) - } - return - } catch { - // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) - if error.isMLAssetUnavailable { - print("Local ML assets unavailable for Planner. Falling back to cloud model...") - } else { - print("Local model failed for Planner: \(error.localizedDescription). Falling back to cloud model...") - } - } - } - - // Fallback to cloud model - isUsingLocalModel = false - do { - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) - let model = ai.geminiLanguageModel( - name: "gemini-3.5-flash" - ) - let session = LanguageModelSession( - model: model, - tools: [localPlacesTool], - instructions: instructions - ) - let stream = session.streamResponse( - to: promptText, - generating: Itinerary.self - ) - for try await response in stream { - if Task.isCancelled { break } - self.itinerary = response.content - } - } catch { - if !Task.isCancelled { - self.error = error - } - } - } - } - - // MARK: - Feature 3: Visual Identifier - public func identifySelectedImage() async { - guard let image = selectedImage else { return } - stopActiveTask() - inProgress = true - error = nil - identifiedObject = nil - isUsingLocalModel = false - - activeTask = Task { - defer { self.inProgress = false } - - guard let cgImage = image.cgImage, - let imageData = image.jpegData(compressionQuality: 0.8) else { return } - - let instructions = Instructions { - "You are a visual object identifier." - } - - let availability = SystemLanguageModel.default.availability - - // Try local model first if it reports available and not forced to cloud - if !forceCloudModel && availability == .available { - isUsingLocalModel = true - do { - let session = LanguageModelSession( - model: SystemLanguageModel.default, - instructions: instructions - ) - let response = try await session.respond( - generating: IdentifiedObject.self - ) { - "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." - Attachment(cgImage) - } - if !Task.isCancelled { - self.identifiedObject = response.content - } - return - } catch { - // Fall back to cloud model if local model fails (e.g. assets not downloaded on simulator) - if error.isMLAssetUnavailable { - print("Local ML assets unavailable for Vision ID. Falling back to cloud model...") - } else { - print("Local model failed for Vision ID: \(error.localizedDescription). Falling back to cloud model...") - } - } - } - - // Fallback to cloud model - isUsingLocalModel = false - do { - let ai = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) - let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - let session = LanguageModelSession( - model: model, - instructions: instructions - ) - let response = try await session.respond( - generating: IdentifiedObject.self - ) { - "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." - InlineDataPart(data: imageData, mimeType: "image/jpeg") - } - - if !Task.isCancelled { - self.identifiedObject = response.content - } - } catch { - if !Task.isCancelled { - self.error = error - } - } - } - } -} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift new file mode 100644 index 000000000..e9087cfd4 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift @@ -0,0 +1,58 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +public enum ModelPreference: String, CaseIterable, Identifiable { + case auto = "Auto (Local First)" + case cloud = "Cloud Only" + + public var id: String { rawValue } +} + +@available(iOS 27.0, *) +@MainActor +public class FoundationModelsBaseViewModel: ObservableObject { + @Published public var inProgress = false + @Published public var error: Error? + @Published public var modelPreference: ModelPreference = .auto + @Published public var isUsingLocalModel: Bool = false + + internal var activeTask: Task? + public let backendType: BackendOption + + public init(backendType: BackendOption) { + self.backendType = backendType + } + + public func stopActiveTask() { + activeTask?.cancel() + activeTask = nil + inProgress = false + } + + internal func getFirebaseAI() -> FirebaseAI { + switch backendType { + case .googleAI: + return FirebaseAI.firebaseAI(backend: .googleAI()) + case .vertexAI: + // Using "global" as location for Foundation Models fallback compatibility + return FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift new file mode 100644 index 000000000..30ed11e8f --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift @@ -0,0 +1,100 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +@MainActor +public final class HybridAIViewModel: FoundationModelsBaseViewModel { + @Published public var inputText: String = + "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." + @Published public var outputSummary: TextSummary? + + public func runSummarization() async { + stopActiveTask() + inProgress = true + error = nil + outputSummary = nil + + activeTask = Task { + defer { self.inProgress = false } + + let instructions = Instructions { + "Your job is to summarize the provided text in exactly 2 bullet points." + } + + let availability = SystemLanguageModel.default.availability + + // Try local model first if it reports available and not forced to cloud + if modelPreference == .auto, availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: instructions + ) + let response = try await session.respond( + to: inputText, + generating: TextSummary.self + ) + if !Task.isCancelled { + self.outputSummary = response.content + } + return + } catch { + // Fall back to cloud model if local model fails + if error.isMLAssetUnavailable { + print("Local ML assets unavailable. Falling back to cloud model...") + } else { + print( + "Local model failed: \(error.localizedDescription). Falling back to cloud model..." + ) + } + } + } + + // Fallback to cloud model + isUsingLocalModel = false + do { + let ai = getFirebaseAI() + let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-lite") + let session = LanguageModelSession( + model: model, + instructions: instructions + ) + let response = try await session.respond( + to: inputText, + generating: TextSummary.self + ) + if !Task.isCancelled { + self.outputSummary = response.content + } + } catch { + if !Task.isCancelled { + if let mlMessage = error.mlAssetErrorMessage { + self.error = NSError(domain: "FirebaseAIExample", + code: 1, + userInfo: [NSLocalizedDescriptionKey: mlMessage]) + } else { + self.error = error + } + } + } + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift new file mode 100644 index 000000000..0cab4c461 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import Combine +import FoundationModels +import FirebaseAILogic + +@available(iOS 27.0, *) +@MainActor +public final class VisionIDViewModel: FoundationModelsBaseViewModel { + @Published public var selectedImage: UIImage? + @Published public var identifiedObject: IdentifiedObject? + + public func identifySelectedImage() async { + guard let image = selectedImage else { return } + stopActiveTask() + inProgress = true + error = nil + identifiedObject = nil + isUsingLocalModel = false + + activeTask = Task { + defer { self.inProgress = false } + + guard let cgImage = image.cgImage, + let imageData = image.jpegData(compressionQuality: 0.8) else { return } + + let instructions = Instructions { + "You are a visual object identifier." + } + + let availability = SystemLanguageModel.default.availability + + // Try local model first if it reports available and not forced to cloud + if modelPreference == .auto, availability == .available { + isUsingLocalModel = true + do { + let session = LanguageModelSession( + model: SystemLanguageModel.default, + instructions: instructions + ) + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + Attachment(cgImage) + } + if !Task.isCancelled { + self.identifiedObject = response.content + } + return + } catch { + if error.isMLAssetUnavailable { + print( + "Local ML assets unavailable for Vision ID. Falling back to cloud model..." + ) + } else { + print( + "Local model failed for Vision ID: \(error.localizedDescription). Falling back to cloud model..." + ) + } + } + } + + // Fallback to cloud model + isUsingLocalModel = false + do { + let ai = getFirebaseAI() + let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + let session = LanguageModelSession( + model: model, + instructions: instructions + ) + let response = try await session.respond( + generating: IdentifiedObject.self + ) { + "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." + InlineDataPart(data: imageData, mimeType: "image/jpeg") + } + + if !Task.isCancelled { + self.identifiedObject = response.content + } + } catch { + if !Task.isCancelled { + self.error = error + } + } + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/FoundationModelsContainer.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/FoundationModelsContainer.swift new file mode 100644 index 000000000..40c2f09ad --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/FoundationModelsContainer.swift @@ -0,0 +1,72 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +@available(iOS 27.0, *) +struct FoundationModelsContainer: View { + @ObservedObject var viewModel: VM + @State private var presentErrorDetails = false + let title: String + let content: (VM) -> Content + + var body: some View { + ZStack { + ScrollView { + content(viewModel) + .padding() + .frame(maxWidth: .infinity, alignment: .topLeading) + } + + if viewModel.inProgress { + ProgressOverlay() + } + } + .background(Color(.systemGroupedBackground)) + .navigationTitle(title) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItemGroup(placement: .topBarTrailing) { + if viewModel.inProgress { + Button(action: { + viewModel.stopActiveTask() + }) { + Image(systemName: "stop.circle") + .font(.title3) + } + } + + Menu { + Picker("Model Preference", selection: $viewModel.modelPreference) { + ForEach(ModelPreference.allCases) { pref in + Text(pref.rawValue).tag(pref) + } + } + } label: { + Image(systemName: "cpu") + } + } + } + .sheet(isPresented: $presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .onChange(of: viewModel.error != nil) { oldValue, newValue in + if newValue { + presentErrorDetails = true + } + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift new file mode 100644 index 000000000..0185b4271 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift @@ -0,0 +1,80 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +@available(iOS 27.0, *) +struct HybridAIView: View { + @ObservedObject var viewModel: HybridAIViewModel + + var body: some View { + VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 8) { + Text("Input Text") + .font(.headline) + TextEditor(text: $viewModel.inputText) + .frame(height: 120) + .padding(4) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(8) + } + + Button(action: { + Task { + await viewModel.runSummarization() + } + }) { + HStack { + Spacer() + Image(systemName: "sparkles") + Text("Summarise") + Spacer() + } + .padding() + .foregroundColor(.white) + .background(Color.blue) + .cornerRadius(10) + } + .disabled(viewModel.inProgress) + + if let summary = viewModel.outputSummary { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Summary Points") + .font(.headline) + Spacer() + + ModelIndicatorView(isUsingLocalModel: viewModel.isUsingLocalModel) + } + + VStack(alignment: .leading, spacing: 8) { + ForEach(summary.summaryPoints, id: \.self) { point in + HStack(alignment: .top, spacing: 6) { + Text("•") + .bold() + Text(point) + .font(.subheadline) + } + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity.combined(with: .slide)) + } + } + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/ModelIndicatorView.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/ModelIndicatorView.swift new file mode 100644 index 000000000..4901872fd --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/ModelIndicatorView.swift @@ -0,0 +1,32 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +struct ModelIndicatorView: View { + let isUsingLocalModel: Bool + + var body: some View { + HStack(spacing: 4) { + Image(systemName: isUsingLocalModel ? "iphone" : "cloud.fill") + Text(isUsingLocalModel ? "Local (Apple)" : "Cloud (Gemini)") + } + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(isUsingLocalModel ? Color.green.opacity(0.2) : Color.purple.opacity(0.2)) + .foregroundColor(isUsingLocalModel ? .green : .purple) + .cornerRadius(8) + } +} diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift new file mode 100644 index 000000000..0e00d2455 --- /dev/null +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift @@ -0,0 +1,99 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI +import PhotosUI + +@available(iOS 27.0, *) +@MainActor +struct VisionIDView: View { + @ObservedObject var viewModel: VisionIDViewModel + @Binding var photosPickerItem: PhotosPickerItem? + + @MainActor + var body: some View { + let selectedImage = viewModel.selectedImage + + VStack(alignment: .leading, spacing: 16) { + Text("Select or Snap a Photo to Identify") + .font(.headline) + + PhotosPicker(selection: $photosPickerItem, matching: .images) { + VStack(spacing: 12) { + if let image = selectedImage { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxHeight: 200) + .cornerRadius(12) + } else { + VStack(spacing: 8) { + Image(systemName: "photo.badge.plus") + .font(.system(size: 40)) + Text("Select an Image") + } + .frame(maxWidth: .infinity) + .frame(height: 180) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(12) + } + } + } + .onChange(of: photosPickerItem) { oldItem, newItem in + Task { + if let data = try? await newItem?.loadTransferable(type: Data.self), + let image = UIImage(data: data) { + viewModel.selectedImage = image + await viewModel.identifySelectedImage() + } + } + } + + if let identified = viewModel.identifiedObject { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Identification Result") + .font(.headline) + Spacer() + ModelIndicatorView(isUsingLocalModel: viewModel.isUsingLocalModel) + } + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Name:") + .bold() + Text(identified.name) + } + HStack { + Text("Category:") + .bold() + Text(identified.category) + } + VStack(alignment: .leading, spacing: 4) { + Text("Description:") + .bold() + Text(identified.description) + .foregroundColor(.secondary) + } + } + .padding() + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemGroupedBackground)) + .cornerRadius(10) + } + .transition(.opacity) + } + } + } +} diff --git a/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift b/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift index 0731fba0c..d0b26d063 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift @@ -14,9 +14,9 @@ import Foundation -enum BackendOption: String, CaseIterable, Identifiable { +public enum BackendOption: String, CaseIterable, Identifiable { case googleAI = "Google AI" case vertexAI = "Firebase Vertex AI" - var id: String { rawValue } + public var id: String { rawValue } } diff --git a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift index 4f0129345..d69fd2595 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/Sample.swift @@ -34,6 +34,7 @@ public struct Sample: Identifiable { public let liveGenerationConfig: LiveGenerationConfig? public let fileDataParts: [FileDataPart]? public let tip: InlineTip? + public let isFoundationModel: Bool public init(title: String, description: String, @@ -47,7 +48,8 @@ public struct Sample: Identifiable { generationConfig: GenerationConfig? = nil, liveGenerationConfig: LiveGenerationConfig? = nil, fileDataParts: [FileDataPart]? = nil, - tip: InlineTip? = nil) { + tip: InlineTip? = nil, + isFoundationModel: Bool = false) { self.title = title self.description = description self.useCases = useCases @@ -61,6 +63,7 @@ public struct Sample: Identifiable { self.liveGenerationConfig = liveGenerationConfig self.fileDataParts = fileDataParts self.tip = tip + self.isFoundationModel = isFoundationModel } } @@ -310,10 +313,18 @@ extension Sample { tip: InlineTip(text: "Try asking the model to change the background color"), ), Sample( - title: "Apple Foundation Models", - description: "Demonstrates how to integrate Firebase AI Logic (Gemini) with Apple's Foundation Models framework (Apple Intelligence).", - useCases: [.text, .image, .functionCalling], - navRoute: "AppleAIScreen" + title: "Hybrid AI", + description: "Summarize and translate text using local Foundation Models (Apple Intelligence) with automatic cloud fallback.", + useCases: [.text], + navRoute: "FoundationModelsHybrid", + isFoundationModel: true + ), + Sample( + title: "Vision ID", + description: "Identify objects and landmarks in images using multimodal Foundation Models.", + useCases: [.image], + navRoute: "FoundationModelsVisionID", + isFoundationModel: true ), ] diff --git a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift index bfaf543f7..079284e4a 100644 --- a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift +++ b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift @@ -16,57 +16,58 @@ import Foundation import FoundationModels extension Error { - /// Returns true if this error indicates that required ML assets (like safety guardrails) - /// are missing from the device or simulator. - public var isMLAssetUnavailable: Bool { - // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable - if #available(iOS 26.0, *) { - if let genError = self as? LanguageModelSession.GenerationError { - if case .assetsUnavailable = genError { - return true - } - } + /// Returns true if this error indicates that required ML assets (like safety guardrails) + /// are missing from the device or simulator. + public var isMLAssetUnavailable: Bool { + // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable + if #available(iOS 26.0, *) { + if let genError = self as? LanguageModelSession.GenerationError { + if case .assetsUnavailable = genError { + return true } - - let nsError = self as NSError - - // 2. Check for the specific SensitiveContentAnalysisML error (Code 15) - if nsError.domain == "com.apple.SensitiveContentAnalysisML" && nsError.code == 15 { - return true - } - - // 3. Check for underlying GenerativeFunctionsFoundation error (Code 1020000) - // This is often found in the UserInfo of higher-level errors. - if nsError.domain == "com.apple.GenerativeFunctionsFoundation.GenerativeError" && nsError.code == 1020000 { - return true - } - - // 4. Recursive check for underlying errors - if let underlying = nsError.userInfo[NSUnderlyingErrorKey] as? Error { - return underlying.isMLAssetUnavailable - } - - return false + } + } + + let nsError = self as NSError + + // 2. Check for the specific SensitiveContentAnalysisML error (Code 15) + if nsError.domain == "com.apple.SensitiveContentAnalysisML", nsError.code == 15 { + return true + } + + // 3. Check for underlying GenerativeFunctionsFoundation error (Code 1020000) + // This is often found in the UserInfo of higher-level errors. + if nsError.domain == "com.apple.GenerativeFunctionsFoundation.GenerativeError", + nsError.code == 1_020_000 { + return true } - - /// Returns a user-friendly message for ML asset errors, particularly for simulators. - public var mlAssetErrorMessage: String? { - guard isMLAssetUnavailable else { return nil } - - #if targetEnvironment(simulator) - return """ - Apple Intelligence assets are missing in this simulator. - - To fix this: - 1. Open Settings in the simulator. - 2. Go to Apple Intelligence & Siri. - 3. Toggle Apple Intelligence ON. - 4. Wait for assets to download (check the status in Settings). - - Alternatively, use a physical device with Apple Intelligence support. - """ - #else - return "Apple Intelligence assets are not yet ready on this device. Please ensure Apple Intelligence is enabled in Settings and that model assets have finished downloading." - #endif + + // 4. Recursive check for underlying errors + if let underlying = nsError.userInfo[NSUnderlyingErrorKey] as? Error { + return underlying.isMLAssetUnavailable } + + return false + } + + /// Returns a user-friendly message for ML asset errors, particularly for simulators. + public var mlAssetErrorMessage: String? { + guard isMLAssetUnavailable else { return nil } + + #if targetEnvironment(simulator) + return """ + Apple Intelligence assets are missing in this simulator. + + To fix this: + 1. Open Settings in the simulator. + 2. Go to Apple Intelligence & Siri. + 3. Toggle Apple Intelligence ON. + 4. Wait for assets to download (check the status in Settings). + + Alternatively, use a physical device with Apple Intelligence support. + """ + #else + return "Apple Intelligence assets are not yet ready on this device. Please ensure Apple Intelligence is enabled in Settings and that model assets have finished downloading." + #endif + } } diff --git a/firebaseai/FirebaseDocs.md b/firebaseai/FirebaseDocs.md new file mode 100644 index 000000000..4b4fabb17 --- /dev/null +++ b/firebaseai/FirebaseDocs.md @@ -0,0 +1,625 @@ +### Guiding Principles + +* **Prioritize a continuous narrative:** Keep the developer in the flow of the code rather than forcing them to jump to external documentation for core configuration steps. ([thread](https://docs.google.com/document/d/1zBitr-qR9cqG0tt1c8AYFrafMM2FcGkJvFVCv2D_SDs/edit?disco=AAAB7xVQlE0)) +* **Balance "Self-contained" vs. "Linked":** Provide the "essential" steps (the specific Fuji-related tweaks) directly in the guide to avoid fragmentation. +* **Contextual linking:** Link to authoritative documentation only the first time a feature (e.g., Firebase App Check, AI Logic) is mentioned. Subsequent mentions should focus on the implementation flow. +* **Code transparency:** Code snippets must reflect incremental progress that is in sync with configuration steps (e.g., create app on console, then configure Firebase in app). +* **Compile-ready:** The app should be in a state that compiles and runs at each major step. +* **Minimize context switching:** Avoid forcing unnecessary transitions between Xcode and the Firebase console. + +### TODOs + +* Identify where caveats should be added for readers extending an existing app. + +### Outline + +To view the current outline, select the overflow menu (three dots) on the current document tab and select *Show Outline*. +![][image1] + +### Key Decisions + +* **DevSite Location:** Put it under [https://firebase.google.com/docs/ai-logic/**solutions**/overview](https://firebase.google.com/docs/ai-logic/solutions/overview) +* **Title:** [See resolved thread](https://docs.google.com/document/d/1zBitr-qR9cqG0tt1c8AYFrafMM2FcGkJvFVCv2D_SDs/edit?disco=AAAB83hTjGU). + * Refer to the feature like “this integration” +* **Branch Name:** `wwdc26-preview` + * Thread for if we run into issues with the branch not updating for clients ([thread](https://docs.google.com/document/d/1zBitr-qR9cqG0tt1c8AYFrafMM2FcGkJvFVCv2D_SDs/edit?disco=AAAB83oWffw)). + +--- + +**BEGIN PUBLIC CONTENT** + +# **\[page\] Get started: Access the Gemini API through Apple's Foundation Models framework** + +**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. +Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. + +\ \ + +This guide shows you how to get started using the Firebase AI Logic integration to access the Gemini API through Apple's Foundation Models framework. + +To protect access to Gemini models, this guide also shows you how to set up \[Firebase App Check\]([https://firebase.google.com/docs/app-check](https://firebase.google.com/docs/app-check)), which is ***critical*** even during development. + +## Prerequisites + +* Install the latest \[Xcode 27 beta\]([https://developer.apple.com/download/all/?q=Xcode](https://developer.apple.com/download/all/?q=Xcode)). +* An Apple platform simulator or a physical device, both running the corresponding beta OS version (for example, iOS 27 beta). +* A new Xcode project of an Apple platforms app using a **SwiftUI interface**. + +### Supported Gemini models + +The integration with Apple’s Foundation Models framework supports the following Gemini models. + +* General purpose models + * `gemini-3.1-pro-preview` + * `gemini-3.5-flash` + * `gemini-3.1-flash-lite` +* Image-generating models + * `gemini-3-pro-image-preview` (aka "Nano Banana Pro") + * `gemini-3.1-flash-image-preview` (aka "Nano Banana 2") + * `gemini-2.5-flash-image` (aka "Nano Banana") + +Gemini Live API models and Imagen models are ***not*** supported. Note that Gemini 2.5 models are technically supported, but they're not recommended for new projects and require special configuration not covered in these guides. + +## Step 1: Create a Firebase project + +We recommend starting with a new Firebase project to explore this integration. + +1. Sign into the \[Firebase console\]([https://console.firebase.google.com/](https://console.firebase.google.com/)). +2. Click **Create a new Firebase project**. +3. Follow the on-screen instructions. You do **not** need to enable Google Analytics. + +## Step 2: Connect your app to Firebase + +To connect your app to Firebase, you must register it with your Firebase project and add a configuration file to your codebase. + +1. In the center of the \[project overview\]([https://console.firebase.google.com/project/\_/overview](https://console.firebase.google.com/project/_/overview)) page, click the **iOS+** icon to launch the setup workflow. +2. Register your app: + 1. Enter your app's **bundle ID**. Make sure it matches the bundle ID of the project you're building in Xcode. + 2. Click **Register app**. +3. Add the Firebase configuration file. + This file contains the settings for the Firebase SDK to connect to your Firebase project. + 1. Click **Download GoogleService-Info.plist** to get your configuration file. + 2. Move `GoogleService-Info.plist` into the root of your Xcode project and add it to all targets. + 3. Click **Next** in the Firebase console. +4. The workflow in the console provides *generic* instructions for adding the Firebase SDK to your app, so skip ahead to the next step in this guide for specific instructions for Firebase AI Logic. + +## Step 3: Add the Firebase libraries and initialize Firebase in your app + +1. Use Swift Package Manager to add the Firebase AI Logic library: + 1. In Xcode, with your app project open, select **File** \> **Add Packages**. + 2. Enter the Firebase Apple SDK repository URL: + +``` +https://github.com/firebase/firebase-ios-sdk +``` + + 3. Select the Dependency Rule as **Branch** and enter **`wwdc26-preview`**. + 4. Click **Add Package**. Xcode will resolve and download the dependencies. + 5. When prompted, add the **`FirebaseAILogic`** library to your app target. +2. Initialize Firebase when your app starts up by adding the following code to your app's main entry point: + +```swift +import SwiftUI +import FirebaseCore + +@main +struct YourApp: App { init() { + FirebaseApp.configure() + } + + var body: some Scene { + WindowGroup { + NavigationView { + ContentView() + } + } + } +} +``` + +## Step 4: Enable and secure Firebase services + +Now that your app is configured to use Firebase, you need to enable the Firebase AI Logic service and protect access to its associated APIs using Firebase App Check. + +### Set up **Firebase AI Logic** in your Firebase project + +1. In the Firebase console, go to **AI Services** \> [**AI Logic**](https://console.firebase.google.com/project/_/ailogic/?useAutoProject=true). +2. Click **Get started** to launch the setup workflow. +3. We recommend choosing the **Gemini Developer API** provider to get started quickly at no cost. + +### Set up **Firebase App Check** in your Firebase project + +**Using Firebase App Check to protect the APIs associated with Firebase AI Logic is *critical*.** When enforced, Firebase App Check only allows incoming requests that are from your actual app and an untampered device. Firebase App Check supports many attestation providers, including Apple's \[App Attest\]([https://developer.apple.com/documentation/devicecheck](https://developer.apple.com/documentation/devicecheck)). + +The following steps are for a baseline, default setup for App Check. Learn more about [additional configuration options for App Check](https://firebase.google.com/docs/ai-logic/app-check) (like adjusting the TTL of tokens and enabling limited-use tokens). + +\[\!TIP\] **Developing a Mac app?** If you're developing a macOS app, skip the below steps and use the \[DeviceCheck provider\]([https://firebase.google.com/docs/app-check/ios/devicecheck-provider](https://firebase.google.com/docs/app-check/ios/devicecheck-provider)) instead of App Attest, which is unsupported on macOS. + +Here's how to register the App Attest provider in the Firebase console: + +1. In the Firebase console, go to **Security** \> \[**App Check\](**[https://console.firebase.google.com/project/\_/appcheck/](https://console.firebase.google.com/project/_/appcheck/)**)**. +2. Click **Get started**. +3. In the **\[Apps** tab**\](**[https://console.firebase.google.com/project/\_/appcheck/apps/](https://console.firebase.google.com/project/_/appcheck/apps/)**)**, register your app to use App Check with the \[**App Attest** provider\]([https://firebase.google.com/docs/app-check/ios/app-attest-provider](https://firebase.google.com/docs/app-check/ios/app-attest-provider)). +4. In the \[**APIs** tab\]([https://console.firebase.google.com/project/\_/appcheck/products/](https://console.firebase.google.com/project/_/appcheck/products/)), select **Firebase AI Logic**, and click **Enforce**. + +**(+) Before distributing your app, configure the App Attest capability** + +**Note:** If you’re going to start with the App Check debug provider (next section), these steps aren't required until you distribute your app. For now, you can skip ahead to the next section. + +1. In Xcode, add the \[App Attest capability\]([https://developer.apple.com/documentation/xcode/adding-capabilities-to-your-app](https://developer.apple.com/documentation/xcode/adding-capabilities-to-your-app)) to your app. +2. In your Xcode project's `.entitlements` file, set the \[App Attest environment\]([https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.devicecheck.appattest-environment](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.devicecheck.appattest-environment)) to `production`. + +### Configure the App Check Debug Provider for local development + +For local development, configure the \[App Check Debug Provider\]([https://firebase.google.com/docs/app-check/ios/debug-provider](https://firebase.google.com/docs/app-check/ios/debug-provider)). Setting up this provider bypasses attestation during development so you can verify your app's logic without modifying the production security configuration you set up above. + +1. In your Xcode project, import `FirebaseAppCheck` and initialize App Check with the debug provider factory before you configure `FirebaseApp`. + +```swift +import SwiftUI +import FirebaseCore +import FirebaseAppCheck + +@main +struct YourApp: App { + init() { + let providerFactory = AppCheckDebugProviderFactory() + AppCheck.setAppCheckProviderFactory(providerFactory) + FirebaseApp.configure() + } + + var body: some Scene { + WindowGroup { + NavigationView { + ContentView() + } + } + } +} + +``` + +2. Obtain your debug token: + 1. Launch your app in the simulator or on your test device. + 2. Open the Xcode console and look for the App Check debug token. Because it is generated at app startup, it should be one of the first few logs you see. It will look similar to this: + + \\<Warning\> \[AppCheckCore\]\[I-GAC004001\] App Check debug token: '123a4567-b89c-12d3-e456-789012345678'.\ + +1. Copy the token (for example, `123a4567-b89c-12d3-e456-789012345678`). + +3. Provide your debug token to App Check within the Firebase console: + 1. In the Firebase console, navigate to **Security** \> **App Check** \> [**Apps**](https://console.firebase.google.com/project/_/appcheck/apps/). + 2. Find your app, click the overflow menu (three dots), and select **Manage debug tokens**. + 3. Click **Add debug token**, enter a name (for example, `My Simulator`), paste the token, and click **Save**. + +For details about the debug provider (including how to get a new debug token), check out the [official App Check docs](https://firebase.google.com/docs/app-check/ios/debug-provider). + +\[\!WARNING\] **Keep your debug token private.** Because this debug token lets anyone access your Firebase AI Logic resources without a valid device, you must keep it private. Do not commit it to a public repository. If a registered token is ever compromised, revoke it immediately in the Firebase console. + +## Step 5: Initialize the AI Logic service in your app + +With Firebase and App Check configured, you can now initialize the Firebase AI Logic service in your app. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") +``` + +## Step 6: Send a request to a Gemini model + +With Firebase AI Logic setup, protected, and initialized in your app, you're ready to send a request to a Gemini model. + +The following example shows the most basic type of request – generating text from a text-only prompt: + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +// Inject the model into Apple's LanguageModelSession. +let session = LanguageModelSession(model: model) + +// Use the session to generate a text response to a prompt. +let response = try await session.respond(to: "Write a story about a magic backpack.") +print(response.content) +``` + +Gemini models also support other types of requests, like [analyzing images and PDFs](#generate-text-from-multimodal-input-\(like-images\)), [generating structured JSON output](#generate-structured-output-\(like-json\)), and [generating and editing images](#generate-and-edit-images-\(using-"nano-banana"-models\)) (using "Nano Banana" models). See examples for these types of requests in the [docs](#[page]-available-capabilities-when-accessing-the-gemini-api-through-apple's-foundation-models-framework) or in the sample app. + +### Stream the response + +You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +// Inject the model into Apple's LanguageModelSession. +let session = LanguageModelSession(model: model) + +// Use the session to generate a streamed text response to a prompt. +let response = try await session.streamResponse(to: "Write a story about a magic backpack.") +print(response.content) +``` + +## Next steps + +* Check out the sample app or a video to see this integration in a real-world app. +* Explore the [available capabilities](#[page]-available-capabilities-when-accessing-the-gemini-api-through-apple's-foundation-models-framework) when accessing the Gemini API through Apple's Foundation Models framework. +* Learn how to [configure the model](#[page]-configuration-options-when-accessing-the-gemini-api-through-apple's-foundation-models-framework) to control its responses, including setting a thinking level. +* Learn how to [provide tools to the model](#[page]-provide-tools-to-the-model-when-accessing-the-gemini-api-through-apple's-foundation-models-framework), including grounding for up-to-date information. +* Read more about \[Firebase App Check\]([https://firebase.google.com/docs/ai-logic/app-check](https://firebase.google.com/docs/ai-logic/app-check)) and how it protects your resources. + +# **\[page\] Available capabilities when accessing the Gemini API through Apple's Foundation Models framework** {#[page]-available-capabilities-when-accessing-the-gemini-api-through-apple's-foundation-models-framework} + +**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. +Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. + +This guide shows you how to send various types of requests to the Gemini API through Apple's Foundation Models framework using the Firebase AI Logic SDK for Apple platforms. + +The examples on this page assume that you've completed the [Get started: Access the Gemini API through Apple's Foundation Models framework](https://firebase.devsite.corp.google.com/docs/ai-logic/apple-foundation-models-framework/get-started). + +This page shows examples for how to send the following types of requests: + +* [Generate text from text-only input](#generate-text-from-text-only-input) +* Generate text from a multi-turn session (chat) +* [Generate text from multimodal input (like images)](#generate-text-from-multimodal-input-\(like-images\)) +* [Generate and edit images](#generate-and-edit-images-\(using-"nano-banana"-models\)) (using "Nano Banana" models) +* [Generate structured output (like JSON)](#generate-structured-output-\(like-json\)) + +## Generate text + +Gemini supports the following capabilities for generating text: + +* [Generate text from text-only input](#generate-text-from-text-only-input) +* Generate text from a multi-turn session (chat) +* [Generate text from multimodal input (like images)](#generate-text-from-multimodal-input-\(like-images\)) + +#### Models that support this capability + +* `gemini-3.1-pro-preview` +* `gemini-3.5-flash` +* `gemini-3.1-flash-lite` + +### Generate text from text-only input {#generate-text-from-text-only-input} + +You can ask a Gemini model to generate text by prompting with text-only input. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +// Provide a prompt that contains text. +let prompt = "Write a story about a magic backpack." + +// For a single-turn interaction, create a new session each time you call the model. +let session = LanguageModelSession(model: model) + +// Generate a text response to the prompt. +let response = try await session.respond(to: prompt) +print(response.content) +``` + +**(+) Stream the response** +You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +// Provide a prompt that contains text. +let prompt = "Write a story about a magic backpack." + +// For a single-turn interaction, create a new session each time you call the model. +let session = LanguageModelSession(model: model) + +// Generate a text response to the prompt. +// To stream the response, use `streamResponse(to:)` instead of `respond(to:)` +let response = try await session.streamResponserespond(to: prompt) +print(response.content) +``` + +### Generate text from a multi-turn session + +``` +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +// Optionally specify previous session interactions. +let transcript = Transcript(entries: [ + .prompt(.init(segments: [.text(.init(content: "Hello, I have 2 dogs in my house."))])), + .response(.init( + assetIDs: [], + segments: [.text(.init(content: "Great to meet you. What would you like to know?"))] + )) +]) + +// Create a session, optionally rehydrating from a transcript. +let session = LanguageModelSession(model: model, transcript: transcript) + +// Generate a text response to the prompt. +let response = try await session.respond(to: "How many paws are in my house?") +print(response.content) +``` + +**(+) Stream the response** +You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +TODO +``` + +### Generate text from multimodal input (like images) {#generate-text-from-multimodal-input-(like-images)} + +**Note**: In the current release for this integration, multimodal media input can be images or documents (like PDFs). Video and audio input are not yet supported. + +You can ask a Gemini model to generate text by prompting with text and a file—providing each input file's `mimeType` and the file itself. Find [requirements and recommendations for input files](https://firebase.devsite.corp.google.com/docs/ai-logic/generate-text?db=rachelsaunders&api=dev#requirements-recommendations-for-input) later on this page. + +The following example shows the basics of how to generate text from a file input by analyzing a single image file provided as inline data (base64-encoded file). + +Note that this example shows providing the file inline, but the SDKs also support [providing a file using a browser/HTTP URL](https://firebase.devsite.corp.google.com/docs/ai-logic/input-file-requirements?db=rachelsaunders). + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +TODO +``` + +**(+) Stream the response** +You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +TODO +``` + +## Generate and edit images (using "Nano Banana" models) {#generate-and-edit-images-(using-"nano-banana"-models)} + +Gemini supports the following capabilities for generating and editing images: + +* [Generate an image from text-only input](#generate-an-image-from-text-only-input) +* [Edit an image](#edit-an-image) +* [Iterate and edit images using multi-turn chat](#iterate-and-edit-images-using-multi-turn-chat) + +**Note**: Image-generating models let you optionally provide an [\`imageConfig\`](#configure-image-output) to specify aspect ratios and image sizes. + +#### Models that support this capability + +* `gemini-3-pro-image-preview` (aka "Nano Banana Pro") +* `gemini-3.1-flash-image-preview` (aka "Nano Banana 2") +* `gemini-2.5-flash-image` (aka "Nano Banana") + +### Generate an image from text-only input {#generate-an-image-from-text-only-input} + +You can ask a Gemini model to generate images by prompting with text. + +Include response modalities of `TEXT` and `IMAGE` in your model configuration (or exclude `TEXT` if you only want image output). + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini image-generating model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-image-preview") + +TODO +``` + +### Edit an image {#edit-an-image} + +You can ask a Gemini model to edit images by prompting with text and one or more images. + +Include response modalities of `TEXT` and `IMAGE` in your model configuration (or exclude `TEXT` if you only want image output). + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini image-generating model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-image-preview") + +TODO +``` + +### Iterate and edit images using multi-turn chat {#iterate-and-edit-images-using-multi-turn-chat} + +Using multi-turn chat, you can iterate with a Gemini model on the images that it generates or that you supply. + +Include response modalities of `TEXT` and `IMAGE` in your model configuration (or exclude `TEXT` if you only want image output), and call `startChat()` and `sendMessage()` to send new user messages. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini image-generating model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-image-preview") + +TODO +``` + +## Generate structured output (like JSON) {#generate-structured-output-(like-json)} + +#### Models that support this capability + +* `gemini-3.1-pro-preview` +* `gemini-3.5-flash` +* `gemini-3.1-flash-lite` + +```swift +import SwiftUI +import FirebaseCore +import FirebaseAILogic + +// Initialize the Vertex AI Gemini API backend service. +// Specify `global` as the location to access the Gemini model. +let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) +// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. +let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") + +TODO +``` + +# **\[page\] Configuration options when accessing the Gemini API through Apple's Foundation Models framework** {#[page]-configuration-options-when-accessing-the-gemini-api-through-apple's-foundation-models-framework} + +**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. +Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. + +The examples on this page assume that you've completed the [Get started: Access the Gemini API through Apple's Foundation Models framework](https://firebase.devsite.corp.google.com/docs/ai-logic/apple-foundation-models-framework/get-started). + +In each request to a model, you can send along a model configuration to control how the model generates a response. Each model offers different configuration options. + +The configuration is maintained for the lifetime of the session. If you want to use a different config, create a new session with that config. + +## System instructions + +*System instructions* are like a "preamble" that you add before the model gets exposed to any further instructions from the end user. It lets you steer the behavior of the model based on your specific needs and use cases. + +```swift +import FoundationModels +import FirebaseCore +import FirebaseAILogic + +let model = FirebaseAI.firebaseAI().geminiLanguageModel(name: "GEMINI_MODEL_NAME") + +// Specify the system instructions as part of creating the LanguageModelSession +let session = LanguageModelSession( + model: model, + instructions: "You are a cat. Your name is Neko." +) + +// ... +``` + +## Configure thinking (aka "reasoning") + +The following is a high-level description of how to configure the amount of *thinking* that a Gemini model can do. Note that Apple calls the thinking capability *reasoning*. For details, best practices, and use cases for thinking, see [Thinking](https://firebase.google.com/docs/ai-logic/thinking) in the general AI Logic docs. + +Note the following: + +* Gemini 3.x models always use thinking; you ***cannot*** disable or turn off thinking for these models. +* Gemini 3.x models always use *dynamic thinking* – the model decides when and how much it thinks up to the configured amount. + +### Set the thinking level + +```swift +TODO +``` + +### Supported thinking level values + +The following table lists the thinking level values that you can set for each model by configuring the model's `thinkingLevel`. + +\< within DevSite share the table from the thinking page \> + +## Configure image output {#configure-image-output} + +When using image-generating Gemini models (like "Nano Banana" models), you can set an \`imageConfig\`. + +```swift +TODO +``` + +\< within DevSite share the supported values from the nano banana page \> + +## General configuration + +Gemini models support some more general configuration parameters, as described in the table later in this section. Note that for Gemini 3.x models, configuring temperature, topK, and topP is not recommended. + +```swift +TODO +``` + +| Parameter | Description | Default value | +| :---- | :---- | :---- | +| Candidate count `candidateCount` | Specifies the number of response variations to return. For each request, you're charged for the output tokens of all candidates, but you're only charged once for the input tokens. Supported values: `1` \- `8` (inclusive) | `1` | +| Frequency penalty `frequencyPenalty` | Controls the probability of including tokens that repeatedly appear in the generated response. Positive values penalize tokens that repeatedly appear in the generated content, decreasing the probability of repeating content. | \--- | +| Max output tokens `maxOutputTokens` | Specifies the maximum number of tokens that can be generated in the response. | \--- | +| Presence penalty `presencePenalty` | Controls the probability of including tokens that already appear in the generated response. Positive values penalize tokens that already appear in the generated content, increasing the probability of generating more diverse content. | \--- | +| Stop sequences `stopSequences` | Specifies a list of strings that tells the model to stop generating content if one of the strings is encountered in the response. | \--- | + +# **\[page\] Provide tools to the model when accessing the Gemini API through Apple's Foundation Models framework** {#[page]-provide-tools-to-the-model-when-accessing-the-gemini-api-through-apple's-foundation-models-framework} + +**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. +Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. + +The examples on this page assume that you've completed the [Get started: Access the Gemini API through Apple's Foundation Models framework](https://firebase.devsite.corp.google.com/docs/ai-logic/apple-foundation-models-framework/get-started). From 424d6d84cba6ce4f1506fd4f6a73cc21662d5ede Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Thu, 2 Jul 2026 23:20:10 +0200 Subject: [PATCH 08/10] refactor(apple-ai): clean up unused variables, simplify error handling, and untrack docs draft --- .../ViewModels/HybridAIViewModel.swift | 8 +- .../ViewModels/VisionIDViewModel.swift | 7 +- firebaseai/FirebaseDocs.md | 625 ------------------ 3 files changed, 4 insertions(+), 636 deletions(-) delete mode 100644 firebaseai/FirebaseDocs.md diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift index 30ed11e8f..64607917e 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift @@ -86,13 +86,7 @@ public final class HybridAIViewModel: FoundationModelsBaseViewModel { } } catch { if !Task.isCancelled { - if let mlMessage = error.mlAssetErrorMessage { - self.error = NSError(domain: "FirebaseAIExample", - code: 1, - userInfo: [NSLocalizedDescriptionKey: mlMessage]) - } else { - self.error = error - } + self.error = error } } } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift index 0cab4c461..49f2d30b4 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift @@ -35,8 +35,7 @@ public final class VisionIDViewModel: FoundationModelsBaseViewModel { activeTask = Task { defer { self.inProgress = false } - guard let cgImage = image.cgImage, - let imageData = image.jpegData(compressionQuality: 0.8) else { return } + guard let cgImage = image.cgImage else { return } let instructions = Instructions { "You are a visual object identifier." @@ -56,7 +55,7 @@ public final class VisionIDViewModel: FoundationModelsBaseViewModel { generating: IdentifiedObject.self ) { "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." - Attachment(cgImage) + Attachment(cgImage).label("image") } if !Task.isCancelled { self.identifiedObject = response.content @@ -88,7 +87,7 @@ public final class VisionIDViewModel: FoundationModelsBaseViewModel { generating: IdentifiedObject.self ) { "Identify the primary object in this image. Be as specific as possible, categorize it, and provide a short 2-sentence description." - InlineDataPart(data: imageData, mimeType: "image/jpeg") + Attachment(cgImage).label("image") } if !Task.isCancelled { diff --git a/firebaseai/FirebaseDocs.md b/firebaseai/FirebaseDocs.md deleted file mode 100644 index 4b4fabb17..000000000 --- a/firebaseai/FirebaseDocs.md +++ /dev/null @@ -1,625 +0,0 @@ -### Guiding Principles - -* **Prioritize a continuous narrative:** Keep the developer in the flow of the code rather than forcing them to jump to external documentation for core configuration steps. ([thread](https://docs.google.com/document/d/1zBitr-qR9cqG0tt1c8AYFrafMM2FcGkJvFVCv2D_SDs/edit?disco=AAAB7xVQlE0)) -* **Balance "Self-contained" vs. "Linked":** Provide the "essential" steps (the specific Fuji-related tweaks) directly in the guide to avoid fragmentation. -* **Contextual linking:** Link to authoritative documentation only the first time a feature (e.g., Firebase App Check, AI Logic) is mentioned. Subsequent mentions should focus on the implementation flow. -* **Code transparency:** Code snippets must reflect incremental progress that is in sync with configuration steps (e.g., create app on console, then configure Firebase in app). -* **Compile-ready:** The app should be in a state that compiles and runs at each major step. -* **Minimize context switching:** Avoid forcing unnecessary transitions between Xcode and the Firebase console. - -### TODOs - -* Identify where caveats should be added for readers extending an existing app. - -### Outline - -To view the current outline, select the overflow menu (three dots) on the current document tab and select *Show Outline*. -![][image1] - -### Key Decisions - -* **DevSite Location:** Put it under [https://firebase.google.com/docs/ai-logic/**solutions**/overview](https://firebase.google.com/docs/ai-logic/solutions/overview) -* **Title:** [See resolved thread](https://docs.google.com/document/d/1zBitr-qR9cqG0tt1c8AYFrafMM2FcGkJvFVCv2D_SDs/edit?disco=AAAB83hTjGU). - * Refer to the feature like “this integration” -* **Branch Name:** `wwdc26-preview` - * Thread for if we run into issues with the branch not updating for clients ([thread](https://docs.google.com/document/d/1zBitr-qR9cqG0tt1c8AYFrafMM2FcGkJvFVCv2D_SDs/edit?disco=AAAB83oWffw)). - ---- - -**BEGIN PUBLIC CONTENT** - -# **\[page\] Get started: Access the Gemini API through Apple's Foundation Models framework** - -**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. -Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. - -\ \ - -This guide shows you how to get started using the Firebase AI Logic integration to access the Gemini API through Apple's Foundation Models framework. - -To protect access to Gemini models, this guide also shows you how to set up \[Firebase App Check\]([https://firebase.google.com/docs/app-check](https://firebase.google.com/docs/app-check)), which is ***critical*** even during development. - -## Prerequisites - -* Install the latest \[Xcode 27 beta\]([https://developer.apple.com/download/all/?q=Xcode](https://developer.apple.com/download/all/?q=Xcode)). -* An Apple platform simulator or a physical device, both running the corresponding beta OS version (for example, iOS 27 beta). -* A new Xcode project of an Apple platforms app using a **SwiftUI interface**. - -### Supported Gemini models - -The integration with Apple’s Foundation Models framework supports the following Gemini models. - -* General purpose models - * `gemini-3.1-pro-preview` - * `gemini-3.5-flash` - * `gemini-3.1-flash-lite` -* Image-generating models - * `gemini-3-pro-image-preview` (aka "Nano Banana Pro") - * `gemini-3.1-flash-image-preview` (aka "Nano Banana 2") - * `gemini-2.5-flash-image` (aka "Nano Banana") - -Gemini Live API models and Imagen models are ***not*** supported. Note that Gemini 2.5 models are technically supported, but they're not recommended for new projects and require special configuration not covered in these guides. - -## Step 1: Create a Firebase project - -We recommend starting with a new Firebase project to explore this integration. - -1. Sign into the \[Firebase console\]([https://console.firebase.google.com/](https://console.firebase.google.com/)). -2. Click **Create a new Firebase project**. -3. Follow the on-screen instructions. You do **not** need to enable Google Analytics. - -## Step 2: Connect your app to Firebase - -To connect your app to Firebase, you must register it with your Firebase project and add a configuration file to your codebase. - -1. In the center of the \[project overview\]([https://console.firebase.google.com/project/\_/overview](https://console.firebase.google.com/project/_/overview)) page, click the **iOS+** icon to launch the setup workflow. -2. Register your app: - 1. Enter your app's **bundle ID**. Make sure it matches the bundle ID of the project you're building in Xcode. - 2. Click **Register app**. -3. Add the Firebase configuration file. - This file contains the settings for the Firebase SDK to connect to your Firebase project. - 1. Click **Download GoogleService-Info.plist** to get your configuration file. - 2. Move `GoogleService-Info.plist` into the root of your Xcode project and add it to all targets. - 3. Click **Next** in the Firebase console. -4. The workflow in the console provides *generic* instructions for adding the Firebase SDK to your app, so skip ahead to the next step in this guide for specific instructions for Firebase AI Logic. - -## Step 3: Add the Firebase libraries and initialize Firebase in your app - -1. Use Swift Package Manager to add the Firebase AI Logic library: - 1. In Xcode, with your app project open, select **File** \> **Add Packages**. - 2. Enter the Firebase Apple SDK repository URL: - -``` -https://github.com/firebase/firebase-ios-sdk -``` - - 3. Select the Dependency Rule as **Branch** and enter **`wwdc26-preview`**. - 4. Click **Add Package**. Xcode will resolve and download the dependencies. - 5. When prompted, add the **`FirebaseAILogic`** library to your app target. -2. Initialize Firebase when your app starts up by adding the following code to your app's main entry point: - -```swift -import SwiftUI -import FirebaseCore - -@main -struct YourApp: App { init() { - FirebaseApp.configure() - } - - var body: some Scene { - WindowGroup { - NavigationView { - ContentView() - } - } - } -} -``` - -## Step 4: Enable and secure Firebase services - -Now that your app is configured to use Firebase, you need to enable the Firebase AI Logic service and protect access to its associated APIs using Firebase App Check. - -### Set up **Firebase AI Logic** in your Firebase project - -1. In the Firebase console, go to **AI Services** \> [**AI Logic**](https://console.firebase.google.com/project/_/ailogic/?useAutoProject=true). -2. Click **Get started** to launch the setup workflow. -3. We recommend choosing the **Gemini Developer API** provider to get started quickly at no cost. - -### Set up **Firebase App Check** in your Firebase project - -**Using Firebase App Check to protect the APIs associated with Firebase AI Logic is *critical*.** When enforced, Firebase App Check only allows incoming requests that are from your actual app and an untampered device. Firebase App Check supports many attestation providers, including Apple's \[App Attest\]([https://developer.apple.com/documentation/devicecheck](https://developer.apple.com/documentation/devicecheck)). - -The following steps are for a baseline, default setup for App Check. Learn more about [additional configuration options for App Check](https://firebase.google.com/docs/ai-logic/app-check) (like adjusting the TTL of tokens and enabling limited-use tokens). - -\[\!TIP\] **Developing a Mac app?** If you're developing a macOS app, skip the below steps and use the \[DeviceCheck provider\]([https://firebase.google.com/docs/app-check/ios/devicecheck-provider](https://firebase.google.com/docs/app-check/ios/devicecheck-provider)) instead of App Attest, which is unsupported on macOS. - -Here's how to register the App Attest provider in the Firebase console: - -1. In the Firebase console, go to **Security** \> \[**App Check\](**[https://console.firebase.google.com/project/\_/appcheck/](https://console.firebase.google.com/project/_/appcheck/)**)**. -2. Click **Get started**. -3. In the **\[Apps** tab**\](**[https://console.firebase.google.com/project/\_/appcheck/apps/](https://console.firebase.google.com/project/_/appcheck/apps/)**)**, register your app to use App Check with the \[**App Attest** provider\]([https://firebase.google.com/docs/app-check/ios/app-attest-provider](https://firebase.google.com/docs/app-check/ios/app-attest-provider)). -4. In the \[**APIs** tab\]([https://console.firebase.google.com/project/\_/appcheck/products/](https://console.firebase.google.com/project/_/appcheck/products/)), select **Firebase AI Logic**, and click **Enforce**. - -**(+) Before distributing your app, configure the App Attest capability** - -**Note:** If you’re going to start with the App Check debug provider (next section), these steps aren't required until you distribute your app. For now, you can skip ahead to the next section. - -1. In Xcode, add the \[App Attest capability\]([https://developer.apple.com/documentation/xcode/adding-capabilities-to-your-app](https://developer.apple.com/documentation/xcode/adding-capabilities-to-your-app)) to your app. -2. In your Xcode project's `.entitlements` file, set the \[App Attest environment\]([https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.devicecheck.appattest-environment](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.developer.devicecheck.appattest-environment)) to `production`. - -### Configure the App Check Debug Provider for local development - -For local development, configure the \[App Check Debug Provider\]([https://firebase.google.com/docs/app-check/ios/debug-provider](https://firebase.google.com/docs/app-check/ios/debug-provider)). Setting up this provider bypasses attestation during development so you can verify your app's logic without modifying the production security configuration you set up above. - -1. In your Xcode project, import `FirebaseAppCheck` and initialize App Check with the debug provider factory before you configure `FirebaseApp`. - -```swift -import SwiftUI -import FirebaseCore -import FirebaseAppCheck - -@main -struct YourApp: App { - init() { - let providerFactory = AppCheckDebugProviderFactory() - AppCheck.setAppCheckProviderFactory(providerFactory) - FirebaseApp.configure() - } - - var body: some Scene { - WindowGroup { - NavigationView { - ContentView() - } - } - } -} - -``` - -2. Obtain your debug token: - 1. Launch your app in the simulator or on your test device. - 2. Open the Xcode console and look for the App Check debug token. Because it is generated at app startup, it should be one of the first few logs you see. It will look similar to this: - - \\<Warning\> \[AppCheckCore\]\[I-GAC004001\] App Check debug token: '123a4567-b89c-12d3-e456-789012345678'.\ - -1. Copy the token (for example, `123a4567-b89c-12d3-e456-789012345678`). - -3. Provide your debug token to App Check within the Firebase console: - 1. In the Firebase console, navigate to **Security** \> **App Check** \> [**Apps**](https://console.firebase.google.com/project/_/appcheck/apps/). - 2. Find your app, click the overflow menu (three dots), and select **Manage debug tokens**. - 3. Click **Add debug token**, enter a name (for example, `My Simulator`), paste the token, and click **Save**. - -For details about the debug provider (including how to get a new debug token), check out the [official App Check docs](https://firebase.google.com/docs/app-check/ios/debug-provider). - -\[\!WARNING\] **Keep your debug token private.** Because this debug token lets anyone access your Firebase AI Logic resources without a valid device, you must keep it private. Do not commit it to a public repository. If a registered token is ever compromised, revoke it immediately in the Firebase console. - -## Step 5: Initialize the AI Logic service in your app - -With Firebase and App Check configured, you can now initialize the Firebase AI Logic service in your app. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") -``` - -## Step 6: Send a request to a Gemini model - -With Firebase AI Logic setup, protected, and initialized in your app, you're ready to send a request to a Gemini model. - -The following example shows the most basic type of request – generating text from a text-only prompt: - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -// Inject the model into Apple's LanguageModelSession. -let session = LanguageModelSession(model: model) - -// Use the session to generate a text response to a prompt. -let response = try await session.respond(to: "Write a story about a magic backpack.") -print(response.content) -``` - -Gemini models also support other types of requests, like [analyzing images and PDFs](#generate-text-from-multimodal-input-\(like-images\)), [generating structured JSON output](#generate-structured-output-\(like-json\)), and [generating and editing images](#generate-and-edit-images-\(using-"nano-banana"-models\)) (using "Nano Banana" models). See examples for these types of requests in the [docs](#[page]-available-capabilities-when-accessing-the-gemini-api-through-apple's-foundation-models-framework) or in the sample app. - -### Stream the response - -You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -// Inject the model into Apple's LanguageModelSession. -let session = LanguageModelSession(model: model) - -// Use the session to generate a streamed text response to a prompt. -let response = try await session.streamResponse(to: "Write a story about a magic backpack.") -print(response.content) -``` - -## Next steps - -* Check out the sample app or a video to see this integration in a real-world app. -* Explore the [available capabilities](#[page]-available-capabilities-when-accessing-the-gemini-api-through-apple's-foundation-models-framework) when accessing the Gemini API through Apple's Foundation Models framework. -* Learn how to [configure the model](#[page]-configuration-options-when-accessing-the-gemini-api-through-apple's-foundation-models-framework) to control its responses, including setting a thinking level. -* Learn how to [provide tools to the model](#[page]-provide-tools-to-the-model-when-accessing-the-gemini-api-through-apple's-foundation-models-framework), including grounding for up-to-date information. -* Read more about \[Firebase App Check\]([https://firebase.google.com/docs/ai-logic/app-check](https://firebase.google.com/docs/ai-logic/app-check)) and how it protects your resources. - -# **\[page\] Available capabilities when accessing the Gemini API through Apple's Foundation Models framework** {#[page]-available-capabilities-when-accessing-the-gemini-api-through-apple's-foundation-models-framework} - -**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. -Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. - -This guide shows you how to send various types of requests to the Gemini API through Apple's Foundation Models framework using the Firebase AI Logic SDK for Apple platforms. - -The examples on this page assume that you've completed the [Get started: Access the Gemini API through Apple's Foundation Models framework](https://firebase.devsite.corp.google.com/docs/ai-logic/apple-foundation-models-framework/get-started). - -This page shows examples for how to send the following types of requests: - -* [Generate text from text-only input](#generate-text-from-text-only-input) -* Generate text from a multi-turn session (chat) -* [Generate text from multimodal input (like images)](#generate-text-from-multimodal-input-\(like-images\)) -* [Generate and edit images](#generate-and-edit-images-\(using-"nano-banana"-models\)) (using "Nano Banana" models) -* [Generate structured output (like JSON)](#generate-structured-output-\(like-json\)) - -## Generate text - -Gemini supports the following capabilities for generating text: - -* [Generate text from text-only input](#generate-text-from-text-only-input) -* Generate text from a multi-turn session (chat) -* [Generate text from multimodal input (like images)](#generate-text-from-multimodal-input-\(like-images\)) - -#### Models that support this capability - -* `gemini-3.1-pro-preview` -* `gemini-3.5-flash` -* `gemini-3.1-flash-lite` - -### Generate text from text-only input {#generate-text-from-text-only-input} - -You can ask a Gemini model to generate text by prompting with text-only input. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -// Provide a prompt that contains text. -let prompt = "Write a story about a magic backpack." - -// For a single-turn interaction, create a new session each time you call the model. -let session = LanguageModelSession(model: model) - -// Generate a text response to the prompt. -let response = try await session.respond(to: prompt) -print(response.content) -``` - -**(+) Stream the response** -You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -// Provide a prompt that contains text. -let prompt = "Write a story about a magic backpack." - -// For a single-turn interaction, create a new session each time you call the model. -let session = LanguageModelSession(model: model) - -// Generate a text response to the prompt. -// To stream the response, use `streamResponse(to:)` instead of `respond(to:)` -let response = try await session.streamResponserespond(to: prompt) -print(response.content) -``` - -### Generate text from a multi-turn session - -``` -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -// Optionally specify previous session interactions. -let transcript = Transcript(entries: [ - .prompt(.init(segments: [.text(.init(content: "Hello, I have 2 dogs in my house."))])), - .response(.init( - assetIDs: [], - segments: [.text(.init(content: "Great to meet you. What would you like to know?"))] - )) -]) - -// Create a session, optionally rehydrating from a transcript. -let session = LanguageModelSession(model: model, transcript: transcript) - -// Generate a text response to the prompt. -let response = try await session.respond(to: "How many paws are in my house?") -print(response.content) -``` - -**(+) Stream the response** -You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -TODO -``` - -### Generate text from multimodal input (like images) {#generate-text-from-multimodal-input-(like-images)} - -**Note**: In the current release for this integration, multimodal media input can be images or documents (like PDFs). Video and audio input are not yet supported. - -You can ask a Gemini model to generate text by prompting with text and a file—providing each input file's `mimeType` and the file itself. Find [requirements and recommendations for input files](https://firebase.devsite.corp.google.com/docs/ai-logic/generate-text?db=rachelsaunders&api=dev#requirements-recommendations-for-input) later on this page. - -The following example shows the basics of how to generate text from a file input by analyzing a single image file provided as inline data (base64-encoded file). - -Note that this example shows providing the file inline, but the SDKs also support [providing a file using a browser/HTTP URL](https://firebase.devsite.corp.google.com/docs/ai-logic/input-file-requirements?db=rachelsaunders). - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -TODO -``` - -**(+) Stream the response** -You can achieve faster interactions by not waiting for the entire result from the model generation, and instead use *streaming* to handle partial results. To stream the response, use `streamResponse(to:)` instead of `respond(to:)`. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -TODO -``` - -## Generate and edit images (using "Nano Banana" models) {#generate-and-edit-images-(using-"nano-banana"-models)} - -Gemini supports the following capabilities for generating and editing images: - -* [Generate an image from text-only input](#generate-an-image-from-text-only-input) -* [Edit an image](#edit-an-image) -* [Iterate and edit images using multi-turn chat](#iterate-and-edit-images-using-multi-turn-chat) - -**Note**: Image-generating models let you optionally provide an [\`imageConfig\`](#configure-image-output) to specify aspect ratios and image sizes. - -#### Models that support this capability - -* `gemini-3-pro-image-preview` (aka "Nano Banana Pro") -* `gemini-3.1-flash-image-preview` (aka "Nano Banana 2") -* `gemini-2.5-flash-image` (aka "Nano Banana") - -### Generate an image from text-only input {#generate-an-image-from-text-only-input} - -You can ask a Gemini model to generate images by prompting with text. - -Include response modalities of `TEXT` and `IMAGE` in your model configuration (or exclude `TEXT` if you only want image output). - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini image-generating model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-image-preview") - -TODO -``` - -### Edit an image {#edit-an-image} - -You can ask a Gemini model to edit images by prompting with text and one or more images. - -Include response modalities of `TEXT` and `IMAGE` in your model configuration (or exclude `TEXT` if you only want image output). - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini image-generating model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-image-preview") - -TODO -``` - -### Iterate and edit images using multi-turn chat {#iterate-and-edit-images-using-multi-turn-chat} - -Using multi-turn chat, you can iterate with a Gemini model on the images that it generates or that you supply. - -Include response modalities of `TEXT` and `IMAGE` in your model configuration (or exclude `TEXT` if you only want image output), and call `startChat()` and `sendMessage()` to send new user messages. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini image-generating model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.1-flash-image-preview") - -TODO -``` - -## Generate structured output (like JSON) {#generate-structured-output-(like-json)} - -#### Models that support this capability - -* `gemini-3.1-pro-preview` -* `gemini-3.5-flash` -* `gemini-3.1-flash-lite` - -```swift -import SwiftUI -import FirebaseCore -import FirebaseAILogic - -// Initialize the Vertex AI Gemini API backend service. -// Specify `global` as the location to access the Gemini model. -let ai = model = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global")) -// Initialize a `geminiLanguageModel` with a Gemini model that supports your use case. -let model = ai.geminiLanguageModel(name: "gemini-3.5-flash") - -TODO -``` - -# **\[page\] Configuration options when accessing the Gemini API through Apple's Foundation Models framework** {#[page]-configuration-options-when-accessing-the-gemini-api-through-apple's-foundation-models-framework} - -**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. -Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. - -The examples on this page assume that you've completed the [Get started: Access the Gemini API through Apple's Foundation Models framework](https://firebase.devsite.corp.google.com/docs/ai-logic/apple-foundation-models-framework/get-started). - -In each request to a model, you can send along a model configuration to control how the model generates a response. Each model offers different configuration options. - -The configuration is maintained for the lifetime of the session. If you want to use a different config, create a new session with that config. - -## System instructions - -*System instructions* are like a "preamble" that you add before the model gets exposed to any further instructions from the end user. It lets you steer the behavior of the model based on your specific needs and use cases. - -```swift -import FoundationModels -import FirebaseCore -import FirebaseAILogic - -let model = FirebaseAI.firebaseAI().geminiLanguageModel(name: "GEMINI_MODEL_NAME") - -// Specify the system instructions as part of creating the LanguageModelSession -let session = LanguageModelSession( - model: model, - instructions: "You are a cat. Your name is Neko." -) - -// ... -``` - -## Configure thinking (aka "reasoning") - -The following is a high-level description of how to configure the amount of *thinking* that a Gemini model can do. Note that Apple calls the thinking capability *reasoning*. For details, best practices, and use cases for thinking, see [Thinking](https://firebase.google.com/docs/ai-logic/thinking) in the general AI Logic docs. - -Note the following: - -* Gemini 3.x models always use thinking; you ***cannot*** disable or turn off thinking for these models. -* Gemini 3.x models always use *dynamic thinking* – the model decides when and how much it thinks up to the configured amount. - -### Set the thinking level - -```swift -TODO -``` - -### Supported thinking level values - -The following table lists the thinking level values that you can set for each model by configuring the model's `thinkingLevel`. - -\< within DevSite share the table from the thinking page \> - -## Configure image output {#configure-image-output} - -When using image-generating Gemini models (like "Nano Banana" models), you can set an \`imageConfig\`. - -```swift -TODO -``` - -\< within DevSite share the supported values from the nano banana page \> - -## General configuration - -Gemini models support some more general configuration parameters, as described in the table later in this section. Note that for Gemini 3.x models, configuring temperature, topK, and topP is not recommended. - -```swift -TODO -``` - -| Parameter | Description | Default value | -| :---- | :---- | :---- | -| Candidate count `candidateCount` | Specifies the number of response variations to return. For each request, you're charged for the output tokens of all candidates, but you're only charged once for the input tokens. Supported values: `1` \- `8` (inclusive) | `1` | -| Frequency penalty `frequencyPenalty` | Controls the probability of including tokens that repeatedly appear in the generated response. Positive values penalize tokens that repeatedly appear in the generated content, decreasing the probability of repeating content. | \--- | -| Max output tokens `maxOutputTokens` | Specifies the maximum number of tokens that can be generated in the response. | \--- | -| Presence penalty `presencePenalty` | Controls the probability of including tokens that already appear in the generated response. Positive values penalize tokens that already appear in the generated content, increasing the probability of generating more diverse content. | \--- | -| Stop sequences `stopSequences` | Specifies a list of strings that tells the model to stop generating content if one of the strings is encountered in the response. | \--- | - -# **\[page\] Provide tools to the model when accessing the Gemini API through Apple's Foundation Models framework** {#[page]-provide-tools-to-the-model-when-accessing-the-gemini-api-through-apple's-foundation-models-framework} - -**Preview**: Accessing the Gemini API through Apple's Foundation Models framework is currently in public preview, which means that it isn't subject to any SLA or deprecation policy and could change in backwards-incompatible ways. -Also, this integration relies on beta APIs, so apps using it cannot be submitted to the App Store until the next Xcode version reaches General Availability (GA) and supports production submissions. - -The examples on this page assume that you've completed the [Get started: Access the Gemini API through Apple's Foundation Models framework](https://firebase.devsite.corp.google.com/docs/ai-logic/apple-foundation-models-framework/get-started). From 11bfbe5dc034619ce24a3e6f42faeec820ae86c8 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Thu, 2 Jul 2026 23:23:47 +0200 Subject: [PATCH 09/10] fix(apple-ai): fix selected backend propagation and simplify async task architecture in ViewModels --- firebaseai/FirebaseAIExample/ContentView.swift | 4 ++-- .../AppleFoundationModels/ViewModels/HybridAIViewModel.swift | 2 +- .../AppleFoundationModels/ViewModels/VisionIDViewModel.swift | 2 +- .../Features/AppleFoundationModels/Views/HybridAIView.swift | 4 +--- .../Features/AppleFoundationModels/Views/VisionIDView.swift | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 5ec4af3cd..2ca70ebad 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -77,7 +77,7 @@ struct ContentView: View { LazyVGrid(columns: columns, spacing: 20) { ForEach(filteredSamples.filter { !$0.isFoundationModel }) { sample in - NavigationLink(destination: destinationView(for: sample)) { + NavigationLink(destination: destinationView(for: sample).id(selectedBackend)) { SampleCardView(sample: sample) } .buttonStyle(PlainButtonStyle()) @@ -94,7 +94,7 @@ struct ContentView: View { LazyVGrid(columns: columns, spacing: 20) { ForEach(filteredSamples.filter { $0.isFoundationModel }) { sample in - NavigationLink(destination: destinationView(for: sample)) { + NavigationLink(destination: destinationView(for: sample).id(selectedBackend)) { SampleCardView(sample: sample) } .buttonStyle(PlainButtonStyle()) diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift index 64607917e..6ceee57ac 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift @@ -25,7 +25,7 @@ public final class HybridAIViewModel: FoundationModelsBaseViewModel { "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." @Published public var outputSummary: TextSummary? - public func runSummarization() async { + public func runSummarization() { stopActiveTask() inProgress = true error = nil diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift index 49f2d30b4..c2046ee32 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift @@ -24,7 +24,7 @@ public final class VisionIDViewModel: FoundationModelsBaseViewModel { @Published public var selectedImage: UIImage? @Published public var identifiedObject: IdentifiedObject? - public func identifySelectedImage() async { + public func identifySelectedImage() { guard let image = selectedImage else { return } stopActiveTask() inProgress = true diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift index 0185b4271..7ca097e82 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/HybridAIView.swift @@ -31,9 +31,7 @@ struct HybridAIView: View { } Button(action: { - Task { - await viewModel.runSummarization() - } + viewModel.runSummarization() }) { HStack { Spacer() diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift index 0e00d2455..6a8d7516a 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Views/VisionIDView.swift @@ -55,7 +55,7 @@ struct VisionIDView: View { if let data = try? await newItem?.loadTransferable(type: Data.self), let image = UIImage(data: data) { viewModel.selectedImage = image - await viewModel.identifySelectedImage() + viewModel.identifySelectedImage() } } } From 8e6da5b34ff6c79dcb2a78b8f644a2ffbd6f2af2 Mon Sep 17 00:00:00 2001 From: Peter Friese Date: Fri, 3 Jul 2026 00:55:03 +0200 Subject: [PATCH 10/10] refactor(apple-ai): change ConversationKit to remote and remove redundant public access modifiers --- .../project.pbxproj | 37 ++++++------------- .../AppleFoundationModels/Models/Models.swift | 12 +++--- .../FoundationModelsBaseViewModel.swift | 20 +++++----- .../ViewModels/HybridAIViewModel.swift | 8 ++-- .../ViewModels/VisionIDViewModel.swift | 8 ++-- .../Shared/Models/BackendOption.swift | 4 +- .../Shared/Util/MLErrorHelper.swift | 4 +- 7 files changed, 40 insertions(+), 53 deletions(-) diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index 43106ee49..5f779987c 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -7,11 +7,9 @@ objects = { /* Begin PBXBuildFile section */ - 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5C2FD7C19F003702C7 /* FirebaseAI */; }; 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */; }; 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D602FD7C19F003702C7 /* FirebaseAppCheck */; }; 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88862D622FD7C19F003702C7 /* FirebaseCore */; }; - 88AAB45E2FF0077E001B143E /* FirebaseAI in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB45D2FF0077E001B143E /* FirebaseAI */; }; 88AAB4602FF0077E001B143E /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB45F2FF0077E001B143E /* FirebaseAppCheck */; }; 88AAB4622FF0077E001B143E /* FirebaseCore in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB4612FF0077E001B143E /* FirebaseCore */; }; 88AAB4652FF007DD001B143E /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 88AAB4642FF007DD001B143E /* ConversationKit */; }; @@ -34,14 +32,12 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - 88AAB45E2FF0077E001B143E /* FirebaseAI in Frameworks */, 88862D5F2FD7C19F003702C7 /* FirebaseAILogic in Frameworks */, 88AAB4652FF007DD001B143E /* ConversationKit in Frameworks */, 88862D612FD7C19F003702C7 /* FirebaseAppCheck in Frameworks */, 88862D632FD7C19F003702C7 /* FirebaseCore in Frameworks */, 88AAB4622FF0077E001B143E /* FirebaseCore in Frameworks */, 88AAB4602FF0077E001B143E /* FirebaseAppCheck in Frameworks */, - 88862D5D2FD7C19F003702C7 /* FirebaseAI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -84,11 +80,9 @@ ); name = FirebaseAIExample; packageProductDependencies = ( - 88862D5C2FD7C19F003702C7 /* FirebaseAI */, 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */, 88862D602FD7C19F003702C7 /* FirebaseAppCheck */, 88862D622FD7C19F003702C7 /* FirebaseCore */, - 88AAB45D2FF0077E001B143E /* FirebaseAI */, 88AAB45F2FF0077E001B143E /* FirebaseAppCheck */, 88AAB4612FF0077E001B143E /* FirebaseCore */, 88AAB4642FF007DD001B143E /* ConversationKit */, @@ -123,7 +117,7 @@ minimizedProjectReferenceProxies = 1; packageReferences = ( 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, - 88AAB4632FF007DD001B143E /* XCLocalSwiftPackageReference "../../../../ConversationKit/ConversationKit" */, + 88AAB4632FF007DD001B143E /* XCRemoteSwiftPackageReference "ConversationKit" */, ); preferredProjectObjectVersion = 77; productRefGroup = 88779D362EC8A9CF0080D023 /* Products */; @@ -282,7 +276,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = YGAZHQXHH4; + DEVELOPMENT_TEAM = ""; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -316,7 +310,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = YGAZHQXHH4; + DEVELOPMENT_TEAM = ""; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_NSMicrophoneUsageDescription = "The app needs access to your microphone to enable live voice conversations with Gemini."; @@ -366,13 +360,6 @@ }; /* End XCConfigurationList section */ -/* Begin XCLocalSwiftPackageReference section */ - 88AAB4632FF007DD001B143E /* XCLocalSwiftPackageReference "../../../../ConversationKit/ConversationKit" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = ../../../../ConversationKit/ConversationKit; - }; -/* End XCLocalSwiftPackageReference section */ - /* Begin XCRemoteSwiftPackageReference section */ 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */ = { isa = XCRemoteSwiftPackageReference; @@ -382,13 +369,17 @@ kind = branch; }; }; + 88AAB4632FF007DD001B143E /* XCRemoteSwiftPackageReference "ConversationKit" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/peterfriese/ConversationKit"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 0.0.4; + }; + }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - 88862D5C2FD7C19F003702C7 /* FirebaseAI */ = { - isa = XCSwiftPackageProductDependency; - productName = FirebaseAI; - }; 88862D5E2FD7C19F003702C7 /* FirebaseAILogic */ = { isa = XCSwiftPackageProductDependency; productName = FirebaseAILogic; @@ -401,11 +392,6 @@ isa = XCSwiftPackageProductDependency; productName = FirebaseCore; }; - 88AAB45D2FF0077E001B143E /* FirebaseAI */ = { - isa = XCSwiftPackageProductDependency; - package = 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; - productName = FirebaseAI; - }; 88AAB45F2FF0077E001B143E /* FirebaseAppCheck */ = { isa = XCSwiftPackageProductDependency; package = 88AAB45C2FF0077E001B143E /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */; @@ -418,6 +404,7 @@ }; 88AAB4642FF007DD001B143E /* ConversationKit */ = { isa = XCSwiftPackageProductDependency; + package = 88AAB4632FF007DD001B143E /* XCRemoteSwiftPackageReference "ConversationKit" */; productName = ConversationKit; }; /* End XCSwiftPackageProductDependency section */ diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift index 7db83a2ec..555917375 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/Models/Models.swift @@ -17,22 +17,22 @@ import FoundationModels @available(iOS 26.0, *) @Generable -public struct IdentifiedObject: Equatable, Codable { +struct IdentifiedObject: Equatable, Codable { @Guide(description: "The name of the primary object or landmark detected.") - public let name: String + let name: String @Guide( description: "The category of the object (e.g. Landmark, Plant, Food, Animal, Device, Clothing)." ) - public let category: String + let category: String @Guide(description: "A short, 2-sentence description of the object and interesting facts.") - public let description: String + let description: String } @available(iOS 26.0, *) @Generable -public struct TextSummary: Equatable, Codable { +struct TextSummary: Equatable, Codable { @Guide(description: "A list of exactly 2 key summary points.") - public let summaryPoints: [String] + let summaryPoints: [String] } diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift index e9087cfd4..b6706505e 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/FoundationModelsBaseViewModel.swift @@ -18,29 +18,29 @@ import Combine import FoundationModels import FirebaseAILogic -public enum ModelPreference: String, CaseIterable, Identifiable { +enum ModelPreference: String, CaseIterable, Identifiable { case auto = "Auto (Local First)" case cloud = "Cloud Only" - public var id: String { rawValue } + var id: String { rawValue } } @available(iOS 27.0, *) @MainActor -public class FoundationModelsBaseViewModel: ObservableObject { - @Published public var inProgress = false - @Published public var error: Error? - @Published public var modelPreference: ModelPreference = .auto - @Published public var isUsingLocalModel: Bool = false +class FoundationModelsBaseViewModel: ObservableObject { + @Published var inProgress = false + @Published var error: Error? + @Published var modelPreference: ModelPreference = .auto + @Published var isUsingLocalModel: Bool = false internal var activeTask: Task? - public let backendType: BackendOption + let backendType: BackendOption - public init(backendType: BackendOption) { + init(backendType: BackendOption) { self.backendType = backendType } - public func stopActiveTask() { + func stopActiveTask() { activeTask?.cancel() activeTask = nil inProgress = false diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift index 6ceee57ac..735d9b09c 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/HybridAIViewModel.swift @@ -20,12 +20,12 @@ import FirebaseAILogic @available(iOS 27.0, *) @MainActor -public final class HybridAIViewModel: FoundationModelsBaseViewModel { - @Published public var inputText: String = +final class HybridAIViewModel: FoundationModelsBaseViewModel { + @Published var inputText: String = "It is the quintessential autumn harvest fruit, famously baked into warm cinnamon pastries, dipped in sticky caramel on Halloween, and traditionally rumored to keep medical professionals at bay if eaten once a day." - @Published public var outputSummary: TextSummary? + @Published var outputSummary: TextSummary? - public func runSummarization() { + func runSummarization() { stopActiveTask() inProgress = true error = nil diff --git a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift index c2046ee32..b3fa7f2ab 100644 --- a/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift +++ b/firebaseai/FirebaseAIExample/Features/AppleFoundationModels/ViewModels/VisionIDViewModel.swift @@ -20,11 +20,11 @@ import FirebaseAILogic @available(iOS 27.0, *) @MainActor -public final class VisionIDViewModel: FoundationModelsBaseViewModel { - @Published public var selectedImage: UIImage? - @Published public var identifiedObject: IdentifiedObject? +final class VisionIDViewModel: FoundationModelsBaseViewModel { + @Published var selectedImage: UIImage? + @Published var identifiedObject: IdentifiedObject? - public func identifySelectedImage() { + func identifySelectedImage() { guard let image = selectedImage else { return } stopActiveTask() inProgress = true diff --git a/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift b/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift index d0b26d063..0731fba0c 100644 --- a/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift +++ b/firebaseai/FirebaseAIExample/Shared/Models/BackendOption.swift @@ -14,9 +14,9 @@ import Foundation -public enum BackendOption: String, CaseIterable, Identifiable { +enum BackendOption: String, CaseIterable, Identifiable { case googleAI = "Google AI" case vertexAI = "Firebase Vertex AI" - public var id: String { rawValue } + var id: String { rawValue } } diff --git a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift index 079284e4a..2a34678e0 100644 --- a/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift +++ b/firebaseai/FirebaseAIExample/Shared/Util/MLErrorHelper.swift @@ -18,7 +18,7 @@ import FoundationModels extension Error { /// Returns true if this error indicates that required ML assets (like safety guardrails) /// are missing from the device or simulator. - public var isMLAssetUnavailable: Bool { + var isMLAssetUnavailable: Bool { // 1. Check for LanguageModelSession.GenerationError.assetsUnavailable if #available(iOS 26.0, *) { if let genError = self as? LanguageModelSession.GenerationError { @@ -51,7 +51,7 @@ extension Error { } /// Returns a user-friendly message for ML asset errors, particularly for simulators. - public var mlAssetErrorMessage: String? { + var mlAssetErrorMessage: String? { guard isMLAssetUnavailable else { return nil } #if targetEnvironment(simulator)