diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx
index f269b27aae..1a53311ad9 100644
--- a/apps/computer-vision/app/_layout.tsx
+++ b/apps/computer-vision/app/_layout.tsx
@@ -27,6 +27,13 @@ export default function Layout() {
title: 'Classification',
}}
/>
+
{
+ let s = 0;
+ for (let i = 0; i < a.length; i++) {
+ s += a[i]! * b[i]!;
+ }
+ return s;
+};
+
+function ImageEmbeddingsContent() {
+ const [selectedImageModel, setSelectedImageModel] = useState(IMAGE_MODEL_OPTIONS[0].value);
+ const [imageUri, setImageUri] = useState(null);
+ const [labels, setLabels] = useState(DEFAULT_LABELS);
+ const [newLabel, setNewLabel] = useState('');
+ const [results, setResults] = useState<{ label: string; score: number }[]>([]);
+ const [latency, setLatency] = useState(null);
+ const [isProcessing, setIsProcessing] = useState(false);
+ const [error, setError] = useState(null);
+
+ const insets = useSafeAreaInsets();
+ const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err)));
+
+ // Zero-shot classification pairs a CLIP image encoder with the CLIP text
+ // encoder and scores the image against each text label by embedding similarity.
+ const imageModel = useImageEmbeddings(selectedImageModel);
+ const textModel = useTextEmbeddings(models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT);
+
+ const ready = imageModel.isReady && textModel.isReady;
+
+ const pickImage = async () => {
+ setError(null);
+ try {
+ const uri = await getImage(false);
+ if (!uri) return;
+ setImageUri(uri);
+ setResults([]);
+ setLatency(null);
+ } catch (e: any) {
+ setError(e.message || String(e));
+ }
+ };
+
+ const classify = async () => {
+ if (!skiaImage || !ready || !imageModel.forward || !textModel.forward) return;
+ setIsProcessing(true);
+ setError(null);
+ try {
+ const start = Date.now();
+ const imageEmbedding = await imageModel.forward(skImageToBuffer(skiaImage));
+ const scored: { label: string; score: number }[] = [];
+ for (const label of labels) {
+ const textEmbedding = await textModel.forward(label);
+ scored.push({ label, score: dot(imageEmbedding, textEmbedding) });
+ }
+ scored.sort((a, b) => b.score - a.score);
+ setLatency(Date.now() - start);
+ setResults(scored);
+ } catch (e: any) {
+ setError(e.message || String(e));
+ } finally {
+ setIsProcessing(false);
+ }
+ };
+
+ const addLabel = () => {
+ const trimmed = newLabel.trim();
+ if (!trimmed || labels.includes(trimmed)) return;
+ setLabels((prev) => [...prev, trimmed]);
+ setNewLabel('');
+ setResults([]);
+ };
+
+ const removeLabel = (label: string) => {
+ setLabels((prev) => prev.filter((l) => l !== label));
+ setResults((prev) => prev.filter((r) => r.label !== label));
+ };
+
+ const activeError = imageModel.error
+ ? String(imageModel.error)
+ : textModel.error
+ ? String(textModel.error)
+ : error;
+
+ return (
+
+
+ Pick an image, then rank text labels by how well CLIP matches them to it (zero-shot
+ classification).
+
+
+ {
+ setSelectedImageModel(model);
+ setResults([]);
+ setLatency(null);
+ }}
+ />
+
+
+
+
+
+
+
+
+
+
+
+
+ {results.length > 0 && (
+
+ Results
+ {results.map((r, i) => (
+
+
+ {i === 0 ? '🥇 ' : ''}
+ {r.label}
+
+ {r.score.toFixed(3)}
+
+ ))}
+
+ )}
+
+
+ Labels
+ {labels.map((label) => (
+
+
+ {label}
+
+ removeLabel(label)} hitSlop={8}>
+ ✕
+
+
+ ))}
+
+
+
+
+
+
+ );
+}
+
+export default function ImageEmbeddingsScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ card: {
+ width: '100%',
+ backgroundColor: '#fff',
+ borderRadius: 12,
+ padding: 16,
+ borderWidth: 1,
+ borderColor: '#e9ecef',
+ marginTop: 16,
+ },
+ cardTitle: {
+ fontSize: 16,
+ fontWeight: '600',
+ color: ColorPalette.strongPrimary,
+ marginBottom: 8,
+ },
+ row: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ paddingVertical: 8,
+ borderBottomWidth: 1,
+ borderBottomColor: '#f1f3f5',
+ },
+ rowLabel: { fontSize: 14, color: '#334155', flex: 1, marginRight: 8 },
+ topLabel: { fontWeight: '700', color: ColorPalette.strongPrimary },
+ rowScore: { fontSize: 13, fontWeight: '600', color: ColorPalette.primary },
+ remove: { fontSize: 16, color: '#94A3B8', paddingHorizontal: 4 },
+ addRow: { flexDirection: 'row', alignItems: 'center', gap: 8, marginTop: 12 },
+ input: {
+ flex: 1,
+ backgroundColor: '#f1f3f5',
+ borderRadius: 10,
+ paddingHorizontal: 12,
+ paddingVertical: 10,
+ fontSize: 14,
+ color: '#0F172A',
+ },
+});
diff --git a/apps/computer-vision/app/index.tsx b/apps/computer-vision/app/index.tsx
index cc44604e67..14fa922e57 100644
--- a/apps/computer-vision/app/index.tsx
+++ b/apps/computer-vision/app/index.tsx
@@ -14,6 +14,9 @@ export default function Home() {
router.navigate('classification/')}>
Classification
+ router.navigate('imageEmbeddings/')}>
+ Image Embeddings
+
router.navigate('detection/')}>
Object Detection
diff --git a/apps/computer-vision/package.json b/apps/computer-vision/package.json
index 01cebec75c..4a47a35f3f 100644
--- a/apps/computer-vision/package.json
+++ b/apps/computer-vision/package.json
@@ -5,6 +5,8 @@
"react-native-executorch": {
"features": [
"classification",
+ "imageEmbeddings",
+ "textEmbeddings",
"instanceSegmentation",
"keypointDetection",
"objectDetection",
diff --git a/apps/computer-vision/utils.ts b/apps/computer-vision/utils.ts
index 2a73d4b5b5..3a24d0341e 100644
--- a/apps/computer-vision/utils.ts
+++ b/apps/computer-vision/utils.ts
@@ -1,6 +1,32 @@
import { Alert } from 'react-native';
import * as ImagePicker from 'expo-image-picker';
import { ImageManipulator, SaveFormat } from 'expo-image-manipulator';
+import type { SkImage } from '@shopify/react-native-skia';
+
+/**
+ * Converts a Skia image into the raw RGBA/HWC image buffer that
+ * react-native-executorch vision tasks accept. Throws if the pixel data cannot
+ * be read.
+ * @param image - The Skia image to read pixels from.
+ * @returns The RGBA/HWC image buffer with its `data`, `width`, `height`,
+ * `format`, and `layout`.
+ */
+export const skImageToBuffer = (image: SkImage) => {
+ const pixels = image.readPixels();
+ if (!pixels) {
+ throw new Error('Failed to read pixels from image');
+ }
+ if (!(pixels instanceof Uint8Array)) {
+ throw new Error('Expected Uint8Array from readPixels');
+ }
+ return {
+ data: pixels,
+ width: image.width(),
+ height: image.height(),
+ format: 'rgba' as const,
+ layout: 'hwc' as const,
+ };
+};
export const getImage = async (
useCamera: boolean,
diff --git a/apps/nlp/app/_layout.tsx b/apps/nlp/app/_layout.tsx
index bdcfc39660..352cf99d44 100644
--- a/apps/nlp/app/_layout.tsx
+++ b/apps/nlp/app/_layout.tsx
@@ -27,6 +27,13 @@ export default function Layout() {
title: 'Tokenizer',
}}
/>
+
);
}
diff --git a/apps/nlp/app/index.tsx b/apps/nlp/app/index.tsx
index 98ff59ab97..57b5aea6ae 100644
--- a/apps/nlp/app/index.tsx
+++ b/apps/nlp/app/index.tsx
@@ -14,6 +14,9 @@ export default function Home() {
router.navigate('tokenizer/')}>
Tokenizer
+ router.navigate('text-embeddings/')}>
+ Text Embeddings
+
);
diff --git a/apps/nlp/app/text-embeddings/index.tsx b/apps/nlp/app/text-embeddings/index.tsx
new file mode 100644
index 0000000000..d7edde3c68
--- /dev/null
+++ b/apps/nlp/app/text-embeddings/index.tsx
@@ -0,0 +1,398 @@
+import React, { useEffect, useState } from 'react';
+import {
+ View,
+ Text,
+ TextInput,
+ ScrollView,
+ StyleSheet,
+ TouchableOpacity,
+ KeyboardAvoidingView,
+ Platform,
+} from 'react-native';
+import { useTextEmbeddings, models, type TextEmbeddingsModel } from 'react-native-executorch';
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelStatus } from '../../components/ModelStatus';
+import { Button } from '../../components/Button';
+import { theme } from '../../theme';
+
+const MODELS: { label: string; value: TextEmbeddingsModel }[] = [
+ { label: 'MiniLM L6', value: models.textEmbeddings.ALL_MINILM_L6_V2 },
+ { label: 'MPNet Base', value: models.textEmbeddings.ALL_MPNET_BASE_V2 },
+ { label: 'MultiQA MiniLM', value: models.textEmbeddings.MULTI_QA_MINILM_L6_COS_V1 },
+ { label: 'MultiQA MPNet', value: models.textEmbeddings.MULTI_QA_MPNET_BASE_DOT_V1 },
+ { label: 'Paraphrase ML', value: models.textEmbeddings.PARAPHRASE_MULTILINGUAL_MINILM_L12_V2 },
+ { label: 'DistilUSE ML', value: models.textEmbeddings.DISTILUSE_BASE_MULTILINGUAL_CASED_V2 },
+ { label: 'CLIP Text', value: models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT },
+];
+
+const STARTER_SENTENCES = [
+ 'The weather is lovely today.',
+ "It's so sunny outside!",
+ 'He drove to the stadium.',
+ 'A man is eating a piece of bread.',
+ 'The cat sleeps on the warm windowsill.',
+];
+
+// These models output L2-normalized embeddings, so cosine similarity is the dot
+// product.
+const cosine = (a: Float32Array, b: Float32Array) => {
+ let dot = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i]! * b[i]!;
+ }
+ return dot;
+};
+
+type Entry = { sentence: string; embedding: Float32Array };
+type Match = { sentence: string; similarity: number };
+
+const isDisposedError = (msg: string) => /disposed/i.test(msg);
+
+function TextEmbeddingsContent() {
+ const [selected, setSelected] = useState(0);
+ const { isReady, downloadProgress, error, forward } = useTextEmbeddings(MODELS[selected]!.value);
+
+ const [library, setLibrary] = useState([]);
+ const [input, setInput] = useState('');
+ const [matches, setMatches] = useState(null);
+ const [queryText, setQueryText] = useState('');
+ const [busy, setBusy] = useState(false);
+ const [runError, setRunError] = useState(null);
+ const [embedMs, setEmbedMs] = useState(null);
+
+ const ready = isReady && !!forward;
+
+ // Re-seed the library with starter sentences whenever the model changes.
+ useEffect(() => {
+ if (!ready || !forward) return;
+ let cancelled = false;
+ (async () => {
+ setBusy(true);
+ setRunError(null);
+ try {
+ const entries: Entry[] = [];
+ for (const sentence of STARTER_SENTENCES) {
+ const embedding = await forward(sentence);
+ if (cancelled) return;
+ entries.push({ sentence, embedding });
+ }
+ setLibrary(entries);
+ } catch (e: any) {
+ if (!cancelled) setRunError(e?.message ?? String(e));
+ } finally {
+ if (!cancelled) setBusy(false);
+ }
+ })();
+ return () => {
+ cancelled = true;
+ };
+ }, [forward, ready]);
+
+ const selectModel = (i: number) => {
+ if (i === selected) return;
+ setSelected(i);
+ setLibrary([]);
+ setMatches(null);
+ setQueryText('');
+ setEmbedMs(null);
+ };
+
+ const findSimilar = async () => {
+ if (!forward || !input.trim() || library.length === 0) return;
+ setBusy(true);
+ setRunError(null);
+ try {
+ const start = Date.now();
+ const q = await forward(input.trim());
+ setEmbedMs(Date.now() - start);
+ const ranked = library
+ .map(({ sentence, embedding }) => ({ sentence, similarity: cosine(q, embedding) }))
+ .sort((a, b) => b.similarity - a.similarity);
+ setQueryText(input.trim());
+ setMatches(ranked);
+ } catch (e: any) {
+ const msg = e?.message ?? String(e);
+ if (!isDisposedError(msg)) setRunError(msg);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const addToLibrary = async () => {
+ if (!forward || !input.trim()) return;
+ setBusy(true);
+ setRunError(null);
+ try {
+ const start = Date.now();
+ const embedding = await forward(input.trim());
+ setEmbedMs(Date.now() - start);
+ setLibrary((prev) => [...prev, { sentence: input.trim(), embedding }]);
+ setInput('');
+ setMatches(null);
+ } catch (e: any) {
+ const msg = e?.message ?? String(e);
+ if (!isDisposedError(msg)) setRunError(msg);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const removeAt = (i: number) => {
+ setLibrary((prev) => prev.filter((_, idx) => idx !== i));
+ setMatches(null);
+ };
+
+ const clearLibrary = () => {
+ setLibrary([]);
+ setMatches(null);
+ setQueryText('');
+ };
+
+ return (
+
+
+
+ Text Embeddings
+
+ Semantic search playground. Build a library of sentences, then find the ones closest in
+ meaning to your query using cosine similarity over the embeddings.
+
+
+ Model
+
+ {MODELS.map((m, i) => (
+ selectModel(i)}
+ >
+
+ {m.label}
+
+
+ ))}
+
+
+
+
+
+ {runError && (
+
+ {runError}
+
+ )}
+
+
+
+ Sentence library ({library.length})
+ {library.length > 0 && (
+
+ Clear
+
+ )}
+
+ {library.length === 0 ? (
+
+ {ready ? 'Library is empty — add a sentence below.' : 'Waiting for the model…'}
+
+ ) : (
+ library.map((item, i) => (
+
+ {item.sentence}
+ removeAt(i)} hitSlop={8}>
+ ✕
+
+
+ ))
+ )}
+
+
+
+ Try your sentence
+
+
+
+
+
+ {embedMs !== null && Embedding time: {embedMs} ms}
+
+
+ {matches && (
+
+ Ranked by similarity
+ to “{queryText}”
+ {matches.map((m, i) => {
+ const pct = Math.max(0, Math.min(1, m.similarity)) * 100;
+ return (
+
+
+
+ {m.sentence}
+
+ {m.similarity.toFixed(3)}
+
+
+
+
+
+ );
+ })}
+
+ )}
+
+
+ );
+}
+
+export default function TextEmbeddingsScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ flex: { flex: 1 },
+ container: { flex: 1, backgroundColor: theme.colors.background },
+ content: { padding: theme.spacing.large, paddingBottom: 40 },
+ card: {
+ backgroundColor: theme.colors.cardBackground,
+ borderRadius: theme.radius.large,
+ padding: 20,
+ marginBottom: 20,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ cardTitle: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: 8,
+ },
+ cardDescription: {
+ fontSize: 14,
+ color: theme.colors.textMuted,
+ lineHeight: 20,
+ marginBottom: 16,
+ },
+ fieldLabel: {
+ fontSize: 12,
+ fontWeight: '600',
+ color: theme.colors.textPlaceholder,
+ textTransform: 'uppercase',
+ letterSpacing: 0.5,
+ marginBottom: 8,
+ },
+ chipRow: { gap: 8, paddingBottom: 4, marginBottom: 12 },
+ chip: {
+ paddingHorizontal: 14,
+ paddingVertical: 8,
+ borderRadius: 20,
+ backgroundColor: '#f1f3f5',
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ chipActive: {
+ backgroundColor: theme.colors.strongPrimary,
+ borderColor: theme.colors.strongPrimary,
+ },
+ chipText: { fontSize: 13, fontWeight: '600', color: theme.colors.textMuted },
+ chipTextActive: { color: '#fff' },
+ sectionTitle: { fontSize: 16, fontWeight: '700', color: '#212529' },
+ libraryHeader: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ marginBottom: 12,
+ },
+ clearLink: { fontSize: 13, fontWeight: '600', color: theme.colors.accent },
+ emptyText: { fontSize: 14, color: theme.colors.textPlaceholder, fontStyle: 'italic' },
+ libraryRow: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ justifyContent: 'space-between',
+ paddingVertical: 8,
+ borderBottomWidth: 1,
+ borderBottomColor: '#f1f3f5',
+ },
+ librarySentence: { flex: 1, fontSize: 14, color: '#495057', marginRight: 10 },
+ removeBtn: { fontSize: 15, color: theme.colors.textPlaceholder, fontWeight: '700' },
+ input: {
+ backgroundColor: '#f1f3f5',
+ borderRadius: theme.radius.small,
+ padding: 12,
+ fontSize: 15,
+ color: '#212529',
+ marginBottom: 16,
+ minHeight: 44,
+ textAlignVertical: 'top',
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ buttonRow: { flexDirection: 'row', gap: theme.spacing.small },
+ statsText: {
+ fontSize: 13,
+ color: theme.colors.textPlaceholder,
+ marginTop: 12,
+ textAlign: 'center',
+ },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginBottom: 20,
+ },
+ errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' },
+ queryText: { fontSize: 13, color: theme.colors.textMuted, marginTop: 2, marginBottom: 14 },
+ matchRow: { marginBottom: 14 },
+ matchHeader: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ marginBottom: 6,
+ },
+ matchSentence: { flex: 1, fontSize: 14, color: '#495057', marginRight: 10 },
+ matchTop: { fontWeight: '700', color: theme.colors.strongPrimary },
+ matchScore: { fontSize: 13, fontWeight: '700', color: theme.colors.textMuted },
+ barTrack: {
+ height: 8,
+ borderRadius: 4,
+ backgroundColor: '#f1f3f5',
+ overflow: 'hidden',
+ },
+ barFill: { height: 8, borderRadius: 4, backgroundColor: '#adb5bd' },
+ barFillTop: { backgroundColor: theme.colors.accent },
+});
diff --git a/apps/nlp/package.json b/apps/nlp/package.json
index 361bd18f48..4216dc89cb 100644
--- a/apps/nlp/package.json
+++ b/apps/nlp/package.json
@@ -4,7 +4,8 @@
"main": "expo-router/entry",
"react-native-executorch": {
"features": [
- "tokenizer"
+ "tokenizer",
+ "textEmbeddings"
]
},
"scripts": {
diff --git a/packages/react-native-executorch/cpp/core/dtype.cpp b/packages/react-native-executorch/cpp/core/dtype.cpp
index d43418a7c9..b183daf0d6 100644
--- a/packages/react-native-executorch/cpp/core/dtype.cpp
+++ b/packages/react-native-executorch/cpp/core/dtype.cpp
@@ -9,10 +9,13 @@ DType parseDType(const std::string &s) {
if (s == "int32") {
return DType::int32;
}
+ if (s == "int64") {
+ return DType::int64;
+ }
if (s == "float32") {
return DType::float32;
}
- throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', or 'float32'");
+ throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', 'int64', or 'float32'");
}
std::string toString(DType dtype) {
@@ -21,6 +24,8 @@ std::string toString(DType dtype) {
return "uint8";
case DType::int32:
return "int32";
+ case DType::int64:
+ return "int64";
case DType::float32:
return "float32";
}
@@ -32,6 +37,8 @@ executorch::aten::ScalarType toScalarType(DType dtype) {
return executorch::aten::ScalarType::Byte;
case DType::int32:
return executorch::aten::ScalarType::Int;
+ case DType::int64:
+ return executorch::aten::ScalarType::Long;
case DType::float32:
return executorch::aten::ScalarType::Float;
}
@@ -43,6 +50,8 @@ DType fromScalarType(executorch::aten::ScalarType st) {
return DType::uint8;
case executorch::aten::ScalarType::Int:
return DType::int32;
+ case executorch::aten::ScalarType::Long:
+ return DType::int64;
case executorch::aten::ScalarType::Float:
return DType::float32;
default:
@@ -57,6 +66,8 @@ size_t elementSize(DType dtype) {
// NOLINTNEXTLINE(bugprone-branch-clone): int32 and float32 are both 4 bytes; the identical branches are intentional.
case DType::int32:
return 4;
+ case DType::int64:
+ return 8;
case DType::float32:
return 4;
}
diff --git a/packages/react-native-executorch/cpp/core/dtype.h b/packages/react-native-executorch/cpp/core/dtype.h
index 7259b6b18f..fafa0a8429 100644
--- a/packages/react-native-executorch/cpp/core/dtype.h
+++ b/packages/react-native-executorch/cpp/core/dtype.h
@@ -8,6 +8,7 @@ namespace rnexecutorch::core::types {
enum class DType {
uint8,
int32,
+ int64,
float32
};
diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp
index 9e5ca8d9c7..642849a315 100644
--- a/packages/react-native-executorch/cpp/core/model.cpp
+++ b/packages/react-native-executorch/cpp/core/model.cpp
@@ -1,10 +1,14 @@
#include "model.h"
+#include
#include
+#include
#include
#include
+#include
#include
#include
+#include
#include "dtype.h"
#include "tensor_helpers.h"
@@ -57,6 +61,44 @@ jsi::Object tensorMetaToJs(jsi::Runtime &rt, const executorch::runtime::TensorIn
return jsTensorMeta;
}
+
+std::vector>
+parseDynamicDims(executorch::extension::Module &module) {
+ std::vector> bounds;
+
+ auto methodNames = module.method_names();
+ if (!methodNames.ok() || !methodNames->contains("get_dynamic_dims")) {
+ return bounds;
+ }
+
+ auto result = module.execute("get_dynamic_dims");
+ if (!result.ok() || result->empty() || !result->at(0).isTensor()) {
+ throw std::runtime_error("get_dynamic_dims is present but did not return a tensor");
+ }
+
+ const auto boundsTensor = result->at(0).toTensor();
+ if (boundsTensor.scalar_type() != executorch::aten::ScalarType::Long || boundsTensor.dim() != 2 ||
+ boundsTensor.size(1) != 3) {
+ throw std::runtime_error("get_dynamic_dims must return an int64 [D, 3] tensor of "
+ "[min, max, step] rows");
+ }
+
+ const auto *data = boundsTensor.const_data_ptr();
+ const auto rows = boundsTensor.size(0);
+ bounds.reserve(static_cast(rows));
+ for (int64_t r = 0; r < rows; ++r) {
+ const int64_t minDim = data[r * 3];
+ const int64_t maxDim = data[r * 3 + 1];
+ const int64_t step = data[r * 3 + 2];
+ if (maxDim < minDim || step < 1) {
+ throw std::runtime_error(std::format("get_dynamic_dims row {} is invalid: expected "
+ "min <= max and step >= 1 but got [{}, {}, {}]",
+ r, minDim, maxDim, step));
+ }
+ bounds.push_back({minDim, maxDim, step});
+ }
+ return bounds;
+}
} // namespace
namespace rnexecutorch::core::model {
@@ -73,6 +115,10 @@ ModelHostObject::ModelHostObject(const std::string &modelPath)
const std::string errorMsg = executorch::runtime::to_string(error);
throw std::runtime_error(std::format("Failed to load model: {}", errorMsg));
}
+
+ if (auto bounds = parseDynamicDims(*etModule_); !bounds.empty()) {
+ dynamicInputBounds_.emplace("forward", std::move(bounds));
+ }
}
jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
@@ -214,6 +260,15 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
std::vector> tensorLocks;
std::unordered_set lockedTensors;
+ // Per-dimension [min, max, step] bounds parsed from get_dynamic_dims
+ // at construction. Absent for statically shaped methods, which then
+ // validate exactly.
+ const std::vector> noBounds;
+ auto boundsIt = self->dynamicInputBounds_.find(methodName);
+ const auto &dynamicInputBounds =
+ boundsIt != self->dynamicInputBounds_.end() ? boundsIt->second : noBounds;
+ size_t boundsOffset = 0;
+
for (size_t i = 0; i < methodMeta.num_inputs(); ++i) {
auto ctx = std::format("execute: inputs[{}]", i);
auto tag = unwrap(rt, ctx, methodMeta.input_tag(i));
@@ -223,7 +278,36 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
case executorch::runtime::Tag::Tensor: {
auto tensorMeta = unwrap(rt, ctx + ": tensor meta", methodMeta.input_tensor_meta(i));
auto expectedDtype = fromScalarType(rt, ctx, tensorMeta.scalar_type());
- auto tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes());
+
+ std::shared_ptr tensorHostObject;
+ if (dynamicInputBounds.empty()) {
+ tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes());
+ } else {
+ // Map bounds by the method-declared rank so mapping is
+ // independent of the caller-supplied shape.
+ const auto rank = tensorMeta.sizes().size();
+ if (boundsOffset + rank > dynamicInputBounds.size()) {
+ throw jsi::JSError(rt, std::format("execute: get_dynamic_dims declares fewer "
+ "dimensions ({}) than forward's tensor "
+ "inputs require",
+ dynamicInputBounds.size()));
+ }
+ tensor::SymbolicShape expectedShape;
+ expectedShape.reserve(rank);
+ for (size_t d = 0; d < rank; ++d) {
+ const auto &row = dynamicInputBounds[boundsOffset + d];
+ tensor::RangeDim rangeDim;
+ rangeDim.min = static_cast(row[0]);
+ rangeDim.max = static_cast(row[1]);
+ if (row[2] > 1) {
+ rangeDim.step = static_cast(row[2]);
+ }
+ expectedShape.emplace_back(rangeDim);
+ }
+ boundsOffset += rank;
+ tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype,
+ std::optional(std::move(expectedShape)));
+ }
if (!lockedTensors.insert(tensorHostObject.get()).second) {
throw jsi::JSError(rt, "execute: Tensor aliasing detected. "
@@ -251,6 +335,12 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
}
}
+ if (!dynamicInputBounds.empty() && boundsOffset != dynamicInputBounds.size()) {
+ throw jsi::JSError(rt, std::format("execute: get_dynamic_dims declares more dimensions ({}) "
+ "than forward's tensor inputs use ({})",
+ dynamicInputBounds.size(), boundsOffset));
+ }
+
auto startTime = std::chrono::high_resolution_clock::now();
auto executeResult = self->etModule_->execute(methodName, inputs);
auto finishTime = std::chrono::high_resolution_clock::now();
diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h
index af8d23dd61..569f030f36 100644
--- a/packages/react-native-executorch/cpp/core/model.h
+++ b/packages/react-native-executorch/cpp/core/model.h
@@ -1,8 +1,11 @@
#pragma once
+#include
+#include
#include
#include
#include
+#include
#include
#include
@@ -31,6 +34,8 @@ class ModelHostObject : public jsi::HostObject,
std::string modelPath_;
std::unique_ptr etModule_;
std::mutex mutex_;
+
+ std::unordered_map>> dynamicInputBounds_;
};
void install_loadModel(jsi::Runtime &rt, jsi::Object &module);
diff --git a/packages/react-native-executorch/cpp/extensions/cv/utils.h b/packages/react-native-executorch/cpp/extensions/cv/utils.h
index a29961d0b6..710124b1d9 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/utils.h
+++ b/packages/react-native-executorch/cpp/extensions/cv/utils.h
@@ -14,6 +14,8 @@ inline int dtypeToCvDepth(rnexecutorch::core::types::DType dtype) {
return CV_32S;
case rnexecutorch::core::types::DType::float32:
return CV_32F;
+ case rnexecutorch::core::types::DType::int64:
+ break;
}
throw std::invalid_argument("unsupported dtype");
}
diff --git a/packages/react-native-executorch/src/core/tensor.ts b/packages/react-native-executorch/src/core/tensor.ts
index 8f5c58e10c..4c54c0d32f 100644
--- a/packages/react-native-executorch/src/core/tensor.ts
+++ b/packages/react-native-executorch/src/core/tensor.ts
@@ -6,7 +6,7 @@ declare const tensorBrand: unique symbol;
* Element data type of a {@link Tensor}.
* @category Types
*/
-export type DType = 'float32' | 'uint8' | 'int32';
+export type DType = 'float32' | 'uint8' | 'int32' | 'int64';
/**
* A native ExecuTorch tensor allocated in C++ memory.
@@ -50,10 +50,10 @@ export type Tensor = {
/**
* Writes data from a typed array into this tensor's native buffer.
* @param src The source typed array. Its size in bytes must match the
- * tensor's size.
+ * tensor's size. Use a `BigInt64Array` for `int64` tensors.
* @returns `this` tensor.
*/
- setData(src: Float32Array | Uint8Array | Int32Array): Tensor;
+ setData(src: Float32Array | Uint8Array | Int32Array | BigInt64Array): Tensor;
/**
* Copies data out of this tensor's native buffer into a typed array.
@@ -62,7 +62,7 @@ export type Tensor = {
* tensor's size.
* @returns The same `dst` array, now filled with tensor data.
*/
- getData(dst: T): T;
+ getData(dst: T): T;
/**
* Passes `this` tensor as the first argument to `fn` and returns the result.
@@ -115,7 +115,7 @@ export type Tensor = {
export function tensor(
dtype: DType,
shape: number[],
- src?: Float32Array | Uint8Array | Int32Array
+ src?: Float32Array | Uint8Array | Int32Array | BigInt64Array
): Tensor {
'worklet';
const t: Tensor = rnexecutorchJsi.createTensor(shape, dtype);
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbeddings.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbeddings.ts
new file mode 100644
index 0000000000..65274c6494
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbeddings.ts
@@ -0,0 +1,88 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+
+import type { ImageBuffer } from '../image';
+import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
+
+/**
+ * Model configuration required to instantiate an image embeddings task runner.
+ * @category Types
+ */
+export type ImageEmbeddingsModel = {
+ readonly modelPath: string;
+ readonly opts: ImagePreprocessorOptions;
+};
+
+/**
+ * Creates an image embeddings runner for executing local Image Embedding
+ * models (e.g. the image encoder of a CLIP model).
+ *
+ * It validates the model input and output requirements, pre-allocates the
+ * static execution tensors, sets up an image preprocessor, and registers clean
+ * disposal hooks to clear all native memory. Pooling and normalization (if any)
+ * are baked into the exported `.pte`; this runner simply preprocesses the image,
+ * runs the forward pass, and returns the raw embedding vector.
+ * @category Typescript API
+ * @param config Image embeddings task configuration containing path and options.
+ * @param runtime Optional worklet runtime thread on which to run the model
+ * execution.
+ * @returns A promise resolving to an object containing the embedding and
+ * disposal controls.
+ */
+export async function createImageEmbeddings(
+ config: ImageEmbeddingsModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Asynchronously computes the embedding vector for the given input image.
+ * @param input The input image buffer.
+ * @returns A promise resolving to the embedding vector.
+ */
+ forward: (input: ImageBuffer) => Promise;
+ /**
+ * Synchronous version of {@link forward} to be executed directly on the
+ * caller or worklet thread.
+ */
+ forwardWorklet: (input: ImageBuffer) => Float32Array;
+}> {
+ const { modelPath, opts } = config;
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 'D'], ['D'])]
+ );
+ const inpShape = meta.inputTensorMeta[0]!.shape;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+
+ const tensors = [tensor('float32', outShape)] as const;
+ const [tEmbedding] = tensors;
+ const preprocessor = createImagePreprocessor(opts, inpShape);
+
+ const dispose = () => {
+ preprocessor.dispose();
+ tensors.forEach((t) => t.dispose());
+ model.dispose();
+ };
+
+ const forwardWorklet = (input: ImageBuffer): Float32Array => {
+ 'worklet';
+ const tInput = preprocessor.process(input);
+ model.execute('forward', [tInput], [tEmbedding]);
+ return tEmbedding.getData(new Float32Array(tEmbedding.numel));
+ };
+
+ const forward = wrapAsync(forwardWorklet, runtime);
+
+ return { forward, forwardWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/extensions/nlp/index.ts b/packages/react-native-executorch/src/extensions/nlp/index.ts
index 6195f07395..8d3975ad9c 100644
--- a/packages/react-native-executorch/src/extensions/nlp/index.ts
+++ b/packages/react-native-executorch/src/extensions/nlp/index.ts
@@ -1,2 +1,3 @@
export * from './tokenizer';
export * from './tasks/tokenization';
+export * from './tasks/textEmbeddings';
diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbeddings.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbeddings.ts
new file mode 100644
index 0000000000..a2bc28fa5e
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbeddings.ts
@@ -0,0 +1,119 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+
+import { loadTokenizer } from '../tokenizer';
+
+/**
+ * Model configuration required to instantiate a text embeddings task runner.
+ * @category Types
+ */
+export type TextEmbeddingsModel = {
+ readonly modelPath: string;
+ readonly tokenizerPath: string;
+};
+
+/**
+ * Creates a text embeddings runner for executing local Text Embedding models
+ * (e.g. sentence-transformers like all-MiniLM-L6-v2).
+ *
+ * It loads the tokenizer and model, validates the model input and output
+ * requirements, pre-allocates the static execution tensors, and registers clean
+ * disposal hooks to clear all native memory. The input text is tokenized and fed
+ * at its exact token length (no padding), truncated only when it exceeds the
+ * model's maximum sequence length; the attention mask is all ones. Pooling and
+ * normalization are baked into the exported `.pte`; this runner runs the forward
+ * pass and returns the raw embedding vector.
+ * @category Typescript API
+ * @param config Text embeddings task configuration containing the model and
+ * tokenizer paths.
+ * @param runtime Optional worklet runtime thread on which to run the model
+ * execution.
+ * @returns A promise resolving to an object containing the embedding and
+ * disposal controls.
+ */
+export async function createTextEmbeddings(
+ config: TextEmbeddingsModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Asynchronously computes the embedding vector for the given input text.
+ * @param input The input text to embed.
+ * @returns A promise resolving to the embedding vector.
+ */
+ forward: (input: string) => Promise;
+ /**
+ * Synchronous version of {@link forward} to be executed directly on the
+ * caller or worklet thread.
+ */
+ forwardWorklet: (input: string) => Float32Array;
+}> {
+ const { modelPath, tokenizerPath } = config;
+ const [model, tokenizer] = await Promise.all([
+ wrapAsync(loadModel, runtime)(modelPath),
+ wrapAsync(loadTokenizer, runtime)(tokenizerPath),
+ ]);
+
+ // Text embedding models take two int64 inputs: the token ids and the
+ // attention mask, both of shape [1, sequence_length].
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('int64', [1, 'L']), SymbolicTensor('int64', [1, 'L'])],
+ [SymbolicTensor('float32', [1, 'D'], ['D'])]
+ );
+ // The models are exported with a dynamic sequence dimension; the declared size
+ // is the upper bound, used only to truncate over-long inputs.
+ const maxSeqLen = meta.inputTensorMeta[0]!.shape[1]!;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+
+ const tensors = [tensor('float32', outShape)] as const;
+ const [tEmbedding] = tensors;
+
+ const dispose = () => {
+ tensors.forEach((t) => t.dispose());
+ tokenizer.dispose();
+ model.dispose();
+ };
+
+ const forwardWorklet = (input: string): Float32Array => {
+ 'worklet';
+ const ids = tokenizer.encode(input);
+ if (ids.length === 0) {
+ throw new Error('createTextEmbeddings: input tokenized to zero tokens');
+ }
+ const len = Math.min(ids.length, maxSeqLen);
+
+ // Feed the exact token length with no padding. The model resizes its dynamic
+ // sequence input to match. Padding would change the result for pooling heads
+ // that are sensitive to it (e.g. DistilUSE's tanh projection). The attention
+ // mask is all ones since every position is a real token.
+ const idsData = new BigInt64Array(len);
+ const maskData = new BigInt64Array(len);
+ for (let i = 0; i < len; i++) {
+ idsData[i] = BigInt(ids[i]!);
+ maskData[i] = 1n;
+ }
+
+ const tokenIds = tensor('int64', [1, len], idsData);
+ const attentionMask = tensor('int64', [1, len], maskData);
+ try {
+ model.execute('forward', [tokenIds, attentionMask], [tEmbedding]);
+ return tEmbedding.getData(new Float32Array(tEmbedding.numel));
+ } finally {
+ tokenIds.dispose();
+ attentionMask.dispose();
+ }
+ };
+
+ const forward = wrapAsync(forwardWorklet, runtime);
+
+ return { forward, forwardWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/hooks/useImageEmbeddings.ts b/packages/react-native-executorch/src/hooks/useImageEmbeddings.ts
new file mode 100644
index 0000000000..476bd5c64b
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useImageEmbeddings.ts
@@ -0,0 +1,45 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createImageEmbeddings,
+ type ImageEmbeddingsModel,
+} from '../extensions/cv/tasks/imageEmbeddings';
+
+/**
+ * React hook to load and run an image embeddings model.
+ *
+ * This hook manages downloading (if it's a remote URL) and loading the model
+ * file, compiling it, tracking download progress and compilation errors, and
+ * cleaning up native model memory when the component unmounts or configuration
+ * changes.
+ * @category Hooks
+ * @param config The image embeddings model configuration.
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, and embedding functions.
+ */
+export function useImageEmbeddings(
+ config: ImageEmbeddingsModel,
+ options?: { preventLoad?: boolean }
+) {
+ const { localPath, downloadProgress, downloadError } = useResourceDownload(
+ config.modelPath,
+ options?.preventLoad
+ );
+ const { model, error } = useModel(
+ createImageEmbeddings,
+ localPath ? { ...config, modelPath: localPath } : null,
+ [localPath]
+ );
+
+ return {
+ isReady: !!model,
+ error: downloadError || error,
+ downloadProgress,
+ localPath,
+ forward: model?.forward,
+ forwardWorklet: model?.forwardWorklet,
+ };
+}
diff --git a/packages/react-native-executorch/src/hooks/useTextEmbeddings.ts b/packages/react-native-executorch/src/hooks/useTextEmbeddings.ts
new file mode 100644
index 0000000000..588ebda012
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useTextEmbeddings.ts
@@ -0,0 +1,51 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createTextEmbeddings,
+ type TextEmbeddingsModel,
+} from '../extensions/nlp/tasks/textEmbeddings';
+
+/**
+ * React hook to load and run a text embeddings model.
+ *
+ * This hook manages downloading (if they are remote URLs) and loading both the
+ * model file and its `tokenizer.json`, tracking download progress and errors,
+ * and cleaning up native memory when the component unmounts or the configuration
+ * changes.
+ * @category Hooks
+ * @param config The text embeddings model configuration (model and tokenizer
+ * paths).
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, and embedding functions.
+ */
+export function useTextEmbeddings(
+ config: TextEmbeddingsModel,
+ options?: { preventLoad?: boolean }
+) {
+ const modelResource = useResourceDownload(config.modelPath, options?.preventLoad);
+ const tokenizerResource = useResourceDownload(config.tokenizerPath, options?.preventLoad);
+
+ const localModelPath = modelResource.localPath;
+ const localTokenizerPath = tokenizerResource.localPath;
+
+ const { model, error } = useModel(
+ createTextEmbeddings,
+ localModelPath && localTokenizerPath
+ ? { modelPath: localModelPath, tokenizerPath: localTokenizerPath }
+ : null,
+ [localModelPath, localTokenizerPath]
+ );
+
+ return {
+ isReady: !!model,
+ error: modelResource.downloadError || tokenizerResource.downloadError || error,
+ downloadProgress: (modelResource.downloadProgress + tokenizerResource.downloadProgress) / 2,
+ localPath: localModelPath,
+ tokenizerPath: localTokenizerPath,
+ forward: model?.forward,
+ forwardWorklet: model?.forwardWorklet,
+ };
+}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index 00557be78b..1e36c8f0f6 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -6,6 +6,8 @@ export * from './hooks/useInstanceSegmenter';
export * from './hooks/useKeypointDetector';
export * from './hooks/useObjectDetector';
export * from './hooks/useTokenizer';
+export * from './hooks/useTextEmbeddings';
+export * from './hooks/useImageEmbeddings';
export * from './hooks/useResourceDownload';
export * from './hooks/useModel';
@@ -20,7 +22,9 @@ export * from './extensions/cv/tasks/semanticSegmentation';
export * from './extensions/cv/tasks/instanceSegmentation';
export * from './extensions/cv/tasks/keypointDetection';
export * from './extensions/cv/tasks/objectDetection';
+export * from './extensions/cv/tasks/imageEmbeddings';
export * from './extensions/nlp/tasks/tokenization';
+export * from './extensions/nlp/tasks/textEmbeddings';
// Core primitives — for library builders and power users
export { tensor } from './core/tensor';
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index 3397288e23..06e00211da 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -4,6 +4,8 @@ import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer';
import type { SemanticSegmentationModel } from './extensions/cv/tasks/semanticSegmentation';
import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection';
import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation';
+import type { ImageEmbeddingsModel } from './extensions/cv/tasks/imageEmbeddings';
+import type { TextEmbeddingsModel } from './extensions/nlp/tasks/textEmbeddings';
import {
IMAGENET_NORM,
IMAGENET1K_LABELS,
@@ -528,6 +530,56 @@ const YOLO26_XLARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoCla
opts: YOLO26_SEG_OPTS,
};
+// =============================================================================
+// Text Embeddings
+// =============================================================================
+const ALL_MINILM_L6_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${VERSION_TAG}/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${VERSION_TAG}/tokenizer.json`,
+};
+const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-all-mpnet-base-v2/${VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${VERSION_TAG}/tokenizer.json`,
+};
+const MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${VERSION_TAG}/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${VERSION_TAG}/tokenizer.json`,
+};
+const MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${VERSION_TAG}/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${VERSION_TAG}/tokenizer.json`,
+};
+const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte`,
+ tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${VERSION_TAG}/tokenizer.json`,
+};
+const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte`,
+ tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`,
+};
+const CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_text_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`,
+};
+
+// =============================================================================
+// Image Embeddings
+// =============================================================================
+const CLIP_IMAGE_EMBEDDINGS_OPTS = {
+ resizeMode: 'stretch' as const,
+ interpolation: 'linear' as const,
+ alpha: 1 / 255.0,
+ beta: 0.0,
+};
+const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32: ImageEmbeddingsModel = {
+ modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_fp32.pte`,
+ opts: CLIP_IMAGE_EMBEDDINGS_OPTS,
+};
+const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8: ImageEmbeddingsModel = {
+ modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_int8.pte`,
+ opts: CLIP_IMAGE_EMBEDDINGS_OPTS,
+};
+
// =============================================================================
// Tokenizers
// =============================================================================
@@ -737,4 +789,41 @@ export const models = {
tokenizer: {
ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER,
},
+ textEmbeddings: {
+ ALL_MINILM_L6_V2: {
+ ...ALL_MINILM_L6_V2_EMBEDDINGS,
+ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS,
+ },
+ ALL_MPNET_BASE_V2: {
+ ...ALL_MPNET_BASE_V2_EMBEDDINGS,
+ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS,
+ },
+ MULTI_QA_MINILM_L6_COS_V1: {
+ ...MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS,
+ XNNPACK_FP32: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS,
+ },
+ MULTI_QA_MPNET_BASE_DOT_V1: {
+ ...MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS,
+ XNNPACK_FP32: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS,
+ },
+ PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: {
+ ...PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS,
+ XNNPACK_8DA4W: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS,
+ },
+ DISTILUSE_BASE_MULTILINGUAL_CASED_V2: {
+ ...DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS,
+ XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS,
+ },
+ CLIP_VIT_BASE_PATCH32_TEXT: {
+ ...CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS,
+ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS,
+ },
+ },
+ imageEmbeddings: {
+ CLIP_VIT_BASE_PATCH32: {
+ ...CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32,
+ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32,
+ XNNPACK_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8,
+ },
+ },
};