Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/computer-vision/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"expo-router": "~56.2.9",
"react": "19.2.3",
"react-native": "0.85.3",
"react-native-blob-util": "^0.24.0",
"react-native-drawer-layout": "^4.2.2",
"react-native-executorch": "workspace:*",
"react-native-gesture-handler": "~2.31.1",
Expand Down
1 change: 1 addition & 0 deletions apps/nlp/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"expo-router": "~56.2.9",
"react": "19.2.3",
"react-native": "0.85.3",
"react-native-blob-util": "^0.24.0",
"react-native-drawer-layout": "^4.2.2",
"react-native-executorch": "workspace:*",
"react-native-gesture-handler": "~2.31.1",
Expand Down
1 change: 1 addition & 0 deletions apps/speech/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"react": "19.2.3",
"react-native": "0.85.3",
"react-native-audio-api": "0.13.1",
"react-native-blob-util": "^0.24.0",
"react-native-device-info": "^15.0.2",
"react-native-drawer-layout": "^4.2.2",
"react-native-executorch": "workspace:*",
Expand Down
4 changes: 2 additions & 2 deletions packages/react-native-executorch/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@
"peerDependencies": {
"react": "*",
"react-native": "*",
"react-native-fs": "^2.20.0",
"react-native-blob-util": "^0.24.0",
"react-native-worklets": "*"
},
"devDependencies": {
Expand All @@ -96,8 +96,8 @@
"del-cli": "^6.0.0",
"react": "19.2.0",
"react-native": "0.83.6",
"react-native-blob-util": "^0.24.0",
"react-native-builder-bob": "^0.40.18",
"react-native-fs": "^2.20.0",
"react-native-worklets": "0.9.1",
"typescript": "~5.9.2"
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import { softmax } from '../../math';
import type { ImageBuffer } from '../image';
Expand Down Expand Up @@ -75,7 +76,7 @@ export async function createClassifier<L>(
classifyWorklet: (input: ImageBuffer, options?: { topk?: number }) => Classification<L>[];
}> {
const { modelPath, classifierOpts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));

const meta = validateModelSchema(
model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import type { ImageBuffer } from '../image';
import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
Expand Down Expand Up @@ -54,7 +55,7 @@ export async function createImageEmbedder(
embedWorklet: (input: ImageBuffer) => Float32Array;
}> {
const { modelPath, opts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));

const meta = validateModelSchema(
model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import type { ImageBuffer } from '../image';
import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
Expand Down Expand Up @@ -114,7 +115,7 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
) => InstanceSegmentationResult<F, L>[];
}> {
const { modelPath, opts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));
const meta = validateModelSchema(
model,
'forward',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor, type Tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import type { ImageBuffer } from '../image';
import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
Expand Down Expand Up @@ -180,7 +181,7 @@ export async function createKeypointDetector<F extends BoxFormat, L extends Prop
}> {
const { modelPath, opts } = config;
const { landmarks } = opts;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));
const meta = validateModelSchema(
model,
'forward',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import type { ResizeMode } from '../ops/image';
import type { ImageBuffer } from '../image';
Expand Down Expand Up @@ -94,7 +95,7 @@ export async function createObjectDetector<F extends BoxFormat, L>(
) => ObjectDetection<F, L>[];
}> {
const { modelPath, opts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));

const meta = validateModelSchema(
model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';
import { randomNormal } from '../../math';
import { loadTokenizer } from '../../nlp/tokenizer';

Expand Down Expand Up @@ -76,8 +77,9 @@ export async function createSdxsTextToImage(
generateWorklet: (prompt: string, seed?: number) => ImageBuffer;
}> {
const { modelPath, tokenizerPath } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath);
const [localModelPath, localTokenizerPath] = await download([modelPath, tokenizerPath]);
const model = await wrapAsync(loadModel, runtime)(localModelPath);
const tokenizer = await wrapAsync(loadTokenizer, runtime)(localTokenizerPath);

validateModelSchema(
model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import type { ImageBuffer } from '../image';
import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
Expand Down Expand Up @@ -119,7 +120,7 @@ export async function createSemanticSegmenter<L extends PropertyKey = string>(
) => SemanticSegmentationResult<L>;
}> {
const { modelPath, opts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));

const meta = validateModelSchema(
model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import type { ImageBuffer } from '../image';
import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
Expand Down Expand Up @@ -67,7 +68,7 @@ export async function createStyleTransfer(
transferStyleWorklet: (input: ImageBuffer) => ImageBuffer;
}> {
const { modelPath, opts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));

const meta = validateModelSchema(
model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import { loadTokenizer } from '../tokenizer';

Expand Down Expand Up @@ -60,9 +61,10 @@ export async function createTextEmbedder(
embedWorklet: (input: string, prompt?: string) => Float32Array;
}> {
const { modelPath, tokenizerPath, defaultPrompt } = config;
const [localModelPath, localTokenizerPath] = await download([modelPath, tokenizerPath]);
const [model, tokenizer] = await Promise.all([
wrapAsync(loadModel, runtime)(modelPath),
wrapAsync(loadTokenizer, runtime)(tokenizerPath),
wrapAsync(loadModel, runtime)(localModelPath),
wrapAsync(loadTokenizer, runtime)(localTokenizerPath),
]);

// Text embedding models take two int64 inputs: the token ids and the
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
import type { WorkletRuntime } from 'react-native-worklets';

import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';
import { loadTokenizer } from '../tokenizer';

/**
* Loads a tokenizer and exposes its operations with lifetime management for the
* `useTokenizer` hook.
* @category Typescript API
* @param tokenizerPath Absolute local path to a `tokenizer.json` file.
* @param tokenizerPath Remote URL or local path to a `tokenizer.json` file.
* Remote URLs are downloaded and cached automatically.
* @param runtime Optional worklet runtime thread to run the tokenizer on.
* @returns A promise resolving to the tokenizer operations and a `dispose`
* handle that releases the native tokenizer.
*/
export async function createTokenizer(tokenizerPath: string, runtime?: WorkletRuntime) {
const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath);
const tokenizer = await wrapAsync(loadTokenizer, runtime)(await download(tokenizerPath));
const dispose = () => tokenizer.dispose();

return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';
import { extractFrames } from '../utils/vadUtils';

/**
Expand Down Expand Up @@ -216,7 +217,7 @@ export async function createFsmnVoiceActivityDetector(
}> {
const { modelPath, defaultOptions } = config;

const model = await wrapAsync(loadModel, runtime)(modelPath);
const model = await wrapAsync(loadModel, runtime)(await download(modelPath));

// Input is [frames, fftLength] with a dynamic frame count; the output is
// [1, frames, classes] where class 0 is the non-speech class. The output frame
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
import { wrapAsync } from '../../../core/runtime';
import { download } from '../../../fetcher';

import { argmax } from '../../../extensions/math';
import { loadTokenizer } from '../../nlp/tokenizer';
Expand Down Expand Up @@ -159,8 +160,9 @@ export async function createWhisperSpeechToText<L extends WhisperLanguage = Whis
streamInsert: (audioChunk: Float32Array) => void;
}> {
const { modelPath, tokenizerPath, supportedLanguages, vadModel } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath);
const [localModelPath, localTokenizerPath] = await download([modelPath, tokenizerPath]);
const model = await wrapAsync(loadModel, runtime)(localModelPath);
const tokenizer = await wrapAsync(loadTokenizer, runtime)(localTokenizerPath);
const voiceDetector = await createFsmnVoiceActivityDetector(vadModel, runtime);

const eotToken = tokenizer.tokenToId('<|endoftext|>')!;
Expand Down
Loading