diff --git a/.vitepress/tsconfig.json b/.vitepress/tsconfig.json index 3ec1a3e4..02138974 100644 --- a/.vitepress/tsconfig.json +++ b/.vitepress/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022", "dom"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, diff --git a/llama/addon/addon.cpp b/llama/addon/addon.cpp index 66e45a12..837e0218 100644 --- a/llama/addon/addon.cpp +++ b/llama/addon/addon.cpp @@ -12,7 +12,10 @@ #include "globals/getMemoryInfo.h" #include +#include +#include +std::mutex backendMutex; bool backendInitialized = false; bool backendDisposed = false; @@ -118,6 +121,8 @@ class AddonBackendLoadWorker : public Napi::AsyncWorker { void Execute() { try { + std::lock_guard lock(backendMutex); + llama_backend_init(); try { @@ -163,6 +168,8 @@ class AddonBackendUnloadWorker : public Napi::AsyncWorker { Napi::Promise::Deferred deferred; void Execute() { + std::lock_guard lock(backendMutex); + try { if (backendInitialized) { backendInitialized = false; @@ -234,6 +241,8 @@ Napi::Value markLoaded(const Napi::CallbackInfo& info) { } Napi::Value addonInit(const Napi::CallbackInfo& info) { + std::lock_guard lock(backendMutex); + if (backendInitialized) { Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(info.Env()); deferred.Resolve(info.Env().Undefined()); @@ -246,6 +255,8 @@ Napi::Value addonInit(const Napi::CallbackInfo& info) { } Napi::Value addonDispose(const Napi::CallbackInfo& info) { + std::lock_guard lock(backendMutex); + if (backendDisposed) { Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(info.Env()); deferred.Resolve(info.Env().Undefined()); @@ -259,7 +270,8 @@ Napi::Value addonDispose(const Napi::CallbackInfo& info) { return worker->GetPromise(); } -static void addonFreeLlamaBackend(Napi::Env env, int* data) { +static void addonFreeLlamaBackend() { + std::lock_guard lock(backendMutex); if (backendDisposed) { return; } @@ -270,6 +282,9 @@ static void addonFreeLlamaBackend(Napi::Env env, int* data) { llama_backend_free(); } } +static void addonFreeLlamaBackendFromFinalizer(Napi::Env env, int* data) { + addonFreeLlamaBackend(); +} Napi::Object registerCallback(Napi::Env env, Napi::Object exports) { exports.DefineProperties({ @@ -306,8 +321,8 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) { llama_log_set(addonLlamaCppLogCallback, nullptr); - exports.AddFinalizer(addonFreeLlamaBackend, static_cast(nullptr)); - + exports.AddFinalizer(addonFreeLlamaBackendFromFinalizer, static_cast(nullptr)); + env.AddCleanupHook(addonFreeLlamaBackend); return exports; } diff --git a/package-lock.json b/package-lock.json index b61411f2..119a60df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,12 +27,11 @@ "log-symbols": "^7.0.1", "nanoid": "^5.1.6", "node-addon-api": "^8.5.0", - "octokit": "^5.0.5", "ora": "^9.3.0", "pretty-ms": "^9.3.0", "proper-lockfile": "^4.1.2", "semver": "^7.7.1", - "simple-git": "^3.31.1", + "simple-git": "^3.32.2", "slice-ansi": "^8.0.0", "stdout-update": "^4.0.1", "strip-ansi": "^7.1.2", @@ -2619,116 +2618,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@octokit/app": { - "version": "16.1.2", - "resolved": "https://registry.npmjs.org/@octokit/app/-/app-16.1.2.tgz", - "integrity": "sha512-8j7sEpUYVj18dxvh0KWj6W/l6uAiVRBl1JBDVRqH1VHKAO/G5eRVl4yEoYACjakWers1DjUkcCHyJNQK47JqyQ==", - "license": "MIT", - "dependencies": { - "@octokit/auth-app": "^8.1.2", - "@octokit/auth-unauthenticated": "^7.0.3", - "@octokit/core": "^7.0.6", - "@octokit/oauth-app": "^8.0.3", - "@octokit/plugin-paginate-rest": "^14.0.0", - "@octokit/types": "^16.0.0", - "@octokit/webhooks": "^14.0.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/auth-app": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-8.2.0.tgz", - "integrity": "sha512-vVjdtQQwomrZ4V46B9LaCsxsySxGoHsyw6IYBov/TqJVROrlYdyNgw5q6tQbB7KZt53v1l1W53RiqTvpzL907g==", - "license": "MIT", - "dependencies": { - "@octokit/auth-oauth-app": "^9.0.3", - "@octokit/auth-oauth-user": "^6.0.2", - "@octokit/request": "^10.0.6", - "@octokit/request-error": "^7.0.2", - "@octokit/types": "^16.0.0", - "toad-cache": "^3.7.0", - "universal-github-app-jwt": "^2.2.0", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/auth-oauth-app": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-9.0.3.tgz", - "integrity": "sha512-+yoFQquaF8OxJSxTb7rnytBIC2ZLbLqA/yb71I4ZXT9+Slw4TziV9j/kyGhUFRRTF2+7WlnIWsePZCWHs+OGjg==", - "license": "MIT", - "dependencies": { - "@octokit/auth-oauth-device": "^8.0.3", - "@octokit/auth-oauth-user": "^6.0.2", - "@octokit/request": "^10.0.6", - "@octokit/types": "^16.0.0", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/auth-oauth-device": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-8.0.3.tgz", - "integrity": "sha512-zh2W0mKKMh/VWZhSqlaCzY7qFyrgd9oTWmTmHaXnHNeQRCZr/CXy2jCgHo4e4dJVTiuxP5dLa0YM5p5QVhJHbw==", - "license": "MIT", - "dependencies": { - "@octokit/oauth-methods": "^6.0.2", - "@octokit/request": "^10.0.6", - "@octokit/types": "^16.0.0", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/auth-oauth-user": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-6.0.2.tgz", - "integrity": "sha512-qLoPPc6E6GJoz3XeDG/pnDhJpTkODTGG4kY0/Py154i/I003O9NazkrwJwRuzgCalhzyIeWQ+6MDvkUmKXjg/A==", - "license": "MIT", - "dependencies": { - "@octokit/auth-oauth-device": "^8.0.3", - "@octokit/oauth-methods": "^6.0.2", - "@octokit/request": "^10.0.6", - "@octokit/types": "^16.0.0", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 20" - } - }, "node_modules/@octokit/auth-token": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-6.0.0.tgz", "integrity": "sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==", + "dev": true, "license": "MIT", "engines": { "node": ">= 20" } }, - "node_modules/@octokit/auth-unauthenticated": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/@octokit/auth-unauthenticated/-/auth-unauthenticated-7.0.3.tgz", - "integrity": "sha512-8Jb1mtUdmBHL7lGmop9mU9ArMRUTRhg8vp0T1VtZ4yd9vEm3zcLwmjQkhNEduKawOOORie61xhtYIhTDN+ZQ3g==", - "license": "MIT", - "dependencies": { - "@octokit/request-error": "^7.0.2", - "@octokit/types": "^16.0.0" - }, - "engines": { - "node": ">= 20" - } - }, "node_modules/@octokit/core": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/auth-token": "^6.0.0", @@ -2747,6 +2651,7 @@ "version": "11.0.2", "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-11.0.2.tgz", "integrity": "sha512-4zCpzP1fWc7QlqunZ5bSEjxc6yLAlRTnDwKtgXfcI/FxxGoqedDG8V2+xJ60bV2kODqcGB+nATdtap/XYq2NZQ==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/types": "^16.0.0", @@ -2760,6 +2665,7 @@ "version": "9.0.3", "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-9.0.3.tgz", "integrity": "sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/request": "^10.0.6", @@ -2770,92 +2676,18 @@ "node": ">= 20" } }, - "node_modules/@octokit/oauth-app": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/@octokit/oauth-app/-/oauth-app-8.0.3.tgz", - "integrity": "sha512-jnAjvTsPepyUaMu9e69hYBuozEPgYqP4Z3UnpmvoIzHDpf8EXDGvTY1l1jK0RsZ194oRd+k6Hm13oRU8EoDFwg==", - "license": "MIT", - "dependencies": { - "@octokit/auth-oauth-app": "^9.0.2", - "@octokit/auth-oauth-user": "^6.0.1", - "@octokit/auth-unauthenticated": "^7.0.2", - "@octokit/core": "^7.0.5", - "@octokit/oauth-authorization-url": "^8.0.0", - "@octokit/oauth-methods": "^6.0.1", - "@types/aws-lambda": "^8.10.83", - "universal-user-agent": "^7.0.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/oauth-authorization-url": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-8.0.0.tgz", - "integrity": "sha512-7QoLPRh/ssEA/HuHBHdVdSgF8xNLz/Bc5m9fZkArJE5bb6NmVkDm3anKxXPmN1zh6b5WKZPRr3697xKT/yM3qQ==", - "license": "MIT", - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/oauth-methods": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@octokit/oauth-methods/-/oauth-methods-6.0.2.tgz", - "integrity": "sha512-HiNOO3MqLxlt5Da5bZbLV8Zarnphi4y9XehrbaFMkcoJ+FL7sMxH/UlUsCVxpddVu4qvNDrBdaTVE2o4ITK8ng==", - "license": "MIT", - "dependencies": { - "@octokit/oauth-authorization-url": "^8.0.0", - "@octokit/request": "^10.0.6", - "@octokit/request-error": "^7.0.2", - "@octokit/types": "^16.0.0" - }, - "engines": { - "node": ">= 20" - } - }, "node_modules/@octokit/openapi-types": { "version": "27.0.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "dev": true, "license": "MIT" }, - "node_modules/@octokit/openapi-webhooks-types": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-webhooks-types/-/openapi-webhooks-types-12.1.0.tgz", - "integrity": "sha512-WiuzhOsiOvb7W3Pvmhf8d2C6qaLHXrWiLBP4nJ/4kydu+wpagV5Fkz9RfQwV2afYzv3PB+3xYgp4mAdNGjDprA==", - "license": "MIT" - }, - "node_modules/@octokit/plugin-paginate-graphql": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-6.0.0.tgz", - "integrity": "sha512-crfpnIoFiBtRkvPqOyLOsw12XsveYuY2ieP6uYDosoUegBJpSVxGwut9sxUgFFcll3VTOTqpUf8yGd8x1OmAkQ==", - "license": "MIT", - "engines": { - "node": ">= 20" - }, - "peerDependencies": { - "@octokit/core": ">=6" - } - }, "node_modules/@octokit/plugin-paginate-rest": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", "integrity": "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==", - "license": "MIT", - "dependencies": { - "@octokit/types": "^16.0.0" - }, - "engines": { - "node": ">= 20" - }, - "peerDependencies": { - "@octokit/core": ">=6" - } - }, - "node_modules/@octokit/plugin-rest-endpoint-methods": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-17.0.0.tgz", - "integrity": "sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/types": "^16.0.0" @@ -2871,6 +2703,7 @@ "version": "8.0.3", "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-8.0.3.tgz", "integrity": "sha512-vKGx1i3MC0za53IzYBSBXcrhmd+daQDzuZfYDd52X5S0M2otf3kVZTVP8bLA3EkU0lTvd1WEC2OlNNa4G+dohA==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/request-error": "^7.0.2", @@ -2888,6 +2721,7 @@ "version": "11.0.3", "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-11.0.3.tgz", "integrity": "sha512-34eE0RkFCKycLl2D2kq7W+LovheM/ex3AwZCYN8udpi6bxsyjZidb2McXs69hZhLmJlDqTSP8cH+jSRpiaijBg==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/types": "^16.0.0", @@ -2904,6 +2738,7 @@ "version": "10.0.7", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-10.0.7.tgz", "integrity": "sha512-v93h0i1yu4idj8qFPZwjehoJx4j3Ntn+JhXsdJrG9pYaX6j/XRz2RmasMUHtNgQD39nrv/VwTWSqK0RNXR8upA==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/endpoint": "^11.0.2", @@ -2920,6 +2755,7 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-7.1.0.tgz", "integrity": "sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/types": "^16.0.0" @@ -2932,34 +2768,12 @@ "version": "16.0.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "dev": true, "license": "MIT", "dependencies": { "@octokit/openapi-types": "^27.0.0" } }, - "node_modules/@octokit/webhooks": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/@octokit/webhooks/-/webhooks-14.2.0.tgz", - "integrity": "sha512-da6KbdNCV5sr1/txD896V+6W0iamFWrvVl8cHkBSPT+YlvmT3DwXa4jxZnQc+gnuTEqSWbBeoSZYTayXH9wXcw==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-webhooks-types": "12.1.0", - "@octokit/request-error": "^7.0.0", - "@octokit/webhooks-methods": "^6.0.0" - }, - "engines": { - "node": ">= 20" - } - }, - "node_modules/@octokit/webhooks-methods": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@octokit/webhooks-methods/-/webhooks-methods-6.0.0.tgz", - "integrity": "sha512-MFlzzoDJVw/GcbfzVC1RLR36QqkTLUf79vLVO3D+xn7r0QgxnFoLZgtrzxiQErAjFUOdH6fas2KeQJ1yr/qaXQ==", - "license": "MIT", - "engines": { - "node": ">= 20" - } - }, "node_modules/@pnpm/config.env-replace": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", @@ -4510,12 +4324,6 @@ "@types/retry": "*" } }, - "node_modules/@types/aws-lambda": { - "version": "8.10.160", - "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.160.tgz", - "integrity": "sha512-uoO4QVQNWFPJMh26pXtmtrRfGshPUSpMZGUyUQY20FhfHEElEBOPKgVmFs1z+kbpyBsRs2JnoOPT7++Z4GA9pA==", - "license": "MIT" - }, "node_modules/@types/bytes": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/@types/bytes/-/bytes-3.1.5.tgz", @@ -6155,6 +5963,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-4.0.0.tgz", "integrity": "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==", + "dev": true, "license": "Apache-2.0" }, "node_modules/birpc": { @@ -6187,6 +5996,7 @@ "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", + "dev": true, "license": "MIT" }, "node_modules/brace-expansion": { @@ -8822,6 +8632,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-3.0.0.tgz", "integrity": "sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==", + "dev": true, "funding": [ { "type": "github", @@ -15073,28 +14884,6 @@ ], "license": "MIT" }, - "node_modules/octokit": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/octokit/-/octokit-5.0.5.tgz", - "integrity": "sha512-4+/OFSqOjoyULo7eN7EA97DE0Xydj/PW5aIckxqQIoFjFwqXKuFCvXUJObyJfBF9Khu4RL/jlDRI9FPaMGfPnw==", - "license": "MIT", - "dependencies": { - "@octokit/app": "^16.1.2", - "@octokit/core": "^7.0.6", - "@octokit/oauth-app": "^8.0.3", - "@octokit/plugin-paginate-graphql": "^6.0.0", - "@octokit/plugin-paginate-rest": "^14.0.0", - "@octokit/plugin-rest-endpoint-methods": "^17.0.0", - "@octokit/plugin-retry": "^8.0.3", - "@octokit/plugin-throttling": "^11.0.3", - "@octokit/request-error": "^7.0.2", - "@octokit/types": "^16.0.0", - "@octokit/webhooks": "^14.0.0" - }, - "engines": { - "node": ">= 20" - } - }, "node_modules/ohash": { "version": "2.0.11", "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", @@ -17232,9 +17021,9 @@ } }, "node_modules/simple-git": { - "version": "3.31.1", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.31.1.tgz", - "integrity": "sha512-oiWP4Q9+kO8q9hHqkX35uuHmxiEbZNTrZ5IPxgMGrJwN76pzjm/jabkZO0ItEcqxAincqGAzL3QHSaHt4+knBg==", + "version": "3.32.2", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.32.2.tgz", + "integrity": "sha512-n/jhNmvYh8dwyfR6idSfpXrFazuyd57jwNMzgjGnKZV/1lTh0HKvPq20v4AQ62rP+l19bWjjXPTCdGHMt0AdrQ==", "license": "MIT", "dependencies": { "@kwsites/file-exists": "^1.1.1", @@ -18144,15 +17933,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/toad-cache": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.0.tgz", - "integrity": "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==", - "license": "MIT", - "engines": { - "node": ">=12" - } - }, "node_modules/tokenx": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/tokenx/-/tokenx-1.3.0.tgz", @@ -18721,16 +18501,11 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/universal-github-app-jwt": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-2.2.2.tgz", - "integrity": "sha512-dcmbeSrOdTnsjGjUfAlqNDJrhxXizjAz94ija9Qw8YkZ1uu0d+GoZzyH+Jb9tIIqvGsadUfwg+22k5aDqqwzbw==", - "license": "MIT" - }, "node_modules/universal-user-agent": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.3.tgz", "integrity": "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==", + "dev": true, "license": "ISC" }, "node_modules/universalify": { diff --git a/package.json b/package.json index 6418d3e8..8bc394b4 100644 --- a/package.json +++ b/package.json @@ -203,12 +203,11 @@ "log-symbols": "^7.0.1", "nanoid": "^5.1.6", "node-addon-api": "^8.5.0", - "octokit": "^5.0.5", "ora": "^9.3.0", "pretty-ms": "^9.3.0", "proper-lockfile": "^4.1.2", "semver": "^7.7.1", - "simple-git": "^3.31.1", + "simple-git": "^3.32.2", "slice-ansi": "^8.0.0", "stdout-update": "^4.0.1", "strip-ansi": "^7.1.2", diff --git a/packages/@node-llama-cpp/linux-arm64/tsconfig.json b/packages/@node-llama-cpp/linux-arm64/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/linux-arm64/tsconfig.json +++ b/packages/@node-llama-cpp/linux-arm64/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/linux-armv7l/tsconfig.json b/packages/@node-llama-cpp/linux-armv7l/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/linux-armv7l/tsconfig.json +++ b/packages/@node-llama-cpp/linux-armv7l/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json b/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json b/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64-cuda/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json b/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64-vulkan/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/linux-x64/tsconfig.json b/packages/@node-llama-cpp/linux-x64/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/linux-x64/tsconfig.json +++ b/packages/@node-llama-cpp/linux-x64/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json b/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json +++ b/packages/@node-llama-cpp/mac-arm64-metal/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/mac-x64/tsconfig.json b/packages/@node-llama-cpp/mac-x64/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/mac-x64/tsconfig.json +++ b/packages/@node-llama-cpp/mac-x64/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/win-arm64/tsconfig.json b/packages/@node-llama-cpp/win-arm64/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/win-arm64/tsconfig.json +++ b/packages/@node-llama-cpp/win-arm64/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json b/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64-cuda-ext/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json b/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64-cuda/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json b/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64-vulkan/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/@node-llama-cpp/win-x64/tsconfig.json b/packages/@node-llama-cpp/win-x64/tsconfig.json index f6f82db3..527d791c 100644 --- a/packages/@node-llama-cpp/win-x64/tsconfig.json +++ b/packages/@node-llama-cpp/win-x64/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/packages/create-node-llama-cpp/tsconfig.json b/packages/create-node-llama-cpp/tsconfig.json index 2dd4bffc..733d0ff7 100644 --- a/packages/create-node-llama-cpp/tsconfig.json +++ b/packages/create-node-llama-cpp/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -13,7 +13,7 @@ "forceConsistentCasingInFileNames": true, "noFallthroughCasesInSwitch": true, "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/src/bindings/Llama.ts b/src/bindings/Llama.ts index f1700303..4d3615f6 100644 --- a/src/bindings/Llama.ts +++ b/src/bindings/Llama.ts @@ -16,6 +16,7 @@ import { LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, LlamaNuma } from "./types.js"; import {MemoryOrchestrator, MemoryReservation} from "./utils/MemoryOrchestrator.js"; +import {registerDisposeBeforeExit, unregisterDisposeBeforeExit} from "./utils/disposeBeforeExit.js"; export const LlamaLogLevelToAddonLogLevel: ReadonlyMap = new Map([ [LlamaLogLevel.disabled, 0], @@ -35,6 +36,7 @@ const defaultCPUMinThreadSplitterThreads = 4; export class Llama { /** @internal */ public readonly _bindings: BindingModule; /** @internal */ public readonly _backendDisposeGuard = new DisposeGuard(); + /** @internal */ private readonly _selfWeakRef: WeakRef; /** @internal */ public readonly _memoryLock = {}; /** @internal */ public readonly _consts: ReturnType; /** @internal */ public readonly _vramOrchestrator: MemoryOrchestrator; @@ -106,6 +108,7 @@ export class Llama { this._logLevel = this._debug ? LlamaLogLevel.debug : (logLevel ?? LlamaLogLevel.debug); + this._selfWeakRef = new WeakRef(this); const previouslyLoaded = bindings.markLoaded(); @@ -159,8 +162,9 @@ export class Llama { release: llamaCppRelease.release }); - this._onExit = this._onExit.bind(this); - process.on("exit", this._onExit); + this._onBeforeExit = this._onBeforeExit.bind(this); + process.once("beforeExit", this._onBeforeExit); + registerDisposeBeforeExit(this._selfWeakRef); } public async dispose() { @@ -171,6 +175,9 @@ export class Llama { this.onDispose.dispatchEvent(); await this._backendDisposeGuard.acquireDisposeLock(); await this._bindings.dispose(); + + process.off("beforeExit", this._onBeforeExit); + unregisterDisposeBeforeExit(this._selfWeakRef); } /** @hidden */ @@ -484,7 +491,7 @@ export class Llama { } /** @internal */ - private _onExit() { + private _onBeforeExit() { if (this._pendingLog != null && this._pendingLogLevel != null) { this._callLogger(this._pendingLogLevel, this._pendingLog); this._pendingLog = null; diff --git a/src/bindings/getLlama.ts b/src/bindings/getLlama.ts index 856de48c..e1b49753 100644 --- a/src/bindings/getLlama.ts +++ b/src/bindings/getLlama.ts @@ -66,17 +66,34 @@ export type LlamaOptions = { /** * Set what build method to use. - * - **`"auto"`**: If a local build is found, use it. - * Otherwise, if a prebuilt binary is found, use it. - * Otherwise, build from source. - * - **`"never"`**: If a local build is found, use it. - * Otherwise, if a prebuilt binary is found, use it. - * Otherwise, throw a `NoBinaryFoundError` error. + * - **`"auto"`**: + * Iterate over all available {@link gpu} types from best to worst, and for each GPU type: + * - If a local build is found, use it. + * - Otherwise, if a prebuilt binary is found and working, use it. + * + * Otherwise, if no binary is available for any GPU type, build from source. + * - **`"never"`**: + * Iterate over all available {@link gpu} types from best to worst, and for each GPU type: + * - If a local build is found, use it. + * - Otherwise, if a prebuilt binary is found and working, use it. + * + * Otherwise, if no binary is available for any GPU type, throw a `NoBinaryFoundError` error. * - **`"forceRebuild"`**: Always build from source. * Be cautious with this option, as it will cause the build to fail on Windows when the binaries are in use by another process. - * - **`"try"`**: If a local build is found, use it. - * Otherwise, try to build from source and use the resulting binary. - * If building from source fails, use a prebuilt binary if found. + * - **`"try"`**: + * Iterate over all available {@link gpu} types from best to worst, and for each GPU type: + * - If a local build is found, use it. + * + * If no local build is found, iterate over all available {@link gpu} types from best to worst, and for each GPU type: + * - Try to build from source and use the resulting binary. + * + * If no build was successful, iterate over all available {@link gpu} types from best to worst, and for each GPU type: + * - Use a prebuilt binary if found and working. + * - **`"autoAttempt"`**: + * Iterate over all available {@link gpu} types from best to worst, and for each GPU type: + * - If a local build is found, use it. + * - Otherwise, if a prebuilt binary is found and working, use it. + * - Otherwise, build from source and use the resulting binary. * * When running from inside an Asar archive in Electron, building from source is not possible, so it'll never build from source. * To allow building from source in Electron apps, make sure you ship `node-llama-cpp` as an unpacked module. @@ -84,7 +101,7 @@ export type LlamaOptions = { * Defaults to `"auto"`. * On Electron, defaults to `"never"`. */ - build?: "auto" | "never" | "forceRebuild" | "try", + build?: "auto" | "never" | "forceRebuild" | "try" | "autoAttempt", /** * Set custom CMake options for llama.cpp @@ -522,7 +539,7 @@ export async function getLlamaForOptions({ build = "auto"; } - if (build === "auto" || build === "never") { + if (build === "auto" || build === "autoAttempt" || build === "never") { for (let i = 0; i < buildGpusToTry.length; i++) { const gpu = buildGpusToTry[i]; const isLastItem = i === buildGpusToTry.length - 1; @@ -571,13 +588,63 @@ export async function getLlamaForOptions({ return llama; } + + if (canBuild && build === "autoAttempt") { + const llamaCppRepoCloned = await isLlamaCppRepoCloned(); + if (!llamaCppRepoCloned && !skipDownload) { + llamaCppInfo.repo = defaultLlamaCppGitHubRepo; + llamaCppInfo.release = defaultLlamaCppRelease; + + if (isGithubReleaseNeedsResolving(llamaCppInfo.release)) { + const [owner, name] = defaultLlamaCppGitHubRepo.split("/"); + llamaCppInfo.release = await resolveGithubRelease(owner!, name!, llamaCppInfo.release); + } + } + + let llama: Llama | undefined = undefined; + try { + llama = await buildAndLoadLlamaBinary({ + buildOptions, + skipDownload, + logLevel, + logger, + updateLastBuildInfoOnCompile, + maxThreads, + vramPadding, + ramPadding, + skipLlamaInit, + debug, + numa + }); + } catch (err) { + console.error( + getConsoleLogPrefix() + + `Failed to build llama.cpp with ${getPrettyBuildGpuName(gpu)} support. ` + + ( + !isLastItem + ? `falling back to ${getPrettyBuildGpuName(buildGpusToTry[i + 1])}. ` + : "" + ) + + "Error:", + err + ); + continue; + } + + if (llama != null) { + if (dryRun) + await llama.dispose(); + + return llama; + } + } } } if (shouldLogNoGlibcWarningIfNoBuildIsAvailable && progressLogs) await logNoGlibcWarning(); - if (!canBuild) + if (!canBuild || build === "autoAttempt") throw new NoBinaryFoundError(); const llamaCppRepoCloned = await isLlamaCppRepoCloned(); diff --git a/src/bindings/utils/cloneLlamaCppRepo.ts b/src/bindings/utils/cloneLlamaCppRepo.ts index 6f8a0d61..d6f3f370 100644 --- a/src/bindings/utils/cloneLlamaCppRepo.ts +++ b/src/bindings/utils/cloneLlamaCppRepo.ts @@ -1,5 +1,5 @@ import path from "path"; -import simpleGit, {SimpleGit} from "simple-git"; +import {simpleGit, SimpleGit} from "simple-git"; import chalk from "chalk"; import fs from "fs-extra"; import which from "which"; diff --git a/src/bindings/utils/compileLLamaCpp.ts b/src/bindings/utils/compileLLamaCpp.ts index d66ea4a1..3daaa721 100644 --- a/src/bindings/utils/compileLLamaCpp.ts +++ b/src/bindings/utils/compileLLamaCpp.ts @@ -155,7 +155,7 @@ export async function compileLlamaCpp(buildOptions: BuildOptions, compileOptions if (ciMode) { if (!cmakeCustomOptions.has("CMAKE_OSX_DEPLOYMENT_TARGET")) - cmakeCustomOptions.set("CMAKE_OSX_DEPLOYMENT_TARGET", "14"); + cmakeCustomOptions.set("CMAKE_OSX_DEPLOYMENT_TARGET", "14.0"); if (!cmakeCustomOptions.has("GGML_OPENMP")) cmakeCustomOptions.set("GGML_OPENMP", "OFF"); diff --git a/src/bindings/utils/detectGlibc.ts b/src/bindings/utils/detectGlibc.ts index 84365694..416ce8e0 100644 --- a/src/bindings/utils/detectGlibc.ts +++ b/src/bindings/utils/detectGlibc.ts @@ -1,4 +1,5 @@ import process from "process"; +import fs from "fs-extra"; import {BinaryPlatform} from "./getPlatform.js"; import {asyncEvery} from "./asyncEvery.js"; import {asyncSome} from "./asyncSome.js"; @@ -21,33 +22,65 @@ export async function detectGlibc({ "/usr/lib/armv7l-linux-gnu" ]; - return await asyncEvery([ - asyncSome([ - hasFileInPath("libc.so", librarySearchPaths), - hasFileInPath("libc.so.5", librarySearchPaths), - hasFileInPath("libc.so.6", librarySearchPaths), - hasFileInPath("libc.so.7", librarySearchPaths) // for when the next version comes out - ]), - asyncSome([ - hasFileInPath("ld-linux.so", librarySearchPaths), - hasFileInPath("ld-linux.so.1", librarySearchPaths), - hasFileInPath("ld-linux.so.2", librarySearchPaths), - hasFileInPath("ld-linux.so.3", librarySearchPaths), // for when the next version comes out - hasFileInPath("ld-linux-x86-64.so", librarySearchPaths), - hasFileInPath("ld-linux-x86-64.so.1", librarySearchPaths), - hasFileInPath("ld-linux-x86-64.so.2", librarySearchPaths), - hasFileInPath("ld-linux-x86-64.so.3", librarySearchPaths), // for when the next version comes out - hasFileInPath("ld-linux-aarch64.so", librarySearchPaths), - hasFileInPath("ld-linux-aarch64.so.1", librarySearchPaths), - hasFileInPath("ld-linux-aarch64.so.2", librarySearchPaths), - hasFileInPath("ld-linux-aarch64.so.3", librarySearchPaths), // for when the next version comes out - hasFileInPath("ld-linux-armv7l.so", librarySearchPaths), - hasFileInPath("ld-linux-armv7l.so.1", librarySearchPaths), - hasFileInPath("ld-linux-armv7l.so.2", librarySearchPaths), - hasFileInPath("ld-linux-armv7l.so.3", librarySearchPaths) // for when the next version comes out - ]) + const glibcFileNames = [ + "libc.so", + "libc.so.5", + "libc.so.6", + "libc.so.7" // for when the next version comes out + ]; + + const dynamicLoaderFileNames = [ + "ld-linux.so", + "ld-linux.so.1", + "ld-linux.so.2", + "ld-linux.so.3", // for when the next version comes out + "ld-linux-x86-64.so", + "ld-linux-x86-64.so.1", + "ld-linux-x86-64.so.2", + "ld-linux-x86-64.so.3", // for when the next version comes out + "ld-linux-aarch64.so", + "ld-linux-aarch64.so.1", + "ld-linux-aarch64.so.2", + "ld-linux-aarch64.so.3", // for when the next version comes out + "ld-linux-armv7l.so", + "ld-linux-armv7l.so.1", + "ld-linux-armv7l.so.2", + "ld-linux-armv7l.so.3" // for when the next version comes out + ]; + + const foundGlibC = await asyncEvery([ + asyncSome(glibcFileNames.map((fileName) => hasFileInPath(fileName, librarySearchPaths))), + asyncSome(dynamicLoaderFileNames.map((fileName) => hasFileInPath(fileName, librarySearchPaths))) ]); + if (foundGlibC) + return true; + + const loadedLibraries = await getLoadedSharedLibraries(); + return glibcFileNames.some((fileName) => loadedLibraries.has(fileName)) && + dynamicLoaderFileNames.some((fileName) => loadedLibraries.has(fileName)); } return false; } + +async function getLoadedSharedLibraries(trimPath = true) { + const libraries = new Set(); + + try { + const content = await fs.readFile("/proc/self/maps", "utf8"); + for (const line of content.split("\n")) { + const match = line.match(/\s(\/.*\.so(\.\d+)*)$/); + const lib = match?.[1]; + if (lib != null && lib !== "") { + if (trimPath) + libraries.add(lib.split("/").pop()!); + else + libraries.add(lib); + } + } + } catch (err) { + // do nothing + } + + return libraries; +} diff --git a/src/bindings/utils/disposeBeforeExit.ts b/src/bindings/utils/disposeBeforeExit.ts new file mode 100644 index 00000000..1e826807 --- /dev/null +++ b/src/bindings/utils/disposeBeforeExit.ts @@ -0,0 +1,50 @@ +let disposeRefs = new Set(); + +export function registerDisposeBeforeExit(ref: DisposableWeakRef) { + if (disposeRefs.size === 0) + process.once("beforeExit", onBeforeExit); + + disposeRefs.add(ref); +} + +export function unregisterDisposeBeforeExit(ref: DisposableWeakRef) { + const hadRefs = disposeRefs.size !== 0; + + disposeRefs.delete(ref); + + if (hadRefs && disposeRefs.size === 0) + process.off("beforeExit", onBeforeExit); +} + +function onBeforeExit() { + const refs = disposeRefs; + disposeRefs = new Set(); + + const promises: Promise[] = []; + + for (const ref of refs) { + const disposeTarget = ref.deref(); + if (disposeTarget == null) + continue; + + if (isAsyncDisposable(disposeTarget)) + promises.push(disposeTarget[Symbol.asyncDispose]()); + else if (isDisposable(disposeTarget)) + disposeTarget[Symbol.dispose](); + } + + if (promises.length > 0) + return Promise.all(promises).then(() => undefined); + + return undefined; +} + +type DisposableWeakRef = WeakRef<{[Symbol.dispose](): void} | {[Symbol.asyncDispose](): Promise}>; + +function isAsyncDisposable(target: any): target is {[Symbol.asyncDispose](): Promise} { + return Symbol.asyncDispose in target && target[Symbol.asyncDispose] instanceof Function; +} + +function isDisposable(target: any): target is {[Symbol.dispose](): void} { + return Symbol.dispose in target && target[Symbol.dispose] instanceof Function; +} diff --git a/src/cli/commands/ChatCommand.ts b/src/cli/commands/ChatCommand.ts index fe631d91..2e720d63 100644 --- a/src/cli/commands/ChatCommand.ts +++ b/src/cli/commands/ChatCommand.ts @@ -78,7 +78,7 @@ type ChatCommand = { meter: boolean, timing: boolean, noMmap: boolean, - noDirectIo: boolean, + useDirectIo: boolean, printTimings: boolean }; @@ -365,10 +365,10 @@ export const ChatCommand: CommandModule = { default: false, description: "Disable mmap (memory-mapped file) usage" }) - .option("noDirectIo", { + .option("useDirectIo", { type: "boolean", default: false, - description: "Disable Direct I/O usage when available" + description: "Use Direct I/O usage when available" }) .option("printTimings", { alias: "pt", @@ -384,7 +384,7 @@ export const ChatCommand: CommandModule = { topP, seed, xtc, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, reasoningBudget, noHistory, - environmentFunctions, tokenPredictionDraftModel, tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, noDirectIo, + environmentFunctions, tokenPredictionDraftModel, tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, useDirectIo, printTimings }) { try { @@ -395,7 +395,7 @@ export const ChatCommand: CommandModule = { gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, reasoningBudget, noHistory, environmentFunctions, tokenPredictionDraftModel, tokenPredictionModelContextSize, - debug, numa, meter, timing, noMmap, noDirectIo, printTimings + debug, numa, meter, timing, noMmap, useDirectIo, printTimings }); } catch (err) { await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing @@ -413,7 +413,7 @@ async function RunChat({ threads, temperature, minP, topK, topP, seed, xtc, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, reasoningBudget, noHistory, environmentFunctions, tokenPredictionDraftModel, - tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, noDirectIo, printTimings + tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, useDirectIo, printTimings }: ChatCommand) { if (contextSize === -1) contextSize = undefined; if (gpuLayers === -1) gpuLayers = undefined; @@ -440,7 +440,6 @@ async function RunChat({ }); const logBatchSize = batchSize != null; const useMmap = !noMmap && llama.supportsMmap; - const useDirectIo = !noDirectIo; const resolvedModelPath = await resolveCommandGgufPath(modelArg, llama, headers, { flashAttention, diff --git a/src/cli/commands/CompleteCommand.ts b/src/cli/commands/CompleteCommand.ts index e6a178ce..aeeb7117 100644 --- a/src/cli/commands/CompleteCommand.ts +++ b/src/cli/commands/CompleteCommand.ts @@ -60,7 +60,7 @@ type CompleteCommand = { meter: boolean, timing: boolean, noMmap: boolean, - noDirectIo: boolean, + useDirectIo: boolean, printTimings: boolean }; @@ -285,10 +285,10 @@ export const CompleteCommand: CommandModule = { default: false, description: "Disable mmap (memory-mapped file) usage" }) - .option("noDirectIo", { + .option("useDirectIo", { type: "boolean", default: false, - description: "Disable Direct I/O usage when available" + description: "Use Direct I/O usage when available" }) .option("printTimings", { alias: "pt", @@ -303,7 +303,7 @@ export const CompleteCommand: CommandModule = { topP, seed, xtc, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, tokenPredictionDraftModel, tokenPredictionModelContextSize, - debug, numa, meter, timing, noMmap, noDirectIo, printTimings + debug, numa, meter, timing, noMmap, useDirectIo, printTimings }) { try { await RunCompletion({ @@ -311,7 +311,7 @@ export const CompleteCommand: CommandModule = { threads, temperature, minP, topK, topP, seed, xtc, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, - tokenPredictionDraftModel, tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, noDirectIo, printTimings + tokenPredictionDraftModel, tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, useDirectIo, printTimings }); } catch (err) { await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing @@ -327,7 +327,7 @@ async function RunCompletion({ threads, temperature, minP, topK, topP, seed, xtc, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, - tokenPredictionDraftModel, tokenPredictionModelContextSize, maxTokens, debug, numa, meter, timing, noMmap, noDirectIo, printTimings + tokenPredictionDraftModel, tokenPredictionModelContextSize, maxTokens, debug, numa, meter, timing, noMmap, useDirectIo, printTimings }: CompleteCommand) { if (contextSize === -1) contextSize = undefined; if (gpuLayers === -1) gpuLayers = undefined; @@ -352,7 +352,6 @@ async function RunCompletion({ }); const logBatchSize = batchSize != null; const useMmap = !noMmap && llama.supportsMmap; - const useDirectIo = !noDirectIo; const resolvedModelPath = await resolveCommandGgufPath(modelArg, llama, headers, { flashAttention, diff --git a/src/cli/commands/InfillCommand.ts b/src/cli/commands/InfillCommand.ts index 4d2a6b86..5a69a3a7 100644 --- a/src/cli/commands/InfillCommand.ts +++ b/src/cli/commands/InfillCommand.ts @@ -62,7 +62,7 @@ type InfillCommand = { meter: boolean, timing: boolean, noMmap: boolean, - noDirectIo: boolean, + useDirectIo: boolean, printTimings: boolean }; @@ -295,10 +295,10 @@ export const InfillCommand: CommandModule = { default: false, description: "Disable mmap (memory-mapped file) usage" }) - .option("noDirectIo", { + .option("useDirectIo", { type: "boolean", default: false, - description: "Disable Direct I/O usage when available" + description: "Use Direct I/O usage when available" }) .option("printTimings", { alias: "pt", @@ -313,7 +313,7 @@ export const InfillCommand: CommandModule = { topP, seed, xtc, gpuLayers, repeatPenalty, lastTokensRepeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, tokenPredictionDraftModel, tokenPredictionModelContextSize, - debug, numa, meter, timing, noMmap, noDirectIo, printTimings + debug, numa, meter, timing, noMmap, useDirectIo, printTimings }) { try { await RunInfill({ @@ -321,7 +321,7 @@ export const InfillCommand: CommandModule = { swaFullCache, threads, temperature, minP, topK, topP, seed, xtc, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, maxTokens, - tokenPredictionDraftModel, tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, noDirectIo, printTimings + tokenPredictionDraftModel, tokenPredictionModelContextSize, debug, numa, meter, timing, noMmap, useDirectIo, printTimings }); } catch (err) { await new Promise((accept) => setTimeout(accept, 0)); // wait for logs to finish printing @@ -337,7 +337,7 @@ async function RunInfill({ swaFullCache, threads, temperature, minP, topK, topP, seed, xtc, gpuLayers, lastTokensRepeatPenalty, repeatPenalty, penalizeRepeatingNewLine, repeatFrequencyPenalty, repeatPresencePenalty, dryRepeatPenaltyStrength, dryRepeatPenaltyBase, dryRepeatPenaltyAllowedLength, dryRepeatPenaltyLastTokens, - tokenPredictionDraftModel, tokenPredictionModelContextSize, maxTokens, debug, numa, meter, timing, noMmap, noDirectIo, printTimings + tokenPredictionDraftModel, tokenPredictionModelContextSize, maxTokens, debug, numa, meter, timing, noMmap, useDirectIo, printTimings }: InfillCommand) { if (contextSize === -1) contextSize = undefined; if (gpuLayers === -1) gpuLayers = undefined; @@ -362,7 +362,6 @@ async function RunInfill({ }); const logBatchSize = batchSize != null; const useMmap = !noMmap && llama.supportsMmap; - const useDirectIo = !noDirectIo; const resolvedModelPath = await resolveCommandGgufPath(modelArg, llama, headers, { flashAttention, diff --git a/src/utils/gitReleaseBundles.ts b/src/utils/gitReleaseBundles.ts index 9def90b3..e2159932 100644 --- a/src/utils/gitReleaseBundles.ts +++ b/src/utils/gitReleaseBundles.ts @@ -1,6 +1,6 @@ import path from "path"; import fs from "fs-extra"; -import simpleGit from "simple-git"; +import {simpleGit} from "simple-git"; import {currentReleaseGitBundlePath, builtinLlamaCppGitHubRepo, llamaCppDirectory, enableRecursiveClone} from "../config.js"; import {getBinariesGithubRelease} from "../bindings/utils/binariesGithubRelease.js"; import {isGithubReleaseNeedsResolving} from "./resolveGithubRelease.js"; diff --git a/src/utils/resolveGithubRelease.ts b/src/utils/resolveGithubRelease.ts index a9cb449e..ac280b94 100644 --- a/src/utils/resolveGithubRelease.ts +++ b/src/utils/resolveGithubRelease.ts @@ -1,43 +1,205 @@ -import {Octokit} from "octokit"; import {getConsoleLogPrefix} from "./getConsoleLogPrefix.js"; export async function resolveGithubRelease(githubOwner: string, githubRepo: string, release: string) { - const octokit = new Octokit(); + const githubClient = new GitHubClient(); const repo = githubOwner + "/" + githubRepo; - type GithubReleaseType = Awaited> | - Awaited>; + type GithubReleaseType = GitHubRelease; let githubRelease: GithubReleaseType | null = null; try { - if (release === "latest") { - githubRelease = await octokit.rest.repos.getLatestRelease({ + if (release === "latest") + githubRelease = await githubClient.getLatestRelease({ owner: githubOwner, repo: githubRepo }); - } else { - githubRelease = await octokit.rest.repos.getReleaseByTag({ + else + githubRelease = await githubClient.getReleaseByTag({ owner: githubOwner, repo: githubRepo, tag: release }); - } } catch (err) { console.error(getConsoleLogPrefix() + "Failed to fetch llama.cpp release info", err); } - if (githubRelease == null) { + if (githubRelease == null) throw new Error(`Failed to find release "${release}" of "${repo}"`); - } - if (githubRelease.data.tag_name == null) { + if (githubRelease.tag_name == null) throw new Error(`Failed to find tag of release "${release}" of "${repo}"`); - } - return githubRelease.data.tag_name; + return githubRelease.tag_name; } export function isGithubReleaseNeedsResolving(release: string) { return release === "latest"; } + +const defaultGitHubApiBase = "https://api.github.com"; +const defaultGitHubApiVersion: GitHubApiVersion = "2022-11-28"; + +type GitHubApiVersion = "2022-11-28" | (string & {}); + +type GitHubClientOptions = { + token?: string, + + /** + * GitHub REST API base URL. + * + * Defaults to `https://api.github.com`. + */ + apiBase?: string, + + /** + * GitHub REST API version header. + * + * Defaults to `"2022-11-28"`. + */ + apiVersion?: GitHubApiVersion, + + userAgent?: string +}; + +type GitHubRelease = { + url: string, + "html_url": string, + "assets_url": string, + "upload_url": string, + + id: number, + "node_id": string, + + "tag_name": string, + "target_commitish": string, + name: string | null, + body: string | null, + + draft: boolean, + prerelease: boolean, + + "created_at": string, // ISO date-time + "published_at": string | null, // ISO date-time + + author: GitHubUser | null, + + assets: GitHubReleaseAsset[], + + "tarball_url": string | null, + "zipball_url": string | null +}; + +type GitHubUser = { + login: string, + id: number, + "node_id": string, + "avatar_url": string, + "html_url": string, + type: string, + "site_admin": boolean +}; + +type GitHubReleaseAsset = { + url: string, + id: number, + "node_id": string, + + name: string, + label: string | null, + "content_type": string, + state: string, + size: number, + "download_count": number, + + "browser_download_url": string, + + "created_at": string, // ISO date-time + "updated_at": string, // ISO date-time + + uploader: GitHubUser | null +}; + +type GitHubApiError = Error & { + status: number, + url: string, + bodyText?: string, + headers?: Record +}; + +class GitHubClient { + private readonly _clientOptions: GitHubClientOptions; + + public constructor(clientOptions: GitHubClientOptions = {}) { + this._clientOptions = clientOptions; + } + + public async getLatestRelease({ + owner, repo + }: { + owner: string, + repo: string + }): Promise { + return this._fetchJson( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/releases/latest` + ); + } + + public async getReleaseByTag({ + owner, repo, tag + }: { + owner: string, + repo: string, + tag: string + }): Promise { + return this._fetchJson( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/releases/tags/${encodeURIComponent(tag)}` + ); + } + + private async _fetchJson( + path: string + ): Promise { + const url = this._getApiBase() + path; + + const headers: Record = { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": this._clientOptions.apiVersion ?? defaultGitHubApiVersion + }; + + if (this._clientOptions.token != null && this._clientOptions.token !== "") + headers.Authorization = "Bearer " + this._clientOptions.token; + + if (this._clientOptions.userAgent != null && this._clientOptions.userAgent !== "") + headers["User-Agent"] = this._clientOptions.userAgent; + + + const res = await fetch(url, { + method: "GET", + headers + }); + + if (!res.ok) { + const err = new Error( + `GitHub API error ${res.status} ${res.statusText}` + ) as GitHubApiError; + + err.status = res.status; + err.url = url; + err.headers = Object.fromEntries(res.headers.entries()); + try { + err.bodyText = await res.text(); + } catch { + err.bodyText = undefined; + } + + throw err; + } + + return (await res.json()) as T; + } + + private _getApiBase() { + return this._clientOptions?.apiBase ?? defaultGitHubApiBase; + } +} diff --git a/templates/electron-typescript-react/tsconfig.json b/templates/electron-typescript-react/tsconfig.json index 272aef7e..5ac1a523 100644 --- a/templates/electron-typescript-react/tsconfig.json +++ b/templates/electron-typescript-react/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es2022", "lib": ["es2022", "DOM", "DOM.Iterable"], - "module": "es2022", + "module": "node16", "skipLibCheck": true, "esModuleInterop": true, diff --git a/templates/electron-typescript-react/tsconfig.node.json b/templates/electron-typescript-react/tsconfig.node.json index 9ac8fa5e..83634c20 100644 --- a/templates/electron-typescript-react/tsconfig.node.json +++ b/templates/electron-typescript-react/tsconfig.node.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es2022", "lib": ["es2022"], - "module": "es2022", + "module": "node16", "skipLibCheck": true, "esModuleInterop": true, diff --git a/templates/node-typescript/tsconfig.json b/templates/node-typescript/tsconfig.json index 5ceb4f63..eddee9ab 100644 --- a/templates/node-typescript/tsconfig.json +++ b/templates/node-typescript/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -15,7 +15,7 @@ "noUncheckedIndexedAccess": true, "moduleDetection": "force", "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true, diff --git a/tsconfig.json b/tsconfig.json index 3a49af10..74531cda 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": ["es2022"], - "module": "es2022", + "module": "node16", "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, @@ -15,7 +15,7 @@ "noUncheckedIndexedAccess": true, "moduleDetection": "force", "skipLibCheck": true, - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": false, "strictNullChecks": true, "isolatedModules": true,