add custom tiny tfjs module for node.js v18+

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tiny-tensorflow/bert-tiny-multilingual/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+tiny-tensorflow/toxicity-tfjs/tfnapi-v8/tensorflow.dll filter=lfs diff=lfs merge=lfs -text

tiny-tensorflow/bert-tiny-multilingual/saved_model.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84cb4153c11646eb331f7707570b8b3abe1f57ae5919c0bec80990ac6c4e8961
+size 14166066

tiny-tensorflow/bert-tiny-multilingual/variables/variables.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcddc18a03c9cf78e873c24440335a972bdb6a266b7fe8e5edceb87fb95809e4
+size 755860212

tiny-tensorflow/predict.js

@@ -0,0 +1,12 @@
+async function run() {
+    const { vitchen_toxicity_encode } = require('./bert-tiny-multilingual/bert-tokenizer');
+    const {loadSavedModel, tensor} = require('./bert-tiny-multilingual/node');
+    const model = await loadSavedModel('./toxicity9');
+    const input = vitchen_toxicity_encode(`What's up?`);
+    let t = tensor(input, [192], 'int32');
+    const prediction = model.predict({
+        input_ids: t
+    })['output_0'];
+    console.log(prediction.dataSync()[0])
+}
+run()

tiny-tensorflow/toxicity-tfjs/bert-tokenizer.js

@@ -0,0 +1,69 @@
+const symbols = new Set(['!', '`', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '~' , '_', '+', '=', '[', ']', '{', '}', '\\', '|', ':', ';', '\'', '"', '<', '>', ',', '.', '?', '/']);
+const toxicityTokens = require('./vitchen-multilingual-uncased.json');
+
+/**
+ * @param {string} text The content of a Discord message
+ */
+function vitchen_toxicity_encode(text) {
+    let unclean_words = text.replace('`', '').replace('\n', ' ').toLowerCase().split(' '), ids = [101]; // [CLS]
+    for (let i = 0; i < unclean_words.length && ids.length < 192; i++) {
+        let cur = unclean_words[i];
+        if (cur === '') continue;
+        let curToken = getToken(cur);
+        if (curToken) ids.push(curToken);
+        else {
+            let splitBySymbols = [], preSymbol = ''
+            for (let j = 0; j < cur.length; j++) {
+                if (symbols.has(cur[j])) {
+                    if (preSymbol !== '') splitBySymbols.push(preSymbol);
+                    splitBySymbols.push(cur[j]);
+                    preSymbol = '';
+                } else {
+                    preSymbol += cur[j]
+                }
+            }
+            splitBySymbols.push(preSymbol);
+            for (let word of splitBySymbols) {
+                let k = word.length, hangman = '', checkpoint = 0;
+                while (k !== checkpoint && hangman !== word) {
+                    for (k; k >= checkpoint + 1; k--) {
+                        let checkpointJourney = word.substring(checkpoint, k), token = getToken(hangman.length === 0 ? checkpointJourney : '##' + checkpointJourney);
+                        if (token) {
+                            ids.push(token);
+                            hangman += checkpointJourney;
+                            checkpoint = k;
+                            k = word.length;
+                            break;
+                        }
+                    }
+                }
+                if (k === checkpoint && hangman !== word) ids.push(100) // [UNK]
+                let lastcheckpointToken = getToken(word.substring(checkpoint));
+                if (lastcheckpointToken) ids.push(lastcheckpointToken);
+            }
+        }
+    }
+    if (ids.length < 192) {
+        ids.push(102);
+        while (ids.length < 192) {
+            ids.push(0);
+        }
+    } else if (ids.length === 192) {
+        ids[191] = 102 // [SEP]
+    } else {
+        ids = ids.slice(0, 192);
+        ids[191] = 102 // [SEP]
+    }
+    return ids;
+}
+
+/**
+ * @param {string} string 
+ * @returns {number|undefined}
+ */
+function getToken(string) {
+    if (string === '') return undefined;
+    return toxicityTokens[string];
+}
+
+module.exports.vitchen_toxicity_encode = vitchen_toxicity_encode;

tiny-tensorflow/toxicity-tfjs/config.json

@@ -0,0 +1,70 @@
+{
+    "name": "@tensorflow/tfjs-node",
+    "version": "3.19.0",
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
+    "gypfile": true,
+    "license": "Apache-2.0",
+    "repository": {
+      "type": "git",
+      "url": "https://github.com/tensorflow/tfjs.git",
+      "directory": "tfjs-node"
+    },
+    "engines": {
+      "node": ">=8.11.0"
+    },
+    "scripts": {
+      "build": "tsc && npx mkdirp dist/proto && cp src/proto/api_pb.js dist/proto/api_pb.js",
+      "build-ci": "tsc && npx mkdirp dist/proto && cp src/proto/api_pb.js dist/proto/api_pb.js",
+      "build-link-package": "cd ../link-package && yarn build-deps-for tfjs-node tfjs",
+      "build-union": "cd ../tfjs && yarn && yarn build",
+      "build-union-ci": "cd ../tfjs && yarn && yarn build-ci",
+      "build-deps": "yarn build-link-package && yarn build-union",
+      "build-deps-ci": "yarn build-link-package && yarn build-union-ci",
+      "build-npm": "./scripts/build-npm.sh",
+      "build-addon": "./scripts/build-and-upload-addon.sh",
+      "build-addon-from-source": "node-pre-gyp install --build-from-source",
+      "clean-deps": "rm -rf deps && rm -rf lib",
+      "coverage": "nyc yarn ts-node -P tsconfig.test.json src/run_tests.ts",
+      "enable-gpu": "node scripts/install.js gpu download && yarn && yarn build-addon-from-source",
+      "ensure-cpu-gpu-packages-align": "node scripts/ensure-cpu-gpu-packages-align.js",
+      "format": "clang-format -i -style=Google binding/*.cc binding/*.h",
+      "install": "node scripts/install.js",
+      "install-from-source": "yarn clean-deps && yarn && yarn build-addon-from-source",
+      "link-local": "yalc link",
+      "lint": "tslint -p . -t verbose",
+      "prep": "cd node_modules/@tensorflow/tfjs-core && yarn && yarn build",
+      "publish-local": "yarn prep && yalc push",
+      "publish-npm": "npm publish",
+      "test": "yarn && yarn build-deps && yarn build && ts-node --transpile-only --skip-ignore -P tsconfig.test.json src/run_tests.ts",
+      "test-dev": "tsc && ts-node --transpile-only --skip-ignore -P tsconfig.test.json src/run_tests.ts",
+      "test-ci": "ts-node --transpile-only --skip-ignore -P tsconfig.test.json src/run_tests.ts",
+      "upload-windows-addon": "./scripts/build-and-upload-windows-addon.bat"
+    },
+    "dependencies": {
+      "@mapbox/node-pre-gyp": "1.0.9",
+      "adm-zip": "^0.5.2",
+      "google-protobuf": "^3.9.2",
+      "https-proxy-agent": "^2.2.1",
+      "progress": "^2.0.0",
+      "rimraf": "^2.6.2",
+      "path": "^0.12.7",
+      "tar": "^4.4.6",
+      "util": "^0.12.4"
+    },
+    "binary": {
+      "module_name": "tfjs_binding",
+      "module_path": "./tfnapi-v{napi_build_version}",
+      "host": "https://storage.googleapis.com/tf-builds/pre-built-binary",
+      "remote_path": "./tfnapi-v{napi_build_version}/{version}/",
+      "napi_versions": [
+        3,
+        4,
+        5,
+        6,
+        7,
+        8
+      ]
+    }
+  }
+  

tiny-tensorflow/toxicity-tfjs/tfnapi-v8/tensorflow.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba650c0d99b0275086789d6ec8de2c200cca8e29d800ef7cefd7f318aeb1e9e5
+size 200814080