Liam Dyer
commited on
Commit
•
5a31018
1
Parent(s):
3f5871c
Anthropic Endpoint Support (#923)
Browse files- .env +1 -0
- README.md +45 -0
- package-lock.json +30 -24
- package.json +1 -0
- src/lib/server/endpoints/anthropic/endpointAnthropic.ts +95 -0
- src/lib/server/endpoints/endpoints.ts +6 -0
- src/lib/server/models.ts +2 -0
.env
CHANGED
@@ -9,6 +9,7 @@ COOKIE_NAME=hf-chat
|
|
9 |
HF_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
|
10 |
HF_API_ROOT=https://api-inference.huggingface.co/models
|
11 |
OPENAI_API_KEY=#your openai api key here
|
|
|
12 |
|
13 |
HF_ACCESS_TOKEN=#LEGACY! Use HF_TOKEN instead
|
14 |
|
|
|
9 |
HF_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
|
10 |
HF_API_ROOT=https://api-inference.huggingface.co/models
|
11 |
OPENAI_API_KEY=#your openai api key here
|
12 |
+
ANTHROPIC_API_KEY=#your anthropic api key here
|
13 |
|
14 |
HF_ACCESS_TOKEN=#LEGACY! Use HF_TOKEN instead
|
15 |
|
README.md
CHANGED
@@ -459,6 +459,51 @@ MODELS=`[
|
|
459 |
]`
|
460 |
```
|
461 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
462 |
#### Amazon
|
463 |
|
464 |
You can also specify your Amazon SageMaker instance as an endpoint for chat-ui. The config goes like this:
|
|
|
459 |
]`
|
460 |
```
|
461 |
|
462 |
+
#### Anthropic
|
463 |
+
|
464 |
+
We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
|
465 |
+
|
466 |
+
```
|
467 |
+
MODELS=`[
|
468 |
+
{
|
469 |
+
"name": "claude-3-sonnet-20240229",
|
470 |
+
"displayName": "Claude 3 Sonnet",
|
471 |
+
"description": "Ideal balance of intelligence and speed",
|
472 |
+
"parameters": {
|
473 |
+
"max_new_tokens": 4096,
|
474 |
+
},
|
475 |
+
"endpoints": [
|
476 |
+
{
|
477 |
+
"type": "anthropic",
|
478 |
+
// optionals
|
479 |
+
"apiKey": "sk-ant-...",
|
480 |
+
"baseURL": "https://api.anthropic.com",
|
481 |
+
defaultHeaders: {},
|
482 |
+
defaultQuery: {}
|
483 |
+
}
|
484 |
+
]
|
485 |
+
},
|
486 |
+
{
|
487 |
+
"name": "claude-3-opus-20240229",
|
488 |
+
"displayName": "Claude 3 Opus",
|
489 |
+
"description": "Most powerful model for highly complex tasks",
|
490 |
+
"parameters": {
|
491 |
+
"max_new_tokens": 4096
|
492 |
+
},
|
493 |
+
"endpoints": [
|
494 |
+
{
|
495 |
+
"type": "anthropic",
|
496 |
+
// optionals
|
497 |
+
"apiKey": "sk-ant-...",
|
498 |
+
"baseURL": "https://api.anthropic.com",
|
499 |
+
defaultHeaders: {},
|
500 |
+
defaultQuery: {}
|
501 |
+
}
|
502 |
+
]
|
503 |
+
}
|
504 |
+
]`
|
505 |
+
```
|
506 |
+
|
507 |
#### Amazon
|
508 |
|
509 |
You can also specify your Amazon SageMaker instance as an endpoint for chat-ui. The config goes like this:
|
package-lock.json
CHANGED
@@ -8,6 +8,7 @@
|
|
8 |
"name": "chat-ui",
|
9 |
"version": "0.7.0",
|
10 |
"dependencies": {
|
|
|
11 |
"@huggingface/hub": "^0.5.1",
|
12 |
"@huggingface/inference": "^2.6.3",
|
13 |
"@iconify-json/bi": "^1.1.21",
|
@@ -122,6 +123,30 @@
|
|
122 |
"url": "https://github.com/sponsors/antfu"
|
123 |
}
|
124 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
125 |
"node_modules/@cspotcode/source-map-support": {
|
126 |
"version": "0.8.1",
|
127 |
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
|
@@ -2013,7 +2038,6 @@
|
|
2013 |
"version": "2.6.5",
|
2014 |
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz",
|
2015 |
"integrity": "sha512-OZsUlr2nxvkqUFLSaY2ZbA+P1q22q+KrlxWOn/38RX+u5kTkYL2mTujEpzUhGkS+K/QCYp9oagfXG39XOzyySg==",
|
2016 |
-
"optional": true,
|
2017 |
"dependencies": {
|
2018 |
"@types/node": "*",
|
2019 |
"form-data": "^4.0.0"
|
@@ -2409,7 +2433,6 @@
|
|
2409 |
"version": "3.0.0",
|
2410 |
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
2411 |
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
2412 |
-
"optional": true,
|
2413 |
"dependencies": {
|
2414 |
"event-target-shim": "^5.0.0"
|
2415 |
},
|
@@ -2462,7 +2485,6 @@
|
|
2462 |
"version": "4.5.0",
|
2463 |
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
|
2464 |
"integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
|
2465 |
-
"optional": true,
|
2466 |
"dependencies": {
|
2467 |
"humanize-ms": "^1.2.1"
|
2468 |
},
|
@@ -2630,8 +2652,7 @@
|
|
2630 |
"node_modules/base-64": {
|
2631 |
"version": "0.1.0",
|
2632 |
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
|
2633 |
-
"integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
|
2634 |
-
"optional": true
|
2635 |
},
|
2636 |
"node_modules/base64-js": {
|
2637 |
"version": "1.5.1",
|
@@ -2899,7 +2920,6 @@
|
|
2899 |
"version": "0.0.2",
|
2900 |
"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
|
2901 |
"integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
|
2902 |
-
"optional": true,
|
2903 |
"engines": {
|
2904 |
"node": "*"
|
2905 |
}
|
@@ -3096,7 +3116,6 @@
|
|
3096 |
"version": "0.0.2",
|
3097 |
"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
|
3098 |
"integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
|
3099 |
-
"optional": true,
|
3100 |
"engines": {
|
3101 |
"node": "*"
|
3102 |
}
|
@@ -3352,7 +3371,6 @@
|
|
3352 |
"version": "1.3.0",
|
3353 |
"resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
|
3354 |
"integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
|
3355 |
-
"optional": true,
|
3356 |
"dependencies": {
|
3357 |
"base-64": "^0.1.0",
|
3358 |
"md5": "^2.3.0"
|
@@ -3724,7 +3742,6 @@
|
|
3724 |
"version": "5.0.1",
|
3725 |
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
3726 |
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
3727 |
-
"optional": true,
|
3728 |
"engines": {
|
3729 |
"node": ">=6"
|
3730 |
}
|
@@ -3907,14 +3924,12 @@
|
|
3907 |
"node_modules/form-data-encoder": {
|
3908 |
"version": "1.7.2",
|
3909 |
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
|
3910 |
-
"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
|
3911 |
-
"optional": true
|
3912 |
},
|
3913 |
"node_modules/formdata-node": {
|
3914 |
"version": "4.4.1",
|
3915 |
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
|
3916 |
"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
|
3917 |
-
"optional": true,
|
3918 |
"dependencies": {
|
3919 |
"node-domexception": "1.0.0",
|
3920 |
"web-streams-polyfill": "4.0.0-beta.3"
|
@@ -4206,7 +4221,6 @@
|
|
4206 |
"version": "1.2.1",
|
4207 |
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
|
4208 |
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
|
4209 |
-
"optional": true,
|
4210 |
"dependencies": {
|
4211 |
"ms": "^2.0.0"
|
4212 |
}
|
@@ -4344,8 +4358,7 @@
|
|
4344 |
"node_modules/is-buffer": {
|
4345 |
"version": "1.1.6",
|
4346 |
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
|
4347 |
-
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
|
4348 |
-
"optional": true
|
4349 |
},
|
4350 |
"node_modules/is-builtin-module": {
|
4351 |
"version": "3.2.1",
|
@@ -4819,7 +4832,6 @@
|
|
4819 |
"version": "2.3.0",
|
4820 |
"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
|
4821 |
"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
|
4822 |
-
"optional": true,
|
4823 |
"dependencies": {
|
4824 |
"charenc": "0.0.2",
|
4825 |
"crypt": "0.0.2",
|
@@ -5116,7 +5128,6 @@
|
|
5116 |
"url": "https://paypal.me/jimmywarting"
|
5117 |
}
|
5118 |
],
|
5119 |
-
"optional": true,
|
5120 |
"engines": {
|
5121 |
"node": ">=10.5.0"
|
5122 |
}
|
@@ -5125,7 +5136,6 @@
|
|
5125 |
"version": "2.7.0",
|
5126 |
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
5127 |
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
5128 |
-
"optional": true,
|
5129 |
"dependencies": {
|
5130 |
"whatwg-url": "^5.0.0"
|
5131 |
},
|
@@ -5144,20 +5154,17 @@
|
|
5144 |
"node_modules/node-fetch/node_modules/tr46": {
|
5145 |
"version": "0.0.3",
|
5146 |
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
5147 |
-
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
5148 |
-
"optional": true
|
5149 |
},
|
5150 |
"node_modules/node-fetch/node_modules/webidl-conversions": {
|
5151 |
"version": "3.0.1",
|
5152 |
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
5153 |
-
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
5154 |
-
"optional": true
|
5155 |
},
|
5156 |
"node_modules/node-fetch/node_modules/whatwg-url": {
|
5157 |
"version": "5.0.0",
|
5158 |
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
5159 |
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
5160 |
-
"optional": true,
|
5161 |
"dependencies": {
|
5162 |
"tr46": "~0.0.3",
|
5163 |
"webidl-conversions": "^3.0.0"
|
@@ -8176,7 +8183,6 @@
|
|
8176 |
"version": "4.0.0-beta.3",
|
8177 |
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
|
8178 |
"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
|
8179 |
-
"optional": true,
|
8180 |
"engines": {
|
8181 |
"node": ">= 14"
|
8182 |
}
|
|
|
8 |
"name": "chat-ui",
|
9 |
"version": "0.7.0",
|
10 |
"dependencies": {
|
11 |
+
"@anthropic-ai/sdk": "^0.17.1",
|
12 |
"@huggingface/hub": "^0.5.1",
|
13 |
"@huggingface/inference": "^2.6.3",
|
14 |
"@iconify-json/bi": "^1.1.21",
|
|
|
123 |
"url": "https://github.com/sponsors/antfu"
|
124 |
}
|
125 |
},
|
126 |
+
"node_modules/@anthropic-ai/sdk": {
|
127 |
+
"version": "0.17.1",
|
128 |
+
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.17.1.tgz",
|
129 |
+
"integrity": "sha512-ke/JGfaa4sc1PB58L4B9hXI/BlJphXc696+cVX8Z8gQt51l++a9umZTN/7UymV8Dcat6KKYNQE8P8yeeyAldHg==",
|
130 |
+
"dependencies": {
|
131 |
+
"@types/node": "^18.11.18",
|
132 |
+
"@types/node-fetch": "^2.6.4",
|
133 |
+
"abort-controller": "^3.0.0",
|
134 |
+
"agentkeepalive": "^4.2.1",
|
135 |
+
"digest-fetch": "^1.3.0",
|
136 |
+
"form-data-encoder": "1.7.2",
|
137 |
+
"formdata-node": "^4.3.2",
|
138 |
+
"node-fetch": "^2.6.7",
|
139 |
+
"web-streams-polyfill": "^3.2.1"
|
140 |
+
}
|
141 |
+
},
|
142 |
+
"node_modules/@anthropic-ai/sdk/node_modules/web-streams-polyfill": {
|
143 |
+
"version": "3.3.3",
|
144 |
+
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
|
145 |
+
"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
|
146 |
+
"engines": {
|
147 |
+
"node": ">= 8"
|
148 |
+
}
|
149 |
+
},
|
150 |
"node_modules/@cspotcode/source-map-support": {
|
151 |
"version": "0.8.1",
|
152 |
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
|
|
|
2038 |
"version": "2.6.5",
|
2039 |
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz",
|
2040 |
"integrity": "sha512-OZsUlr2nxvkqUFLSaY2ZbA+P1q22q+KrlxWOn/38RX+u5kTkYL2mTujEpzUhGkS+K/QCYp9oagfXG39XOzyySg==",
|
|
|
2041 |
"dependencies": {
|
2042 |
"@types/node": "*",
|
2043 |
"form-data": "^4.0.0"
|
|
|
2433 |
"version": "3.0.0",
|
2434 |
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
2435 |
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
|
|
2436 |
"dependencies": {
|
2437 |
"event-target-shim": "^5.0.0"
|
2438 |
},
|
|
|
2485 |
"version": "4.5.0",
|
2486 |
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
|
2487 |
"integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
|
|
|
2488 |
"dependencies": {
|
2489 |
"humanize-ms": "^1.2.1"
|
2490 |
},
|
|
|
2652 |
"node_modules/base-64": {
|
2653 |
"version": "0.1.0",
|
2654 |
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
|
2655 |
+
"integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
|
|
|
2656 |
},
|
2657 |
"node_modules/base64-js": {
|
2658 |
"version": "1.5.1",
|
|
|
2920 |
"version": "0.0.2",
|
2921 |
"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
|
2922 |
"integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
|
|
|
2923 |
"engines": {
|
2924 |
"node": "*"
|
2925 |
}
|
|
|
3116 |
"version": "0.0.2",
|
3117 |
"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
|
3118 |
"integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
|
|
|
3119 |
"engines": {
|
3120 |
"node": "*"
|
3121 |
}
|
|
|
3371 |
"version": "1.3.0",
|
3372 |
"resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
|
3373 |
"integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
|
|
|
3374 |
"dependencies": {
|
3375 |
"base-64": "^0.1.0",
|
3376 |
"md5": "^2.3.0"
|
|
|
3742 |
"version": "5.0.1",
|
3743 |
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
3744 |
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
|
|
3745 |
"engines": {
|
3746 |
"node": ">=6"
|
3747 |
}
|
|
|
3924 |
"node_modules/form-data-encoder": {
|
3925 |
"version": "1.7.2",
|
3926 |
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
|
3927 |
+
"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
|
|
|
3928 |
},
|
3929 |
"node_modules/formdata-node": {
|
3930 |
"version": "4.4.1",
|
3931 |
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
|
3932 |
"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
|
|
|
3933 |
"dependencies": {
|
3934 |
"node-domexception": "1.0.0",
|
3935 |
"web-streams-polyfill": "4.0.0-beta.3"
|
|
|
4221 |
"version": "1.2.1",
|
4222 |
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
|
4223 |
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
|
|
|
4224 |
"dependencies": {
|
4225 |
"ms": "^2.0.0"
|
4226 |
}
|
|
|
4358 |
"node_modules/is-buffer": {
|
4359 |
"version": "1.1.6",
|
4360 |
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
|
4361 |
+
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
|
|
|
4362 |
},
|
4363 |
"node_modules/is-builtin-module": {
|
4364 |
"version": "3.2.1",
|
|
|
4832 |
"version": "2.3.0",
|
4833 |
"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
|
4834 |
"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
|
|
|
4835 |
"dependencies": {
|
4836 |
"charenc": "0.0.2",
|
4837 |
"crypt": "0.0.2",
|
|
|
5128 |
"url": "https://paypal.me/jimmywarting"
|
5129 |
}
|
5130 |
],
|
|
|
5131 |
"engines": {
|
5132 |
"node": ">=10.5.0"
|
5133 |
}
|
|
|
5136 |
"version": "2.7.0",
|
5137 |
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
5138 |
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
|
|
5139 |
"dependencies": {
|
5140 |
"whatwg-url": "^5.0.0"
|
5141 |
},
|
|
|
5154 |
"node_modules/node-fetch/node_modules/tr46": {
|
5155 |
"version": "0.0.3",
|
5156 |
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
5157 |
+
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
|
|
5158 |
},
|
5159 |
"node_modules/node-fetch/node_modules/webidl-conversions": {
|
5160 |
"version": "3.0.1",
|
5161 |
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
5162 |
+
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
|
|
5163 |
},
|
5164 |
"node_modules/node-fetch/node_modules/whatwg-url": {
|
5165 |
"version": "5.0.0",
|
5166 |
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
5167 |
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
|
|
5168 |
"dependencies": {
|
5169 |
"tr46": "~0.0.3",
|
5170 |
"webidl-conversions": "^3.0.0"
|
|
|
8183 |
"version": "4.0.0-beta.3",
|
8184 |
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
|
8185 |
"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
|
|
|
8186 |
"engines": {
|
8187 |
"node": ">= 14"
|
8188 |
}
|
package.json
CHANGED
@@ -82,6 +82,7 @@
|
|
82 |
"zod": "^3.22.3"
|
83 |
},
|
84 |
"optionalDependencies": {
|
|
|
85 |
"aws4fetch": "^1.0.17",
|
86 |
"openai": "^4.14.2"
|
87 |
}
|
|
|
82 |
"zod": "^3.22.3"
|
83 |
},
|
84 |
"optionalDependencies": {
|
85 |
+
"@anthropic-ai/sdk": "^0.17.1",
|
86 |
"aws4fetch": "^1.0.17",
|
87 |
"openai": "^4.14.2"
|
88 |
}
|
src/lib/server/endpoints/anthropic/endpointAnthropic.ts
ADDED
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { z } from "zod";
|
2 |
+
import { ANTHROPIC_API_KEY } from "$env/static/private";
|
3 |
+
import type { Endpoint } from "../endpoints";
|
4 |
+
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
5 |
+
|
6 |
+
export const endpointAnthropicParametersSchema = z.object({
|
7 |
+
weight: z.number().int().positive().default(1),
|
8 |
+
model: z.any(),
|
9 |
+
type: z.literal("anthropic"),
|
10 |
+
baseURL: z.string().url().default("https://api.anthropic.com"),
|
11 |
+
apiKey: z.string().default(ANTHROPIC_API_KEY ?? "sk-"),
|
12 |
+
defaultHeaders: z.record(z.string()).optional(),
|
13 |
+
defaultQuery: z.record(z.string()).optional(),
|
14 |
+
});
|
15 |
+
|
16 |
+
export async function endpointAnthropic(
|
17 |
+
input: z.input<typeof endpointAnthropicParametersSchema>
|
18 |
+
): Promise<Endpoint> {
|
19 |
+
const { baseURL, apiKey, model, defaultHeaders, defaultQuery } =
|
20 |
+
endpointAnthropicParametersSchema.parse(input);
|
21 |
+
let Anthropic;
|
22 |
+
try {
|
23 |
+
Anthropic = (await import("@anthropic-ai/sdk")).default;
|
24 |
+
} catch (e) {
|
25 |
+
throw new Error("Failed to import @anthropic-ai/sdk", { cause: e });
|
26 |
+
}
|
27 |
+
|
28 |
+
const anthropic = new Anthropic({
|
29 |
+
apiKey,
|
30 |
+
baseURL,
|
31 |
+
defaultHeaders,
|
32 |
+
defaultQuery,
|
33 |
+
});
|
34 |
+
|
35 |
+
return async ({ messages, preprompt }) => {
|
36 |
+
let system = preprompt;
|
37 |
+
if (messages?.[0]?.from === "system") {
|
38 |
+
system = messages[0].content;
|
39 |
+
}
|
40 |
+
|
41 |
+
const messagesFormatted = messages
|
42 |
+
.filter((message) => message.from !== "system")
|
43 |
+
.map((message) => ({
|
44 |
+
role: message.from,
|
45 |
+
content: message.content,
|
46 |
+
})) as unknown as {
|
47 |
+
role: "user" | "assistant";
|
48 |
+
content: string;
|
49 |
+
}[];
|
50 |
+
|
51 |
+
let tokenId = 0;
|
52 |
+
return (async function* () {
|
53 |
+
const stream = anthropic.messages.stream({
|
54 |
+
model: model.id ?? model.name,
|
55 |
+
messages: messagesFormatted,
|
56 |
+
max_tokens: model.parameters?.max_new_tokens,
|
57 |
+
temperature: model.parameters?.temperature,
|
58 |
+
top_p: model.parameters?.top_p,
|
59 |
+
top_k: model.parameters?.top_k,
|
60 |
+
stop_sequences: model.parameters?.stop,
|
61 |
+
system,
|
62 |
+
});
|
63 |
+
while (true) {
|
64 |
+
const result = await Promise.race([stream.emitted("text"), stream.emitted("end")]);
|
65 |
+
|
66 |
+
// Stream end
|
67 |
+
if (result === undefined) {
|
68 |
+
yield {
|
69 |
+
token: {
|
70 |
+
id: tokenId++,
|
71 |
+
text: "",
|
72 |
+
logprob: 0,
|
73 |
+
special: true,
|
74 |
+
},
|
75 |
+
generated_text: await stream.finalText(),
|
76 |
+
details: null,
|
77 |
+
} satisfies TextGenerationStreamOutput;
|
78 |
+
return;
|
79 |
+
}
|
80 |
+
|
81 |
+
// Text delta
|
82 |
+
yield {
|
83 |
+
token: {
|
84 |
+
id: tokenId++,
|
85 |
+
text: result as unknown as string,
|
86 |
+
special: false,
|
87 |
+
logprob: 0,
|
88 |
+
},
|
89 |
+
generated_text: null,
|
90 |
+
details: null,
|
91 |
+
} satisfies TextGenerationStreamOutput;
|
92 |
+
}
|
93 |
+
})();
|
94 |
+
};
|
95 |
+
}
|
src/lib/server/endpoints/endpoints.ts
CHANGED
@@ -6,6 +6,10 @@ import endpointAws, { endpointAwsParametersSchema } from "./aws/endpointAws";
|
|
6 |
import { endpointOAIParametersSchema, endpointOai } from "./openai/endpointOai";
|
7 |
import endpointLlamacpp, { endpointLlamacppParametersSchema } from "./llamacpp/endpointLlamacpp";
|
8 |
import endpointOllama, { endpointOllamaParametersSchema } from "./ollama/endpointOllama";
|
|
|
|
|
|
|
|
|
9 |
|
10 |
// parameters passed when generating text
|
11 |
export interface EndpointParameters {
|
@@ -28,6 +32,7 @@ export type EndpointGenerator<T extends CommonEndpoint> = (parameters: T) => End
|
|
28 |
// list of all endpoint generators
|
29 |
export const endpoints = {
|
30 |
tgi: endpointTgi,
|
|
|
31 |
aws: endpointAws,
|
32 |
openai: endpointOai,
|
33 |
llamacpp: endpointLlamacpp,
|
@@ -35,6 +40,7 @@ export const endpoints = {
|
|
35 |
};
|
36 |
|
37 |
export const endpointSchema = z.discriminatedUnion("type", [
|
|
|
38 |
endpointAwsParametersSchema,
|
39 |
endpointOAIParametersSchema,
|
40 |
endpointTgiParametersSchema,
|
|
|
6 |
import { endpointOAIParametersSchema, endpointOai } from "./openai/endpointOai";
|
7 |
import endpointLlamacpp, { endpointLlamacppParametersSchema } from "./llamacpp/endpointLlamacpp";
|
8 |
import endpointOllama, { endpointOllamaParametersSchema } from "./ollama/endpointOllama";
|
9 |
+
import {
|
10 |
+
endpointAnthropic,
|
11 |
+
endpointAnthropicParametersSchema,
|
12 |
+
} from "./anthropic/endpointAnthropic";
|
13 |
|
14 |
// parameters passed when generating text
|
15 |
export interface EndpointParameters {
|
|
|
32 |
// list of all endpoint generators
|
33 |
export const endpoints = {
|
34 |
tgi: endpointTgi,
|
35 |
+
anthropic: endpointAnthropic,
|
36 |
aws: endpointAws,
|
37 |
openai: endpointOai,
|
38 |
llamacpp: endpointLlamacpp,
|
|
|
40 |
};
|
41 |
|
42 |
export const endpointSchema = z.discriminatedUnion("type", [
|
43 |
+
endpointAnthropicParametersSchema,
|
44 |
endpointAwsParametersSchema,
|
45 |
endpointOAIParametersSchema,
|
46 |
endpointTgiParametersSchema,
|
src/lib/server/models.ts
CHANGED
@@ -109,6 +109,8 @@ const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
|
|
109 |
switch (args.type) {
|
110 |
case "tgi":
|
111 |
return endpoints.tgi(args);
|
|
|
|
|
112 |
case "aws":
|
113 |
return await endpoints.aws(args);
|
114 |
case "openai":
|
|
|
109 |
switch (args.type) {
|
110 |
case "tgi":
|
111 |
return endpoints.tgi(args);
|
112 |
+
case "anthropic":
|
113 |
+
return endpoints.anthropic(args);
|
114 |
case "aws":
|
115 |
return await endpoints.aws(args);
|
116 |
case "openai":
|