Liam Dyer commited on
Commit
5a31018
1 Parent(s): 3f5871c

Anthropic Endpoint Support (#923)

Browse files
.env CHANGED
@@ -9,6 +9,7 @@ COOKIE_NAME=hf-chat
9
  HF_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
10
  HF_API_ROOT=https://api-inference.huggingface.co/models
11
  OPENAI_API_KEY=#your openai api key here
 
12
 
13
  HF_ACCESS_TOKEN=#LEGACY! Use HF_TOKEN instead
14
 
 
9
  HF_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
10
  HF_API_ROOT=https://api-inference.huggingface.co/models
11
  OPENAI_API_KEY=#your openai api key here
12
+ ANTHROPIC_API_KEY=#your anthropic api key here
13
 
14
  HF_ACCESS_TOKEN=#LEGACY! Use HF_TOKEN instead
15
 
README.md CHANGED
@@ -459,6 +459,51 @@ MODELS=`[
459
  ]`
460
  ```
461
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
462
  #### Amazon
463
 
464
  You can also specify your Amazon SageMaker instance as an endpoint for chat-ui. The config goes like this:
 
459
  ]`
460
  ```
461
 
462
+ #### Anthropic
463
+
464
+ We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
465
+
466
+ ```
467
+ MODELS=`[
468
+ {
469
+ "name": "claude-3-sonnet-20240229",
470
+ "displayName": "Claude 3 Sonnet",
471
+ "description": "Ideal balance of intelligence and speed",
472
+ "parameters": {
473
+ "max_new_tokens": 4096,
474
+ },
475
+ "endpoints": [
476
+ {
477
+ "type": "anthropic",
478
+ // optionals
479
+ "apiKey": "sk-ant-...",
480
+ "baseURL": "https://api.anthropic.com",
481
+ defaultHeaders: {},
482
+ defaultQuery: {}
483
+ }
484
+ ]
485
+ },
486
+ {
487
+ "name": "claude-3-opus-20240229",
488
+ "displayName": "Claude 3 Opus",
489
+ "description": "Most powerful model for highly complex tasks",
490
+ "parameters": {
491
+ "max_new_tokens": 4096
492
+ },
493
+ "endpoints": [
494
+ {
495
+ "type": "anthropic",
496
+ // optionals
497
+ "apiKey": "sk-ant-...",
498
+ "baseURL": "https://api.anthropic.com",
499
+ defaultHeaders: {},
500
+ defaultQuery: {}
501
+ }
502
+ ]
503
+ }
504
+ ]`
505
+ ```
506
+
507
  #### Amazon
508
 
509
  You can also specify your Amazon SageMaker instance as an endpoint for chat-ui. The config goes like this:
package-lock.json CHANGED
@@ -8,6 +8,7 @@
8
  "name": "chat-ui",
9
  "version": "0.7.0",
10
  "dependencies": {
 
11
  "@huggingface/hub": "^0.5.1",
12
  "@huggingface/inference": "^2.6.3",
13
  "@iconify-json/bi": "^1.1.21",
@@ -122,6 +123,30 @@
122
  "url": "https://github.com/sponsors/antfu"
123
  }
124
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  "node_modules/@cspotcode/source-map-support": {
126
  "version": "0.8.1",
127
  "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -2013,7 +2038,6 @@
2013
  "version": "2.6.5",
2014
  "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz",
2015
  "integrity": "sha512-OZsUlr2nxvkqUFLSaY2ZbA+P1q22q+KrlxWOn/38RX+u5kTkYL2mTujEpzUhGkS+K/QCYp9oagfXG39XOzyySg==",
2016
- "optional": true,
2017
  "dependencies": {
2018
  "@types/node": "*",
2019
  "form-data": "^4.0.0"
@@ -2409,7 +2433,6 @@
2409
  "version": "3.0.0",
2410
  "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
2411
  "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
2412
- "optional": true,
2413
  "dependencies": {
2414
  "event-target-shim": "^5.0.0"
2415
  },
@@ -2462,7 +2485,6 @@
2462
  "version": "4.5.0",
2463
  "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
2464
  "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
2465
- "optional": true,
2466
  "dependencies": {
2467
  "humanize-ms": "^1.2.1"
2468
  },
@@ -2630,8 +2652,7 @@
2630
  "node_modules/base-64": {
2631
  "version": "0.1.0",
2632
  "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
2633
- "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==",
2634
- "optional": true
2635
  },
2636
  "node_modules/base64-js": {
2637
  "version": "1.5.1",
@@ -2899,7 +2920,6 @@
2899
  "version": "0.0.2",
2900
  "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
2901
  "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
2902
- "optional": true,
2903
  "engines": {
2904
  "node": "*"
2905
  }
@@ -3096,7 +3116,6 @@
3096
  "version": "0.0.2",
3097
  "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
3098
  "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
3099
- "optional": true,
3100
  "engines": {
3101
  "node": "*"
3102
  }
@@ -3352,7 +3371,6 @@
3352
  "version": "1.3.0",
3353
  "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
3354
  "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
3355
- "optional": true,
3356
  "dependencies": {
3357
  "base-64": "^0.1.0",
3358
  "md5": "^2.3.0"
@@ -3724,7 +3742,6 @@
3724
  "version": "5.0.1",
3725
  "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
3726
  "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
3727
- "optional": true,
3728
  "engines": {
3729
  "node": ">=6"
3730
  }
@@ -3907,14 +3924,12 @@
3907
  "node_modules/form-data-encoder": {
3908
  "version": "1.7.2",
3909
  "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
3910
- "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
3911
- "optional": true
3912
  },
3913
  "node_modules/formdata-node": {
3914
  "version": "4.4.1",
3915
  "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
3916
  "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
3917
- "optional": true,
3918
  "dependencies": {
3919
  "node-domexception": "1.0.0",
3920
  "web-streams-polyfill": "4.0.0-beta.3"
@@ -4206,7 +4221,6 @@
4206
  "version": "1.2.1",
4207
  "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
4208
  "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
4209
- "optional": true,
4210
  "dependencies": {
4211
  "ms": "^2.0.0"
4212
  }
@@ -4344,8 +4358,7 @@
4344
  "node_modules/is-buffer": {
4345
  "version": "1.1.6",
4346
  "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
4347
- "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
4348
- "optional": true
4349
  },
4350
  "node_modules/is-builtin-module": {
4351
  "version": "3.2.1",
@@ -4819,7 +4832,6 @@
4819
  "version": "2.3.0",
4820
  "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
4821
  "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
4822
- "optional": true,
4823
  "dependencies": {
4824
  "charenc": "0.0.2",
4825
  "crypt": "0.0.2",
@@ -5116,7 +5128,6 @@
5116
  "url": "https://paypal.me/jimmywarting"
5117
  }
5118
  ],
5119
- "optional": true,
5120
  "engines": {
5121
  "node": ">=10.5.0"
5122
  }
@@ -5125,7 +5136,6 @@
5125
  "version": "2.7.0",
5126
  "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
5127
  "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
5128
- "optional": true,
5129
  "dependencies": {
5130
  "whatwg-url": "^5.0.0"
5131
  },
@@ -5144,20 +5154,17 @@
5144
  "node_modules/node-fetch/node_modules/tr46": {
5145
  "version": "0.0.3",
5146
  "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
5147
- "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
5148
- "optional": true
5149
  },
5150
  "node_modules/node-fetch/node_modules/webidl-conversions": {
5151
  "version": "3.0.1",
5152
  "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
5153
- "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
5154
- "optional": true
5155
  },
5156
  "node_modules/node-fetch/node_modules/whatwg-url": {
5157
  "version": "5.0.0",
5158
  "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
5159
  "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
5160
- "optional": true,
5161
  "dependencies": {
5162
  "tr46": "~0.0.3",
5163
  "webidl-conversions": "^3.0.0"
@@ -8176,7 +8183,6 @@
8176
  "version": "4.0.0-beta.3",
8177
  "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
8178
  "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
8179
- "optional": true,
8180
  "engines": {
8181
  "node": ">= 14"
8182
  }
 
8
  "name": "chat-ui",
9
  "version": "0.7.0",
10
  "dependencies": {
11
+ "@anthropic-ai/sdk": "^0.17.1",
12
  "@huggingface/hub": "^0.5.1",
13
  "@huggingface/inference": "^2.6.3",
14
  "@iconify-json/bi": "^1.1.21",
 
123
  "url": "https://github.com/sponsors/antfu"
124
  }
125
  },
126
+ "node_modules/@anthropic-ai/sdk": {
127
+ "version": "0.17.1",
128
+ "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.17.1.tgz",
129
+ "integrity": "sha512-ke/JGfaa4sc1PB58L4B9hXI/BlJphXc696+cVX8Z8gQt51l++a9umZTN/7UymV8Dcat6KKYNQE8P8yeeyAldHg==",
130
+ "dependencies": {
131
+ "@types/node": "^18.11.18",
132
+ "@types/node-fetch": "^2.6.4",
133
+ "abort-controller": "^3.0.0",
134
+ "agentkeepalive": "^4.2.1",
135
+ "digest-fetch": "^1.3.0",
136
+ "form-data-encoder": "1.7.2",
137
+ "formdata-node": "^4.3.2",
138
+ "node-fetch": "^2.6.7",
139
+ "web-streams-polyfill": "^3.2.1"
140
+ }
141
+ },
142
+ "node_modules/@anthropic-ai/sdk/node_modules/web-streams-polyfill": {
143
+ "version": "3.3.3",
144
+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
145
+ "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
146
+ "engines": {
147
+ "node": ">= 8"
148
+ }
149
+ },
150
  "node_modules/@cspotcode/source-map-support": {
151
  "version": "0.8.1",
152
  "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
 
2038
  "version": "2.6.5",
2039
  "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz",
2040
  "integrity": "sha512-OZsUlr2nxvkqUFLSaY2ZbA+P1q22q+KrlxWOn/38RX+u5kTkYL2mTujEpzUhGkS+K/QCYp9oagfXG39XOzyySg==",
 
2041
  "dependencies": {
2042
  "@types/node": "*",
2043
  "form-data": "^4.0.0"
 
2433
  "version": "3.0.0",
2434
  "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
2435
  "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
 
2436
  "dependencies": {
2437
  "event-target-shim": "^5.0.0"
2438
  },
 
2485
  "version": "4.5.0",
2486
  "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
2487
  "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
 
2488
  "dependencies": {
2489
  "humanize-ms": "^1.2.1"
2490
  },
 
2652
  "node_modules/base-64": {
2653
  "version": "0.1.0",
2654
  "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
2655
+ "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
 
2656
  },
2657
  "node_modules/base64-js": {
2658
  "version": "1.5.1",
 
2920
  "version": "0.0.2",
2921
  "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
2922
  "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
 
2923
  "engines": {
2924
  "node": "*"
2925
  }
 
3116
  "version": "0.0.2",
3117
  "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
3118
  "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
 
3119
  "engines": {
3120
  "node": "*"
3121
  }
 
3371
  "version": "1.3.0",
3372
  "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
3373
  "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
 
3374
  "dependencies": {
3375
  "base-64": "^0.1.0",
3376
  "md5": "^2.3.0"
 
3742
  "version": "5.0.1",
3743
  "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
3744
  "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
 
3745
  "engines": {
3746
  "node": ">=6"
3747
  }
 
3924
  "node_modules/form-data-encoder": {
3925
  "version": "1.7.2",
3926
  "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
3927
+ "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
 
3928
  },
3929
  "node_modules/formdata-node": {
3930
  "version": "4.4.1",
3931
  "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
3932
  "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
 
3933
  "dependencies": {
3934
  "node-domexception": "1.0.0",
3935
  "web-streams-polyfill": "4.0.0-beta.3"
 
4221
  "version": "1.2.1",
4222
  "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
4223
  "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
 
4224
  "dependencies": {
4225
  "ms": "^2.0.0"
4226
  }
 
4358
  "node_modules/is-buffer": {
4359
  "version": "1.1.6",
4360
  "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
4361
+ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
 
4362
  },
4363
  "node_modules/is-builtin-module": {
4364
  "version": "3.2.1",
 
4832
  "version": "2.3.0",
4833
  "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
4834
  "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
 
4835
  "dependencies": {
4836
  "charenc": "0.0.2",
4837
  "crypt": "0.0.2",
 
5128
  "url": "https://paypal.me/jimmywarting"
5129
  }
5130
  ],
 
5131
  "engines": {
5132
  "node": ">=10.5.0"
5133
  }
 
5136
  "version": "2.7.0",
5137
  "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
5138
  "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
 
5139
  "dependencies": {
5140
  "whatwg-url": "^5.0.0"
5141
  },
 
5154
  "node_modules/node-fetch/node_modules/tr46": {
5155
  "version": "0.0.3",
5156
  "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
5157
+ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
 
5158
  },
5159
  "node_modules/node-fetch/node_modules/webidl-conversions": {
5160
  "version": "3.0.1",
5161
  "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
5162
+ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
 
5163
  },
5164
  "node_modules/node-fetch/node_modules/whatwg-url": {
5165
  "version": "5.0.0",
5166
  "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
5167
  "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
 
5168
  "dependencies": {
5169
  "tr46": "~0.0.3",
5170
  "webidl-conversions": "^3.0.0"
 
8183
  "version": "4.0.0-beta.3",
8184
  "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
8185
  "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
 
8186
  "engines": {
8187
  "node": ">= 14"
8188
  }
package.json CHANGED
@@ -82,6 +82,7 @@
82
  "zod": "^3.22.3"
83
  },
84
  "optionalDependencies": {
 
85
  "aws4fetch": "^1.0.17",
86
  "openai": "^4.14.2"
87
  }
 
82
  "zod": "^3.22.3"
83
  },
84
  "optionalDependencies": {
85
+ "@anthropic-ai/sdk": "^0.17.1",
86
  "aws4fetch": "^1.0.17",
87
  "openai": "^4.14.2"
88
  }
src/lib/server/endpoints/anthropic/endpointAnthropic.ts ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { z } from "zod";
2
+ import { ANTHROPIC_API_KEY } from "$env/static/private";
3
+ import type { Endpoint } from "../endpoints";
4
+ import type { TextGenerationStreamOutput } from "@huggingface/inference";
5
+
6
+ export const endpointAnthropicParametersSchema = z.object({
7
+ weight: z.number().int().positive().default(1),
8
+ model: z.any(),
9
+ type: z.literal("anthropic"),
10
+ baseURL: z.string().url().default("https://api.anthropic.com"),
11
+ apiKey: z.string().default(ANTHROPIC_API_KEY ?? "sk-"),
12
+ defaultHeaders: z.record(z.string()).optional(),
13
+ defaultQuery: z.record(z.string()).optional(),
14
+ });
15
+
16
+ export async function endpointAnthropic(
17
+ input: z.input<typeof endpointAnthropicParametersSchema>
18
+ ): Promise<Endpoint> {
19
+ const { baseURL, apiKey, model, defaultHeaders, defaultQuery } =
20
+ endpointAnthropicParametersSchema.parse(input);
21
+ let Anthropic;
22
+ try {
23
+ Anthropic = (await import("@anthropic-ai/sdk")).default;
24
+ } catch (e) {
25
+ throw new Error("Failed to import @anthropic-ai/sdk", { cause: e });
26
+ }
27
+
28
+ const anthropic = new Anthropic({
29
+ apiKey,
30
+ baseURL,
31
+ defaultHeaders,
32
+ defaultQuery,
33
+ });
34
+
35
+ return async ({ messages, preprompt }) => {
36
+ let system = preprompt;
37
+ if (messages?.[0]?.from === "system") {
38
+ system = messages[0].content;
39
+ }
40
+
41
+ const messagesFormatted = messages
42
+ .filter((message) => message.from !== "system")
43
+ .map((message) => ({
44
+ role: message.from,
45
+ content: message.content,
46
+ })) as unknown as {
47
+ role: "user" | "assistant";
48
+ content: string;
49
+ }[];
50
+
51
+ let tokenId = 0;
52
+ return (async function* () {
53
+ const stream = anthropic.messages.stream({
54
+ model: model.id ?? model.name,
55
+ messages: messagesFormatted,
56
+ max_tokens: model.parameters?.max_new_tokens,
57
+ temperature: model.parameters?.temperature,
58
+ top_p: model.parameters?.top_p,
59
+ top_k: model.parameters?.top_k,
60
+ stop_sequences: model.parameters?.stop,
61
+ system,
62
+ });
63
+ while (true) {
64
+ const result = await Promise.race([stream.emitted("text"), stream.emitted("end")]);
65
+
66
+ // Stream end
67
+ if (result === undefined) {
68
+ yield {
69
+ token: {
70
+ id: tokenId++,
71
+ text: "",
72
+ logprob: 0,
73
+ special: true,
74
+ },
75
+ generated_text: await stream.finalText(),
76
+ details: null,
77
+ } satisfies TextGenerationStreamOutput;
78
+ return;
79
+ }
80
+
81
+ // Text delta
82
+ yield {
83
+ token: {
84
+ id: tokenId++,
85
+ text: result as unknown as string,
86
+ special: false,
87
+ logprob: 0,
88
+ },
89
+ generated_text: null,
90
+ details: null,
91
+ } satisfies TextGenerationStreamOutput;
92
+ }
93
+ })();
94
+ };
95
+ }
src/lib/server/endpoints/endpoints.ts CHANGED
@@ -6,6 +6,10 @@ import endpointAws, { endpointAwsParametersSchema } from "./aws/endpointAws";
6
  import { endpointOAIParametersSchema, endpointOai } from "./openai/endpointOai";
7
  import endpointLlamacpp, { endpointLlamacppParametersSchema } from "./llamacpp/endpointLlamacpp";
8
  import endpointOllama, { endpointOllamaParametersSchema } from "./ollama/endpointOllama";
 
 
 
 
9
 
10
  // parameters passed when generating text
11
  export interface EndpointParameters {
@@ -28,6 +32,7 @@ export type EndpointGenerator<T extends CommonEndpoint> = (parameters: T) => End
28
  // list of all endpoint generators
29
  export const endpoints = {
30
  tgi: endpointTgi,
 
31
  aws: endpointAws,
32
  openai: endpointOai,
33
  llamacpp: endpointLlamacpp,
@@ -35,6 +40,7 @@ export const endpoints = {
35
  };
36
 
37
  export const endpointSchema = z.discriminatedUnion("type", [
 
38
  endpointAwsParametersSchema,
39
  endpointOAIParametersSchema,
40
  endpointTgiParametersSchema,
 
6
  import { endpointOAIParametersSchema, endpointOai } from "./openai/endpointOai";
7
  import endpointLlamacpp, { endpointLlamacppParametersSchema } from "./llamacpp/endpointLlamacpp";
8
  import endpointOllama, { endpointOllamaParametersSchema } from "./ollama/endpointOllama";
9
+ import {
10
+ endpointAnthropic,
11
+ endpointAnthropicParametersSchema,
12
+ } from "./anthropic/endpointAnthropic";
13
 
14
  // parameters passed when generating text
15
  export interface EndpointParameters {
 
32
  // list of all endpoint generators
33
  export const endpoints = {
34
  tgi: endpointTgi,
35
+ anthropic: endpointAnthropic,
36
  aws: endpointAws,
37
  openai: endpointOai,
38
  llamacpp: endpointLlamacpp,
 
40
  };
41
 
42
  export const endpointSchema = z.discriminatedUnion("type", [
43
+ endpointAnthropicParametersSchema,
44
  endpointAwsParametersSchema,
45
  endpointOAIParametersSchema,
46
  endpointTgiParametersSchema,
src/lib/server/models.ts CHANGED
@@ -109,6 +109,8 @@ const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
109
  switch (args.type) {
110
  case "tgi":
111
  return endpoints.tgi(args);
 
 
112
  case "aws":
113
  return await endpoints.aws(args);
114
  case "openai":
 
109
  switch (args.type) {
110
  case "tgi":
111
  return endpoints.tgi(args);
112
+ case "anthropic":
113
+ return endpoints.anthropic(args);
114
  case "aws":
115
  return await endpoints.aws(args);
116
  case "openai":