Group tokens randomly (#964)

* Group tokens together

* remove unused comment

* Send final buffer at the end if needed

src/routes/conversation/[id]/+server.ts

@@ -409,6 +409,8 @@ export async function POST({ request, locals, params, getClientAddress }) {
 
 			let hasError = false;
 
+			let buffer = "";
+
 			try {
 				const endpoint = await model.getEndpoint();
 				for await (const output of await endpoint({
@@ -419,12 +421,18 @@ export async function POST({ request, locals, params, getClientAddress }) {
 				})) {
 					// if not generated_text is here it means the generation is not done
 					if (!output.generated_text) {
-						// else we get the next token
 						if (!output.token.special) {
-							update({
-								type: "stream",
-								token: output.token.text,
-							});
+							// 33% chance to send the stream update, with a max buffer size of 30 chars
+							buffer += output.token.text;
+
+							if (Math.random() < 0.33 || buffer.length > 30) {
+								update({
+									type: "stream",
+									token: buffer,
+								});
+								buffer = "";
+							}
+
 							// abort check
 							const date = abortedGenerations.get(convId.toString());
 							if (date && date > promptedAt) {
@@ -466,6 +474,13 @@ export async function POST({ request, locals, params, getClientAddress }) {
 						message: "No output was generated. Something went wrong.",
 					});
 				}
+
+				if (buffer) {
+					update({
+						type: "stream",
+						token: buffer,
+					});
+				}
 			}
 
 			await collections.conversations.updateOne(