support tokenised prompt (online vllm)

preprocessing_molmo.py

@@ -116,6 +116,8 @@ class MolmoProcessor(ProcessorMixin):
         self,
         text: TextInput = None,
         images: ImageInput = None,
+        *,
+        tokens = None,
         **kwargs: Unpack[MolmoProcessorKwargs],
     ):
         output_kwargs = self._merge_kwargs(
@@ -123,12 +125,12 @@ class MolmoProcessor(ProcessorMixin):
             tokenizer_init_kwargs=self.tokenizer.init_kwargs,
             **kwargs,
         )
-
-        tokens = self.get_tokens_input(
-            text,
-            output_kwargs["text_kwargs"]["message_format"],
-            output_kwargs["text_kwargs"]["always_start_with_space"],
-        )
+        if tokens is None:
+            tokens = self.get_tokens_input(
+                text,
+                output_kwargs["text_kwargs"]["message_format"],
+                output_kwargs["text_kwargs"]["always_start_with_space"],
+            )
 
         image_token_id = self.special_token_ids[IMAGE_PROMPT]