Delete handler.py

handler.py

@@ -1,53 +0,0 @@
-from typing import Any, Dict
-
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
-
-
-class EndpointHandler:
-    def __init__(self, path=""):
-        # load model and processor from path
-        self.tokenizer = AutoTokenizer.from_pretrained(path)
-        # try:
-        # config = AutoConfig.from_pretrained(path)
-        model = AutoModelForCausalLM.from_pretrained(
-            path,
-            # return_dict=True,
-            # load_in_8bit=True,
-            device_map="auto",
-            torch_dtype=torch.float16,
-            trust_remote_code=True,
-        )
-        # model.resize_token_embeddings(len(self.tokenizer))
-        # model = PeftModel.from_pretrained(model, path)
-        # except Exception:
-        #     model = AutoModelForCausalLM.from_pretrained(
-        #         path, device_map="auto", load_in_8bit=True, torch_dtype=torch.float16, trust_remote_code=True
-        #     )
-        self.model = model
-        self.device = "cuda" if torch.cuda.is_available() else "cpu"
-
-    def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
-        # process input
-        inputs = data.pop("inputs", data)
-        parameters = data.pop("parameters", None)
-
-        messages=[{ 'role': 'user', 'content': inputs}]
-        inputs = self.tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt").to(self.device)
-
-        # preprocess
-        # inputs = self.tokenizer(inputs, return_tensors="pt").to(self.device)
-
-        # pass inputs with all kwargs in data
-        if parameters is not None:
-            outputs = self.model.generate(inputs, max_new_tokens=880, num_return_sequences=1, eos_token_id=self.tokenizer.eos_token_id, **parameters)
-        else:
-            outputs = self.model.generate(inputs, max_new_tokens=880, num_return_sequences=1, eos_token_id=self.tokenizer.eos_token_id)
-
-        # postprocess the prediction
-        prediction = self.tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
-
-        return [{"generated_text": prediction}]
-
-        # outputs = model.generate(inputs, max_new_tokens=512, do_sample=False, top_k=50, top_p=0.95, num_return_sequences=1, eos_token_id=tokenizer.eos_token_id)
-        # print(tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True))