|
| 1 | +from pathlib import Path |
| 2 | +from typing import Any, Optional, Union |
| 3 | + |
| 4 | +from dsp.modules.lm import LM |
| 5 | + |
| 6 | +## Utility functions to load models |
| 7 | + |
| 8 | + |
| 9 | +def load_tensorrt_model( |
| 10 | + engine_dir: Union[str, Path], |
| 11 | + use_py_session: Optional[bool] = False, |
| 12 | + **kwargs, |
| 13 | +) -> tuple[Any, dict]: |
| 14 | + import tensorrt_llm |
| 15 | + from tensorrt_llm.runtime import ModelRunner, ModelRunnerCpp |
| 16 | + |
| 17 | + runtime_rank = tensorrt_llm.mpi_rank() |
| 18 | + runner_cls = ModelRunner if use_py_session else ModelRunnerCpp |
| 19 | + runner_kwargs = { |
| 20 | + "engine_dir": engine_dir, |
| 21 | + "lora_dir": kwargs.get("lora_dir", None), |
| 22 | + "rank": runtime_rank, |
| 23 | + "lora_ckpt_source": kwargs.get("lora_ckpt_source", "hf"), |
| 24 | + } |
| 25 | + |
| 26 | + if not use_py_session: |
| 27 | + engine_cpp_kwargs = {} |
| 28 | + defaults = { |
| 29 | + "max_batch_size": 1, |
| 30 | + "max_input_len": 1024, |
| 31 | + "max_output_len": 1024, |
| 32 | + "max_beam_width": 1, |
| 33 | + "max_attention_window_size": None, |
| 34 | + "sink_token_length": None, |
| 35 | + } |
| 36 | + |
| 37 | + for key, value in defaults.items(): |
| 38 | + engine_cpp_kwargs[key] = kwargs.get(key, value) |
| 39 | + runner_kwargs.update(**engine_cpp_kwargs) |
| 40 | + |
| 41 | + runner = runner_cls.from_dir(**runner_kwargs) |
| 42 | + return runner, runner_kwargs |
| 43 | + |
| 44 | + |
| 45 | +def tokenize(prompt: Union[list[dict], str], tokenizer: Any, **kwargs) -> list[int]: |
| 46 | + defaults = { |
| 47 | + "add_special_tokens": False, |
| 48 | + "max_input_length": 1024, |
| 49 | + "model_name": None, |
| 50 | + "model_version": None, |
| 51 | + } |
| 52 | + if not isinstance(prompt, str): |
| 53 | + prompt = tokenizer.apply_chat_template(prompt, tokenize=False) |
| 54 | + |
| 55 | + input_ids = [ |
| 56 | + tokenizer.encode( |
| 57 | + prompt, |
| 58 | + add_special_tokens=kwargs.get("add_special_tokens", defaults["add_special_tokens"]), |
| 59 | + truncation=True, |
| 60 | + max_length=kwargs.get("max_input_length", defaults["max_input_length"]), |
| 61 | + ), |
| 62 | + ] |
| 63 | + if ( |
| 64 | + kwargs.get("model_name", defaults["model_name"]) == "ChatGLMForCausalLM" |
| 65 | + and kwargs.get("model_version", defaults["model_version"]) == "glm" |
| 66 | + ): |
| 67 | + input_ids.append(tokenizer.stop_token_id) |
| 68 | + return input_ids |
| 69 | + |
| 70 | + |
| 71 | +class TensorRTModel(LM): |
| 72 | + """TensorRT integration for dspy LM.""" |
| 73 | + |
| 74 | + def __init__(self, model_name_or_path: str, engine_dir: str, **engine_kwargs: dict) -> None: |
| 75 | + """Initialize the TensorRTModel. |
| 76 | +
|
| 77 | + Args: |
| 78 | + model_name_or_path (str): The Huggingface ID or the path where tokenizer files exist. |
| 79 | + engine_dir (str): The folder where the TensorRT .engine file exists. |
| 80 | + **engine_kwargs (Optional[dict]): Additional engine loading keyword arguments. |
| 81 | +
|
| 82 | + Keyword Args: |
| 83 | + use_py_session (bool, optional): Whether to use a Python session or not. Defaults to False. |
| 84 | + lora_dir (str): The directory of LoRA adapter weights. |
| 85 | + lora_task_uids (list[str]): list of LoRA task UIDs; use -1 to disable the LoRA module. |
| 86 | + lora_ckpt_source (str): The source of the LoRA checkpoint. |
| 87 | +
|
| 88 | + If use_py_session is set to False, the following kwargs are supported: |
| 89 | + max_batch_size (int, optional): The maximum batch size. Defaults to 1. |
| 90 | + max_input_len (int, optional): The maximum input context length. Defaults to 1024. |
| 91 | + max_output_len (int, optional): The maximum output context length. Defaults to 1024. |
| 92 | + max_beam_width (int, optional): The maximum beam width, similar to `n` in OpenAI API. Defaults to 1. |
| 93 | + max_attention_window_size (int, optional): The attention window size that controls the |
| 94 | + sliding window attention / cyclic KV cache behavior. Defaults to None. |
| 95 | + sink_token_length (int, optional): The sink token length. Defaults to 1. |
| 96 | + """ |
| 97 | + # Implementation here |
| 98 | + self.model_name_or_path, self.engine_dir = model_name_or_path, engine_dir |
| 99 | + super().__init__(model=self.model_name_or_path) |
| 100 | + try: |
| 101 | + import tensorrt_llm |
| 102 | + except ImportError as exc: |
| 103 | + raise ModuleNotFoundError( |
| 104 | + "You need to install tensorrt-llm to use TensorRTModel", |
| 105 | + ) from exc |
| 106 | + |
| 107 | + try: |
| 108 | + from transformers import AutoTokenizer |
| 109 | + except ImportError as exc: |
| 110 | + raise ModuleNotFoundError( |
| 111 | + "You need to install torch and transformers ", |
| 112 | + "pip install transformers==4.38.2", |
| 113 | + ) from exc |
| 114 | + |
| 115 | + # Configure tokenizer |
| 116 | + self.tokenizer = AutoTokenizer.from_pretrained( |
| 117 | + self.model_name_or_path, |
| 118 | + legacy=False, |
| 119 | + padding_side="left", |
| 120 | + truncation_side="left", |
| 121 | + trust_remote_code=True, |
| 122 | + use_fast=True, |
| 123 | + ) |
| 124 | + |
| 125 | + self.pad_id = ( |
| 126 | + self.tokenizer.eos_token_id if self.tokenizer.pad_token_id is None else self.tokenizer.pad_token_id |
| 127 | + ) |
| 128 | + self.end_id = self.tokenizer.eos_token_id |
| 129 | + |
| 130 | + # Configure TensorRT |
| 131 | + self.runtime_rank = tensorrt_llm.mpi_rank() |
| 132 | + self.runner, self._runner_kwargs = load_tensorrt_model(engine_dir=self.engine_dir, **engine_kwargs) |
| 133 | + self.history: list[dict[str, Any]] = [] |
| 134 | + |
| 135 | + def _generate(self, prompt: Union[list[dict[str, str]], str], **kwargs: dict) -> tuple[list[str], dict]: |
| 136 | + import torch |
| 137 | + |
| 138 | + input_ids = tokenize(prompt=prompt, tokenizer=self.tokenizer, **kwargs) |
| 139 | + input_ids = torch.tensor(input_ids, dtype=torch.int32) |
| 140 | + |
| 141 | + run_kwargs = {} |
| 142 | + defaults = { |
| 143 | + "max_new_tokens": 1024, |
| 144 | + "max_attention_window_size": None, |
| 145 | + "sink_token_length": None, |
| 146 | + "end_id": self.end_id, |
| 147 | + "pad_id": self.pad_id, |
| 148 | + "temperature": 1.0, |
| 149 | + "top_k": 1, |
| 150 | + "top_p": 0.0, |
| 151 | + "num_beams": 1, |
| 152 | + "length_penalty": 1.0, |
| 153 | + "early_stopping": 1, |
| 154 | + "repetition_penalty": 1.0, |
| 155 | + "presence_penalty": 0.0, |
| 156 | + "frequency_penalty": 0.0, |
| 157 | + "stop_words_list": None, |
| 158 | + "bad_words_list": None, |
| 159 | + "streaming": False, |
| 160 | + "return_dict": True, |
| 161 | + "output_log_probs": False, |
| 162 | + "output_cum_log_probs": False, |
| 163 | + "output_sequence_lengths": True, |
| 164 | + } |
| 165 | + |
| 166 | + for k, v in defaults.items(): |
| 167 | + run_kwargs[k] = kwargs.get(k, v) |
| 168 | + |
| 169 | + with torch.no_grad(): |
| 170 | + outputs = self.runner.generate(input_ids, **run_kwargs) |
| 171 | + input_lengths = [x.size(0) for x in input_ids] |
| 172 | + |
| 173 | + output_ids, sequence_lengths = outputs["output_ids"], outputs["sequence_lengths"] |
| 174 | + |
| 175 | + # In case of current version of dspy it will always stay as 1 |
| 176 | + _, num_beams, _ = output_ids.size() |
| 177 | + batch_idx, beams = 0, [] |
| 178 | + |
| 179 | + for beam in range(num_beams): |
| 180 | + output_begin = input_lengths[batch_idx] |
| 181 | + output_end = sequence_lengths[batch_idx][beam] |
| 182 | + outputs = output_ids[batch_idx][beam][output_begin:output_end].tolist() |
| 183 | + output_text = self.tokenizer.decode(outputs) |
| 184 | + beams.append(output_text) |
| 185 | + |
| 186 | + return beams, run_kwargs |
| 187 | + |
| 188 | + def basic_request(self, prompt, **kwargs: dict) -> list[str]: |
| 189 | + raw_kwargs = kwargs |
| 190 | + response, all_kwargs = self._generate(prompt, **kwargs) |
| 191 | + history = { |
| 192 | + "prompt": prompt, |
| 193 | + "response": response, |
| 194 | + "raw_kwargs": raw_kwargs, |
| 195 | + "kwargs": all_kwargs, |
| 196 | + } |
| 197 | + self.history.append(history) |
| 198 | + return response |
| 199 | + |
| 200 | + def __call__( |
| 201 | + self, |
| 202 | + prompt: Union[list[dict[str, str]], str], |
| 203 | + **kwargs, |
| 204 | + ): |
| 205 | + """TensorRTLLM generate method in dspy. |
| 206 | +
|
| 207 | + Args: |
| 208 | + prompt (Union[list[dict[str, str]], str]): The prompt to pass. If prompt is not string |
| 209 | + then it will assume that chat mode / instruct mode is triggered. |
| 210 | + **kwargs (Optional[dict]): Optional keyword arguments. |
| 211 | +
|
| 212 | + Additional Parameters: |
| 213 | + max_new_tokens (int): The maximum number of tokens to output. Defaults to 1024 |
| 214 | + max_attention_window_size (int) Defaults to None |
| 215 | + sink_token_length (int): Defaults to None |
| 216 | + end_id (int): The end of sequence of ID of tokenize, defaults to tokenizer's default |
| 217 | + end id |
| 218 | + pad_id (int): The pd sequence of ID of tokenize, defaults to tokenizer's default end id |
| 219 | + temperature (float): The temperature to control probabilistic behaviour in generation |
| 220 | + Defaults to 1.0 |
| 221 | + top_k (int): Defaults to 1 |
| 222 | + top_p (float): Defaults to 1 |
| 223 | + num_beams: (int): The number of responses to generate. Defaults to 1 |
| 224 | + length_penalty (float): Defaults to 1.0 |
| 225 | + repetition_penalty (float): Defaults to 1.0 |
| 226 | + presence_penalty (float): Defaults to 0.0 |
| 227 | + frequency_penalty (float): Defaults to 0.0 |
| 228 | + early_stopping (int): Use this only when num_beams > 1, Defaults to 1 |
| 229 | + """ |
| 230 | + return self.request(prompt, **kwargs) |
0 commit comments