@@ -49,6 +49,7 @@ def __init__(self, **kwargs):
4949 self .use_sdpa_with_kv_cache_op = kwargs .get ("use_sdpa_with_kv_cache" , False )
5050 self .generate_full_logits = kwargs .get ("generate_full_logits" , False )
5151 self .enable_dynamic_shape = kwargs .get ("enable_dynamic_shape" , False )
52+ self .input_prune_map_path = kwargs .get ("input_prune_map_path" , None )
5253 self .output_prune_map_path = kwargs .get ("output_prune_map_path" , None )
5354 self .max_seq_len = kwargs .get ("max_seq_len" , 128 )
5455 self .args = kwargs .get ("args" , None )
@@ -126,13 +127,20 @@ def __init__(self, **kwargs):
126127 output_prune_map = json .load (f )
127128 # Change keys from string to int (json only supports string keys).
128129 output_prune_map = {int (k ): v for (k , v ) in output_prune_map .items ()}
130+ input_prune_map = None
131+ if self .input_prune_map_path is not None :
132+ with open (self .input_prune_map_path , "r" ) as f :
133+ input_prune_map = json .load (f )
134+ # Change keys from string to int (json only supports string keys).
135+ input_prune_map = {int (k ): v for (k , v ) in input_prune_map .items ()}
129136
130137 model_args : ModelArgs = ModelArgs (
131138 max_seq_len = self .max_seq_len ,
132139 max_batch_size = 1 ,
133140 use_kv_cache = self .use_kv_cache ,
134141 use_sdpa_with_kv_cache_op = self .use_sdpa_with_kv_cache_op ,
135142 generate_full_logits = self .generate_full_logits ,
143+ input_prune_map = input_prune_map ,
136144 output_prune_map = output_prune_map ,
137145 enable_dynamic_shape = self .enable_dynamic_shape ,
138146 ** params ,
@@ -209,9 +217,15 @@ def __init__(self, **kwargs):
209217 print (unexpected )
210218 print ("============= /unexpected ================" )
211219
220+ # Prune the input layer if input_prune_map is provided
221+ if input_prune_map is not None :
222+ from .source_transformation .prune_vocab import prune_input_vocab
223+
224+ self .model_ = prune_input_vocab (self .model_ , input_prune_map )
225+
212226 # Prune the output layer if output_prune_map is provided
213227 if output_prune_map is not None :
214- from .source_transformation .prune_output import prune_output_vocab
228+ from .source_transformation .prune_vocab import prune_output_vocab
215229
216230 self .model_ = prune_output_vocab (self .model_ , output_prune_map )
217231
0 commit comments