File tree Expand file tree Collapse file tree 1 file changed +2
-3
lines changed Expand file tree Collapse file tree 1 file changed +2
-3
lines changed Original file line number Diff line number Diff line change @@ -204,12 +204,11 @@ def _create_prompt(
204204
205205        left  =  start_index 
206206        right  =  start_index  +  4  *  prompt_tokens 
207-         start_tokens  =  list [int ]
208207
209208        while  left  <  right :
210209            mid  =  (left  +  right ) //  2 
211210            test_prompt  =  self .text_creator .create_text (start_index , mid  -  start_index )
212-             test_tokens  =  start_tokens   +   self .processor .encode (test_prompt )
211+             test_tokens  =  self .processor .encode (test_prompt )
213212
214213            if  len (test_tokens ) ==  prompt_tokens :
215214                return  test_tokens 
@@ -219,7 +218,7 @@ def _create_prompt(
219218                right  =  mid 
220219
221220        final_text  =  self .text_creator .create_text (start_index , left  -  start_index )
222-         return  start_tokens   +   self .processor .encode (final_text )
221+         return  self .processor .encode (final_text )
223222
224223
225224class  SyntheticDatasetCreator (DatasetCreator ):
 
 
   
 
     
   
   
          
    
    
     
    
      
     
     
    You can’t perform that action at this time.
  
 
    
  
    
      
        
     
       
      
     
   
 
    
    
  
 
  
 
     
    
0 commit comments