|
| 1 | +{ |
| 2 | + "tests": [ |
| 3 | + { |
| 4 | + "name": "quick_smoke_test_sft", |
| 5 | + "input": { |
| 6 | + "user_id": "user", |
| 7 | + "model_id": "llama-test", |
| 8 | + "run_id": "llama-test", |
| 9 | + "credentials": { |
| 10 | + "wandb_api_key": "", |
| 11 | + "hf_token": "" |
| 12 | + }, |
| 13 | + "args": { |
| 14 | + "base_model": "HuggingFaceTB/SmolLM2-135M", |
| 15 | + "model_type": "AutoModelForCausalLM", |
| 16 | + "tokenizer_type": "AutoTokenizer", |
| 17 | + "load_in_4bit": true, |
| 18 | + "strict": false, |
| 19 | + "datasets": [ |
| 20 | + { |
| 21 | + "path": "mhenrichsen/alpaca_2k_test", |
| 22 | + "type": "alpaca", |
| 23 | + "split": "train[:10%]" |
| 24 | + } |
| 25 | + ], |
| 26 | + "val_set_size": 0.02, |
| 27 | + "output_dir": "./outputs/lora-out", |
| 28 | + "sequence_len": 4096, |
| 29 | + "sample_packing": true, |
| 30 | + "eval_sample_packing": false, |
| 31 | + "pad_to_sequence_len": true, |
| 32 | + "adapter": "qlora", |
| 33 | + "lora_r": 32, |
| 34 | + "lora_alpha": 64, |
| 35 | + "lora_dropout": 0.05, |
| 36 | + "lora_target_linear": true, |
| 37 | + "lora_modules_to_save": [ |
| 38 | + "embed_tokens", |
| 39 | + "lm_head" |
| 40 | + ], |
| 41 | + "gradient_accumulation_steps": 2, |
| 42 | + "micro_batch_size": 1, |
| 43 | + "num_epochs": 1, |
| 44 | + "optimizer": "adamw_torch_fused", |
| 45 | + "lr_scheduler": "cosine", |
| 46 | + "learning_rate": 0.0002, |
| 47 | + "train_on_inputs": false, |
| 48 | + "group_by_length": false, |
| 49 | + "bf16": "auto", |
| 50 | + "tf32": true, |
| 51 | + "gradient_checkpointing": true, |
| 52 | + "logging_steps": 1, |
| 53 | + "flash_attention": true, |
| 54 | + "warmup_steps": 1, |
| 55 | + "evals_per_epoch": 1, |
| 56 | + "eval_max_new_tokens": 128, |
| 57 | + "saves_per_epoch": 1, |
| 58 | + "weight_decay": 0.0, |
| 59 | + "special_tokens": { |
| 60 | + "pad_token": "<|endoftext|>" |
| 61 | + }, |
| 62 | + "max_steps": 20 |
| 63 | + } |
| 64 | + }, |
| 65 | + "timeout": 100000 |
| 66 | + } |
| 67 | + ], |
| 68 | + "config": { |
| 69 | + "gpuTypeId": "NVIDIA GeForce RTX 4090", |
| 70 | + "gpuCount": 1, |
| 71 | + "containerDiskInGb": 200, |
| 72 | + "env": [ |
| 73 | + { |
| 74 | + "key": "TOKENIZER", |
| 75 | + "value": "" |
| 76 | + }, |
| 77 | + { |
| 78 | + "key": "DISABLE_LOG_STATS", |
| 79 | + "value": "true" |
| 80 | + } |
| 81 | + ], |
| 82 | + "allowedCudaVersions": [ |
| 83 | + "12.8", |
| 84 | + "12.7", |
| 85 | + "12.6", |
| 86 | + "12.5", |
| 87 | + "12.4" |
| 88 | + ] |
| 89 | + } |
| 90 | +} |
0 commit comments