| dtype: float16 | |
| merge_method: passthrough | |
| # Layer Interleaving of SOLAR-10.7B-Instruct-v1.0 (Instruction Fine-tuned SOLAR-10.7B-v1.0) | |
| # where n = 48 (The number of layers the model originally had) | |
| # m = 12 (The number of layers the interleaved into model per row) | |
| # M = 36 = 3m (The number of layers interlaced into the model) | |
| # s = 84 = n + M (The number of layers the model will have after the layer interleaving) | |
| # P = ~19.2B = 19.17B parameters = (s/n)*10.7B (The number of parameters the model will have after the DUS and layer interleaving) | |
| slices: | |
| - sources: | |
| - layer_range: [0, 12] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |
| # Interleaved Layer | |
| - sources: | |
| - layer_range: [6, 18] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |
| - sources: | |
| - layer_range: [12, 24] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |
| # Interleaved Layer | |
| - sources: | |
| - layer_range: [18, 30] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |
| - sources: | |
| - layer_range: [24, 36] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |
| # Interleaved Layer | |
| - sources: | |
| - layer_range: [30, 42] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |
| - sources: | |
| - layer_range: [36, 48] | |
| model: /Users/jsarnecki/opt/upstage/SOLAR-10.7B-Instruct-v1.0 | |