Skip to content

Commit c9f17d5

Browse files
ayasyrevayasyrev
authored
Merge pull request #118 from ayasyrev:dev
0.4.1
2 parents ab1abbc + 81e14ab commit c9f17d5

22 files changed

+529
-235
lines changed

‎Nbs/00_ModelConstructor.ipynb renamed to ‎Nbs/01_ModelConstructor.ipynb

+15-7
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,21 @@
8484
"name": "stdout",
8585
"output_type": "stream",
8686
"text": [
87-
"ModelCfg\n",
88-
" in_chans: 3, num_classes: 1000\n",
89-
" expansion: 1, groups: 1, dw: False, div_groups: None\n",
90-
" act_fn: ReLU, sa: False, se: False\n",
91-
" stem sizes: [64], stride on 0\n",
92-
" body sizes [64, 128, 256, 512]\n",
93-
" layers: [2, 2, 2, 2]\n"
87+
"ModelCfg(\n",
88+
" in_chans=3\n",
89+
" num_classes=1000\n",
90+
" block='BasicBlock'\n",
91+
" conv_layer='ConvBnAct'\n",
92+
" block_sizes=[64, 128, 256, 512]\n",
93+
" layers=[2, 2, 2, 2]\n",
94+
" norm='BatchNorm2d'\n",
95+
" act_fn='ReLU'\n",
96+
" expansion=1\n",
97+
" groups=1\n",
98+
" bn_1st=True\n",
99+
" zero_bn=True\n",
100+
" stem_sizes=[64]\n",
101+
" stem_pool=\"MaxPool2d {'kernel_size': 3, 'stride': 2, 'padding': 1}\")\n"
94102
]
95103
}
96104
],

‎Nbs/001_Blocks.ipynb renamed to ‎Nbs/02_1_Blocks.ipynb

-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
"outputs": [],
2828
"source": [
2929
"#hide\n",
30-
"import torch\n",
3130
"import torch.nn as nn\n",
3231
"from functools import partial"
3332
]

‎Nbs/01_layers.ipynb renamed to ‎Nbs/02_2_layers.ipynb

File renamed without changes.

‎Nbs/02_XResNet.ipynb

+8-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
"source": [
1818
"#hide\n",
1919
"import torch\n",
20-
"from typing import List\n",
2120
"\n",
2221
"from functools import partial\n",
2322
"\n",
@@ -98,6 +97,14 @@
9897
"execution_count": null,
9998
"metadata": {},
10099
"outputs": [
100+
{
101+
"name": "stderr",
102+
"output_type": "stream",
103+
"text": [
104+
"/home/aya/mambaforge/envs/mc/lib/python3.10/site-packages/torch/nn/modules/conv.py:137: UserWarning: Failed to initialize NumPy: No module named 'numpy' (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:84.)\n",
105+
" self.weight = Parameter(torch.empty(\n"
106+
]
107+
},
101108
{
102109
"data": {
103110
"text/plain": [

‎Nbs/03_MXResNet.ipynb

+30-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
"outputs": [],
2424
"source": [
2525
"#hide\n",
26-
"from typing import List\n",
2726
"from functools import partial\n",
2827
"\n",
2928
"import torch\n",
@@ -106,6 +105,14 @@
106105
"execution_count": null,
107106
"metadata": {},
108107
"outputs": [
108+
{
109+
"name": "stderr",
110+
"output_type": "stream",
111+
"text": [
112+
"/home/aya/mambaforge/envs/mc/lib/python3.10/site-packages/torch/nn/modules/conv.py:137: UserWarning: Failed to initialize NumPy: No module named 'numpy' (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:84.)\n",
113+
" self.weight = Parameter(torch.empty(\n"
114+
]
115+
},
109116
{
110117
"data": {
111118
"text/plain": [
@@ -664,6 +671,28 @@
664671
"model = MxResNet34.create_model()"
665672
]
666673
},
674+
{
675+
"cell_type": "code",
676+
"execution_count": null,
677+
"metadata": {},
678+
"outputs": [
679+
{
680+
"name": "stdout",
681+
"output_type": "stream",
682+
"text": [
683+
"torch.Size([16, 1000])\n"
684+
]
685+
}
686+
],
687+
"source": [
688+
"#hide\n",
689+
"bs_test = 16\n",
690+
"xb = torch.randn(bs_test, 3, 128, 128)\n",
691+
"y = model(xb)\n",
692+
"print(y.shape)\n",
693+
"assert y.shape == torch.Size([bs_test, 1000]), f\"size\""
694+
]
695+
},
667696
{
668697
"cell_type": "markdown",
669698
"metadata": {},

‎Nbs/04_YaResNet.ipynb

+16-14
Original file line numberDiff line numberDiff line change
@@ -443,39 +443,39 @@
443443
{
444444
"data": {
445445
"text/plain": [
446-
"YaResBlock(\n",
446+
"YaBottleneckBlock(\n",
447447
" (reduce): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
448448
" (convs): Sequential(\n",
449449
" (conv_0): ConvBnAct(\n",
450-
" (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
451-
" (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
450+
" (conv): Conv2d(64, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
451+
" (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
452452
" (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n",
453453
" )\n",
454454
" (conv_1): ConvBnAct(\n",
455-
" (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128, bias=False)\n",
456-
" (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
455+
" (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)\n",
456+
" (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
457457
" (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n",
458458
" )\n",
459459
" (conv_2): ConvBnAct(\n",
460-
" (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
461-
" (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
460+
" (conv): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
461+
" (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
462462
" )\n",
463463
" (se): SEModule(\n",
464464
" (squeeze): AdaptiveAvgPool2d(output_size=1)\n",
465465
" (excitation): Sequential(\n",
466-
" (reduce): Linear(in_features=512, out_features=32, bias=True)\n",
466+
" (reduce): Linear(in_features=128, out_features=8, bias=True)\n",
467467
" (se_act): ReLU(inplace=True)\n",
468-
" (expand): Linear(in_features=32, out_features=512, bias=True)\n",
468+
" (expand): Linear(in_features=8, out_features=128, bias=True)\n",
469469
" (se_gate): Sigmoid()\n",
470470
" )\n",
471471
" )\n",
472472
" (sa): SimpleSelfAttention(\n",
473-
" (conv): Conv1d(512, 512, kernel_size=(1,), stride=(1,), bias=False)\n",
473+
" (conv): Conv1d(128, 128, kernel_size=(1,), stride=(1,), bias=False)\n",
474474
" )\n",
475475
" )\n",
476476
" (id_conv): ConvBnAct(\n",
477-
" (conv): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
478-
" (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
477+
" (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
478+
" (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
479479
" )\n",
480480
" (merge): LeakyReLU(negative_slope=0.01, inplace=True)\n",
481481
")"
@@ -488,8 +488,10 @@
488488
],
489489
"source": [
490490
"#collapse_output\n",
491-
"bl = YaResBlock(\n",
492-
" 4, 64, 128,\n",
491+
"bl = YaBottleneckBlock(\n",
492+
" 64,\n",
493+
" 128,\n",
494+
" expansion=4,\n",
493495
" stride=2,\n",
494496
" pool=pool,\n",
495497
" act_fn=nn.LeakyReLU,\n",

‎Nbs/06_ConvMixer.ipynb

+8
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,14 @@
653653
"#collapse_output\n",
654654
"convmixer_1024_20[1]"
655655
]
656+
},
657+
{
658+
"cell_type": "code",
659+
"execution_count": null,
660+
"id": "dacb4659",
661+
"metadata": {},
662+
"outputs": [],
663+
"source": []
656664
}
657665
],
658666
"metadata": {

‎Nbs/README.ipynb

+110-32
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
"outputs": [],
1919
"source": [
2020
"#hide\n",
21-
"import torch\n",
2221
"from torch import nn"
2322
]
2423
},
@@ -143,13 +142,26 @@
143142
"name": "stdout",
144143
"output_type": "stream",
145144
"text": [
146-
"ModelConstructor\n",
147-
" in_chans: 3, num_classes: 1000\n",
148-
" expansion: 1, groups: 1, dw: False, div_groups: None\n",
149-
" act_fn: ReLU, sa: False, se: False\n",
150-
" stem sizes: [64], stride on 0\n",
151-
" body sizes [64, 128, 256, 512]\n",
152-
" layers: [2, 2, 2, 2]\n"
145+
"ModelConstructor(\n",
146+
" in_chans=3\n",
147+
" num_classes=1000\n",
148+
" block='BasicBlock'\n",
149+
" conv_layer='ConvBnAct'\n",
150+
" block_sizes=[64, 128, 256, 512]\n",
151+
" layers=[2, 2, 2, 2]\n",
152+
" norm='BatchNorm2d'\n",
153+
" act_fn='ReLU'\n",
154+
" expansion=1\n",
155+
" groups=1\n",
156+
" bn_1st=True\n",
157+
" zero_bn=True\n",
158+
" stem_sizes=[64]\n",
159+
" stem_pool=\"MaxPool2d {'kernel_size': 3, 'stride': 2, 'padding': 1}\"\n",
160+
" init_cnn='init_cnn'\n",
161+
" make_stem='make_stem'\n",
162+
" make_layer='make_layer'\n",
163+
" make_body='make_body'\n",
164+
" make_head='make_head')\n"
153165
]
154166
}
155167
],
@@ -404,6 +416,32 @@
404416
"mc.print_changed_fields()"
405417
]
406418
},
419+
{
420+
"cell_type": "markdown",
421+
"metadata": {},
422+
"source": [
423+
"We can compare changed with defaults."
424+
]
425+
},
426+
{
427+
"cell_type": "code",
428+
"execution_count": null,
429+
"metadata": {},
430+
"outputs": [
431+
{
432+
"name": "stdout",
433+
"output_type": "stream",
434+
"text": [
435+
"Changed fields:\n",
436+
"layers: [3, 4, 6, 3] | [2, 2, 2, 2]\n",
437+
"expansion: 4 | 1\n"
438+
]
439+
}
440+
],
441+
"source": [
442+
"mc.print_changed_fields(show_default=True)"
443+
]
444+
},
407445
{
408446
"cell_type": "markdown",
409447
"metadata": {},
@@ -715,7 +753,21 @@
715753
"name": "stdout",
716754
"output_type": "stream",
717755
"text": [
718-
"in_chans=3 num_classes=10 block='BasicBlock' conv_layer='ConvBnAct' block_sizes=[64, 128, 256, 512] layers=[2, 2, 2, 2] norm='BatchNorm2d' act_fn='Mish' expansion=1 groups=1 bn_1st=True zero_bn=True stem_sizes=[64] stem_pool=\"MaxPool2d {'kernel_size': 3, 'stride': 2, 'padding': 1}\"\n"
756+
"ModelCfg(\n",
757+
" in_chans=3\n",
758+
" num_classes=10\n",
759+
" block='BasicBlock'\n",
760+
" conv_layer='ConvBnAct'\n",
761+
" block_sizes=[64, 128, 256, 512]\n",
762+
" layers=[2, 2, 2, 2]\n",
763+
" norm='BatchNorm2d'\n",
764+
" act_fn='Mish'\n",
765+
" expansion=1\n",
766+
" groups=1\n",
767+
" bn_1st=True\n",
768+
" zero_bn=True\n",
769+
" stem_sizes=[64]\n",
770+
" stem_pool=\"MaxPool2d {'kernel_size': 3, 'stride': 2, 'padding': 1}\")\n"
719771
]
720772
}
721773
],
@@ -767,21 +819,13 @@
767819
"execution_count": null,
768820
"metadata": {},
769821
"outputs": [
770-
{
771-
"name": "stdout",
772-
"output_type": "stream",
773-
"text": [
774-
"Deprecated. Pass se_module as se argument, se_reduction as arg to se.\n",
775-
"Deprecated. Pass se_module as se argument, se_reduction as arg to se.\n"
776-
]
777-
},
778822
{
779823
"data": {
780824
"text/plain": [
781825
"ModelConstructor\n",
782826
" in_chans: 3, num_classes: 10\n",
783827
" expansion: 1, groups: 1, dw: False, div_groups: None\n",
784-
" act_fn: ReLU, sa: <class 'model_constructor.layers.SimpleSelfAttention'>, se: SEModule\n",
828+
" act_fn: SELU, sa: <class 'model_constructor.layers.SimpleSelfAttention'>, se: SEModule\n",
785829
" stem sizes: [64], stride on 0\n",
786830
" body sizes [64, 128, 256, 512]\n",
787831
" layers: [2, 2, 2, 2]"
@@ -1383,13 +1427,27 @@
13831427
"name": "stdout",
13841428
"output_type": "stream",
13851429
"text": [
1386-
"YaResNet\n",
1387-
" in_chans: 3, num_classes: 1000\n",
1388-
" expansion: 1, groups: 1, dw: False, div_groups: None\n",
1389-
" act_fn: ReLU, sa: False, se: False\n",
1390-
" stem sizes: [64], stride on 0\n",
1391-
" body sizes [64, 128, 256, 512]\n",
1392-
" layers: [2, 2, 2, 2]\n"
1430+
"ModelConstructor(\n",
1431+
" name='YaResNet'\n",
1432+
" in_chans=3\n",
1433+
" num_classes=1000\n",
1434+
" block='YaBasicBlock'\n",
1435+
" conv_layer='ConvBnAct'\n",
1436+
" block_sizes=[64, 128, 256, 512]\n",
1437+
" layers=[2, 2, 2, 2]\n",
1438+
" norm='BatchNorm2d'\n",
1439+
" act_fn='ReLU'\n",
1440+
" expansion=1\n",
1441+
" groups=1\n",
1442+
" bn_1st=True\n",
1443+
" zero_bn=True\n",
1444+
" stem_sizes=[64]\n",
1445+
" stem_pool=\"MaxPool2d {'kernel_size': 3, 'stride': 2, 'padding': 1}\"\n",
1446+
" init_cnn='init_cnn'\n",
1447+
" make_stem='make_stem'\n",
1448+
" make_layer='make_layer'\n",
1449+
" make_body='make_body'\n",
1450+
" make_head='make_head')\n"
13931451
]
13941452
}
13951453
],
@@ -1482,13 +1540,26 @@
14821540
"name": "stdout",
14831541
"output_type": "stream",
14841542
"text": [
1485-
"YaResnet34\n",
1486-
" in_chans: 3, num_classes: 1000\n",
1487-
" expansion: 1, groups: 1, dw: False, div_groups: None\n",
1488-
" act_fn: ReLU, sa: False, se: False\n",
1489-
" stem sizes: [64], stride on 0\n",
1490-
" body sizes [64, 128, 256, 512]\n",
1491-
" layers: [3, 4, 6, 3]\n"
1543+
"YaResnet34(\n",
1544+
" in_chans=3\n",
1545+
" num_classes=1000\n",
1546+
" block='YaBasicBlock'\n",
1547+
" conv_layer='ConvBnAct'\n",
1548+
" block_sizes=[64, 128, 256, 512]\n",
1549+
" layers=[3, 4, 6, 3]\n",
1550+
" norm='BatchNorm2d'\n",
1551+
" act_fn='ReLU'\n",
1552+
" expansion=1\n",
1553+
" groups=1\n",
1554+
" bn_1st=True\n",
1555+
" zero_bn=True\n",
1556+
" stem_sizes=[64]\n",
1557+
" stem_pool=\"MaxPool2d {'kernel_size': 3, 'stride': 2, 'padding': 1}\"\n",
1558+
" init_cnn='init_cnn'\n",
1559+
" make_stem='xresnet_stem'\n",
1560+
" make_layer='make_layer'\n",
1561+
" make_body='make_body'\n",
1562+
" make_head='make_head')\n"
14921563
]
14931564
}
14941565
],
@@ -1541,6 +1612,13 @@
15411612
"mc = YaResnet50()\n",
15421613
"mc"
15431614
]
1615+
},
1616+
{
1617+
"cell_type": "code",
1618+
"execution_count": null,
1619+
"metadata": {},
1620+
"outputs": [],
1621+
"source": []
15441622
}
15451623
],
15461624
"metadata": {

0 commit comments

Comments
 (0)