main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cdfbdb28-4f9a-43f9-a016-244e9526806e",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Project Final for ECE 556: AI for Radar and Remote Sensing\n",
    "Paul Delgado"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6f93738a-788b-4ee5-981c-089909ffc29e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python packages\n",
    "import csv\n",
    "import datetime\n",
    "from itertools import product\n",
    "import json\n",
    "import os\n",
    "import pathlib\n",
    "import requests\n",
    "import sys\n",
    "import zipfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5e1d1318-00d2-44ec-8268-add786851e0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Third-party packages\n",
    "#import caffe\n",
    "import mne_bids\n",
    "import numpy as np\n",
    "import openneuro\n",
    "import PIL\n",
    "import scipy\n",
    "from scipy import optimize\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acb9ca94-1bd5-48c6-8d94-443e75fa02ed",
   "metadata": {
    "tags": []
   },
   "source": [
    "### I) Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f09c0e8-ae0e-485b-9acb-7bfab3a8d5b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\n"
     ]
    }
   ],
   "source": [
    "# Data settings\n",
    "results_dirpath_str = os.path.normpath(os.path.join(os.getcwd(), 'results'))\n",
    "print(results_dirpath_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "74a51205-d95b-4802-88ca-db60bbdfca53",
   "metadata": {},
   "outputs": [],
   "source": [
    "subjects_list = ['S1', 'S2', 'S3']\n",
    "rois_list = ['VC']\n",
    "network_name_str = 'VGG19'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "252ed572-9daa-4450-837c-517929c7b1ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Images in figure 2\n",
    "image_type = 'natural'\n",
    "image_label_list = ['Img0002', 'Img0011', 'Img0045', 'Img0048']\n",
    "max_iteration = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a8f0af0e-7cc0-4329-b1ba-ec33fbca3a0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PaulDRP\\source\\repos\\DeepImageReconstruction\n"
     ]
    }
   ],
   "source": [
    "# Data Directory\n",
    "dat_dirpath_str = os.path.normpath(os.path.join(os.getcwd(), '..', 'DeepImageReconstruction'))\n",
    "print(dat_dirpath_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d019618b-a542-47fc-8caf-d1e15e68ee24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PaulDRP\\source\\repos\\DeepImageReconstruction\\data\\decodedfeatures\n"
     ]
    }
   ],
   "source": [
    "decoded_features_dirpath_str = os.path.normpath(os.path.join(dat_dirpath_str, 'data', 'decodedfeatures'))\n",
    "print(decoded_features_dirpath_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e7c6652a-81bf-4ad9-a3e6-e25005df6596",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PaulDRP\\source\\repos\\DeepImageReconstruction\\data\\ilsvrc_2012_mean.npy\n"
     ]
    }
   ],
   "source": [
    "img_mean_fp = os.path.normpath(os.path.join(dat_dirpath_str, 'data', 'ilsvrc_2012_mean.npy'))\n",
    "print(img_mean_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3a5e2f3d-e481-4423-a4e6-704a60a37b2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PaulDRP\\source\\repos\\DeepImageReconstruction\\net\\VGG_ILSVRC_19_layers\\VGG_ILSVRC_19_layers.caffemodel\n",
      "C:\\Users\\PaulDRP\\source\\repos\\DeepImageReconstruction\\net\\VGG_ILSVRC_19_layers\\VGG_ILSVRC_19_layers.prototxt\n"
     ]
    }
   ],
   "source": [
    "# CNN model\n",
    "model_fp = os.path.normpath(os.path.join(dat_dirpath_str, 'net', 'VGG_ILSVRC_19_layers', 'VGG_ILSVRC_19_layers.caffemodel'))\n",
    "prototxt_fp = os.path.normpath(os.path.join(dat_dirpath_str, 'net', 'VGG_ILSVRC_19_layers', 'VGG_ILSVRC_19_layers.prototxt'))\n",
    "channel_swap = (2, 1, 0)\n",
    "print(model_fp)\n",
    "print(prototxt_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0882fe5e-61c0-424a-9ffa-baa81f103ccf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PaulDRP\\source\\repos\\DeepImageReconstruction\\data\\estimated_vgg19_cnn_feat_std.mat\n"
     ]
    }
   ],
   "source": [
    "feat_std_fp = os.path.normpath(os.path.join(dat_dirpath_str, 'data', 'estimated_vgg19_cnn_feat_std.mat'))\n",
    "print(feat_std_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8255bb3e-5b2f-48c0-8986-325812681d35",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'ds001506'\n",
    "subject = '01'\n",
    "session = 'perceptionNaturalImageTraining01'\n",
    "run = 1\n",
    "#task = 'trance'\n",
    "#suffix = 'inplane'\n",
    "datatype = 'anat'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5ee856e4-5532-43d6-bcc9-9605e273c1d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "bids_root_dirpath_str = os.path.normpath(os.path.join(os.getcwd(), '..', dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b69443b-0668-4716-afba-2ee5ac21a6ab",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "55a99139-d77a-4699-9903-8d46f07574af",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### II) Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0892b55c-f6c8-4d35-945f-a2e066db94a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_extreme_value(img, pct=1):\n",
    "    if pct < 0:\n",
    "        pct = 0.\n",
    "\n",
    "    if pct > 100:\n",
    "        pct = 100.\n",
    "\n",
    "    img = np.clip(img, np.percentile(img, pct/2.), np.percentile(img, 100-pct/2.))\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "414cce90-176f-4eae-bb05-c54cf4524e36",
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimate_cnn_feat_std(cnn_feat):\n",
    "    feat_ndim = cnn_feat.ndim\n",
    "    feat_size = cnn_feat.shape\n",
    "    # for the case of fc layers\n",
    "    if feat_ndim == 1 or (feat_ndim == 2 and feat_size[0] == 1) or (feat_ndim == 3 and feat_size[1] == 1 and feat_size[2] == 1):\n",
    "        cnn_feat_std = np.std(cnn_feat)\n",
    "    # for the case of conv layers\n",
    "    elif feat_ndim == 3 and (feat_size[1] > 1 or feat_size[2] > 1):\n",
    "        num_of_ch = feat_size[0]\n",
    "        # std for each channel\n",
    "        cnn_feat_std = np.zeros(num_of_ch, dtype='float32')\n",
    "        for j in range(num_of_ch):\n",
    "            feat_ch = cnn_feat[j, :, :]\n",
    "            cnn_feat_std[j] = np.std(feat_ch)\n",
    "        cnn_feat_std = np.mean(cnn_feat_std)  # std averaged across channels\n",
    "    return cnn_feat_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0fef17ca-70fd-4c01-84c9-ec3b5d7d4074",
   "metadata": {},
   "outputs": [],
   "source": [
    "def L2_loss(feat, feat0, mask=1.):\n",
    "    d = feat - feat0\n",
    "    loss = (d*d*mask).sum()\n",
    "    grad = 2 * d * mask\n",
    "    return loss, grad\n",
    "\n",
    "def L1_loss(feat, feat0, mask=1.):\n",
    "    d = feat - feat0\n",
    "    loss = np.abs(d*mask).sum()\n",
    "    grad = np.sign(d)*mask\n",
    "    return loss, grad\n",
    "\n",
    "def inner_loss(feat, feat0, mask=1.):\n",
    "    loss = -(feat*feat0*mask).sum()\n",
    "    grad = -feat0*mask\n",
    "    return loss, grad\n",
    "\n",
    "def gram(feat, mask=1.):\n",
    "    feat = (feat * mask).reshape(feat.shape[0], -1)\n",
    "    feat_gram = np.dot(feat, feat.T)\n",
    "    return feat_gram\n",
    "\n",
    "def gram_loss(feat, feat0, mask=1.):\n",
    "    feat_size = feat.shape[:]\n",
    "    N = feat_size[0]\n",
    "    M = feat_size[1] * feat_size[2]\n",
    "    feat_gram = gram(feat, mask)\n",
    "    feat0_gram = gram(feat0, mask)\n",
    "    feat = feat.reshape(N, M)\n",
    "    loss = ((feat_gram - feat0_gram)**2).sum() / (4*(N**2)*(M**2))\n",
    "    grad = np.dot((feat_gram - feat0_gram), feat).reshape(feat_size) * mask / ((N**2)*(M**2))\n",
    "    return loss, grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1f9f9b2f-15b5-42ba-9121-acca9daa9897",
   "metadata": {},
   "outputs": [],
   "source": [
    "def switch_loss_fun(loss_type):\n",
    "    if loss_type == 'l2':\n",
    "        return L2_loss\n",
    "    elif loss_type == 'l1':\n",
    "        return L1_loss\n",
    "    elif loss_type == 'inner':\n",
    "        return inner_loss\n",
    "    elif loss_type == 'gram':\n",
    "        return gram_loss\n",
    "    else:\n",
    "        raise ValueError('unknown loss function type!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "54c270cf-2666-463d-8970-fbe1ed2df79d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_img(path_to_img):\n",
    "    max_dim = 512\n",
    "    img = tf.io.read_file(path_to_img)\n",
    "    img = tf.image.decode_image(img, channels=3)\n",
    "    img = tf.image.convert_image_dtype(img, tf.float32)\n",
    "    shape = tf.cast(tf.shape(img)[:-1], tf.float32)\n",
    "    long_dim = max(shape)\n",
    "    scale = max_dim / long_dim\n",
    "    new_shape = tf.cast(shape * scale, tf.int32)\n",
    "    img = tf.image.resize(img, new_shape)\n",
    "    img = img[tf.newaxis, :]\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8ae76bb4-df40-4ace-80cb-9d7fc0f078f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def img_preprocess(img, img_mean=np.float32([104, 117, 123])):\n",
    "    '''convert to Caffe's input image layout'''\n",
    "    return np.float32(np.transpose(img, (2, 0, 1))[::-1]) - np.reshape(img_mean, (3, 1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "93eff75d-622e-4893-aca8-31e0dc8eb89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_feature_masks(features, masks=None, channels=None):\n",
    "    feature_masks = {}\n",
    "    for layer in features.keys():\n",
    "        if (masks is None or masks == {} or masks == [] or (layer not in masks.keys())) and (channels is None or channels == {} or channels == [] or (layer not in channels.keys())):  # use all features and all channels\n",
    "            feature_masks[layer] = np.ones_like(features[layer])\n",
    "        elif isinstance(masks, dict) and (layer in masks.keys()) and isinstance(masks[layer], np.ndarray) and masks[layer].ndim == 3 and masks[layer].shape[0] == features[layer].shape[0] and masks[layer].shape[1] == features[layer].shape[1] and masks[layer].shape[2] == features[layer].shape[2]:  # 3D mask\n",
    "            feature_masks[layer] = masks[layer]\n",
    "        # 1D feat and 1D mask\n",
    "        elif isinstance(masks, dict) and (layer in masks.keys()) and isinstance(masks[layer], np.ndarray) and features[layer].ndim == 1 and masks[layer].ndim == 1 and masks[layer].shape[0] == features[layer].shape[0]:\n",
    "            feature_masks[layer] = masks[layer]\n",
    "        elif (masks is None or masks == {} or masks == [] or (layer not in masks.keys())) and isinstance(channels, dict) and (layer in channels.keys()) and isinstance(channels[layer], np.ndarray) and channels[layer].size > 0:  # select channels\n",
    "            mask_2D = np.ones_like(features[layer][0])\n",
    "            mask_3D = np.tile(mask_2D, [len(channels[layer]), 1, 1])\n",
    "            feature_masks[layer] = np.zeros_like(features[layer])\n",
    "            feature_masks[layer][channels[layer], :, :] = mask_3D\n",
    "        # use 2D mask select features for all channels\n",
    "        elif isinstance(masks, dict) and (layer in masks.keys()) and isinstance(masks[layer], np.ndarray) and masks[layer].ndim == 2 and (channels is None or channels == {} or channels == [] or (layer not in channels.keys())):\n",
    "            mask_2D_0 = masks[layer]\n",
    "            mask_size0 = mask_2D_0.shape\n",
    "            mask_size = features[layer].shape[1:]\n",
    "            if mask_size0[0] == mask_size[0] and mask_size0[1] == mask_size[1]:\n",
    "                mask_2D = mask_2D_0\n",
    "            else:\n",
    "                mask_2D = np.ones(mask_size)\n",
    "                n_dim1 = min(mask_size0[0], mask_size[0])\n",
    "                n_dim2 = min(mask_size0[1], mask_size[1])\n",
    "                idx0_dim1 = np.arange(n_dim1) + \\\n",
    "                    round((mask_size0[0] - n_dim1)/2)\n",
    "                idx0_dim2 = np.arange(n_dim2) + \\\n",
    "                    round((mask_size0[1] - n_dim2)/2)\n",
    "                idx_dim1 = np.arange(n_dim1) + round((mask_size[0] - n_dim1)/2)\n",
    "                idx_dim2 = np.arange(n_dim2) + round((mask_size[1] - n_dim2)/2)\n",
    "                mask_2D[idx_dim1, idx_dim2] = mask_2D_0[idx0_dim1, idx0_dim2]\n",
    "            feature_masks[layer] = np.tile(\n",
    "                mask_2D, [features[layer].shape[0], 1, 1])\n",
    "        else:\n",
    "            feature_masks[layer] = 0\n",
    "\n",
    "    return feature_masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9bbcf8e1-47e7-4ca5-9ccd-a1ae210a5456",
   "metadata": {},
   "outputs": [],
   "source": [
    "def img_deprocess(img, img_mean=np.float32([104, 117, 123])):\n",
    "    new_shape = (3, 1, 1)\n",
    "    return np.dstack((img + np.reshape(img_mean, new_shape))[::-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e5788358-2027-40bc-b34e-d3fc952f838f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def obj_fun(img, net, features, feature_masks, layer_weight, loss_fun, save_intermediate, save_intermediate_every, save_intermediate_path, save_intermediate_ext, save_intermediate_postprocess, loss_list=[]):\n",
    "    # reshape img\n",
    "    img_size = net.blobs['data'].data.shape[-3:]\n",
    "    img = img.reshape(img_size)\n",
    "\n",
    "    # save intermediate image\n",
    "    t = len(loss_list)\n",
    "    if save_intermediate and (t % save_intermediate_every == 0):\n",
    "        img_mean = net.transformer.mean['data']\n",
    "        save_path = os.path.join(save_intermediate_path, '%05d.%s' % (t, save_intermediate_ext))\n",
    "        if save_intermediate_postprocess is None:\n",
    "            snapshot_img = img_deprocess(img, img_mean)\n",
    "        else:\n",
    "            snapshot_img = save_intermediate_postprocess(img_deprocess(img, img_mean))\n",
    "        PIL.Image.fromarray(snapshot_img).save(save_path)\n",
    "\n",
    "    # layer_list\n",
    "    layer_list = features.keys()\n",
    "    layer_list = sort_layer_list(net, layer_list)\n",
    "\n",
    "    # cnn forward\n",
    "    net.blobs['data'].data[0] = img.copy()\n",
    "    net.forward(end=layer_list[-1])\n",
    "\n",
    "    # cnn backward\n",
    "    loss = 0.\n",
    "    layer_start = layer_list[-1]\n",
    "    net.blobs[layer_start].diff.fill(0.)\n",
    "    for j in xrange(len(layer_list)):\n",
    "        layer_start_index = len(layer_list) - 1 - j\n",
    "        layer_end_index = len(layer_list) - 1 - j - 1\n",
    "        layer_start = layer_list[layer_start_index]\n",
    "        if layer_end_index >= 0:\n",
    "            layer_end = layer_list[layer_end_index]\n",
    "        else:\n",
    "            layer_end = 'data'\n",
    "        feat_j = net.blobs[layer_start].data[0].copy()\n",
    "        feat0_j = features[layer_start]\n",
    "        mask_j = feature_masks[layer_start]\n",
    "        layer_weight_j = layer_weight[layer_start]\n",
    "        loss_j, grad_j = loss_fun(feat_j, feat0_j, mask_j)\n",
    "        loss_j = layer_weight_j * loss_j\n",
    "        grad_j = layer_weight_j * grad_j\n",
    "        loss = loss + loss_j\n",
    "        g = net.blobs[layer_start].diff[0].copy()\n",
    "        g = g + grad_j\n",
    "        net.blobs[layer_start].diff[0] = g.copy()\n",
    "        if layer_end == 'data':\n",
    "            net.backward(start=layer_start)\n",
    "        else:\n",
    "            net.backward(start=layer_start, end=layer_end)\n",
    "        net.blobs[layer_start].diff.fill(0.)\n",
    "    grad = net.blobs['data'].diff[0].copy()\n",
    "\n",
    "    # reshape gradient\n",
    "    grad = grad.flatten().astype(np.float64)\n",
    "    loss_list.append(loss)\n",
    "    return loss, grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1aa78a9d-26a4-4e2c-8544-a18fe81cc807",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalise_img(img):\n",
    "    img = img - img.min()\n",
    "    if img.max() > 0:\n",
    "        img = img * (255.0/img.max())\n",
    "    img = np.uint8(img)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c5732c9-89cc-47f7-8482-25cde015f3f1",
   "metadata": {},
   "source": [
    "### Reconstruction Image Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "5b8d8651-61a7-497d-9d9d-a7c7148aeb65",
   "metadata": {},
   "outputs": [],
   "source": [
    "def reconstruct_image(features, net,\n",
    "                      layer_weight=None, \n",
    "                      channel=None, \n",
    "                      mask=None, \n",
    "                      initial_image=None, \n",
    "                      loss_type='l2', \n",
    "                      maxiter=500, \n",
    "                      disp=True, \n",
    "                      save_intermediate=False, \n",
    "                      save_intermediate_every=1, \n",
    "                      save_intermediate_path=None,\n",
    "                      save_intermediate_ext='jpg'):\n",
    "    #print(f'model input {model.input.shape}')\n",
    "    # loss function\n",
    "    loss_fun = switch_loss_fun(loss_type)\n",
    "    #loss_fun = tf.keras.losses.MeanSquaredError()\n",
    "\n",
    "    # make dir for saving intermediate\n",
    "    if save_intermediate:\n",
    "        if save_intermediate_path is None:\n",
    "            fn_temp1 = datetime.datetime.now().strftime('%Y%m%dT%H%M%S')\n",
    "            fn_name = f'recon_img_lbfgs_snapshots_{fn_temp1}'\n",
    "            fp_relative = os.path.join(os.getcwd(), fn_name)\n",
    "            save_intermediate_path = os.path.normpath(fp_relative)\n",
    "        if not os.path.exists(save_intermediate_path):\n",
    "            os.makedirs(save_intermediate_path)\n",
    "\n",
    "    # image size\n",
    "    #img_size = net.blobs['data'].data.shape[-3:]\n",
    "    img_size_tuple = net.input.shape[-3:]\n",
    "    #print(f'image size: {img_size_tuple} {type(img_size_tuple)}')\n",
    "\n",
    "    # num of pixel\n",
    "    num_of_pix = np.prod(img_size_tuple)\n",
    "    \n",
    "    # image mean\n",
    "    #img_mean = net.transformer.mean['data']\n",
    "    img_mean_fn_str = 'ilsvrc_2012_mean.npy'\n",
    "    img_mean_fp_str = os.path.normpath(os.path.join(dat_dirpath_str, 'data', img_mean_fn_str))\n",
    "    if not os.path.exists(img_mean_fp_str):\n",
    "        print(f'ERROR: path does not exist: {img_mean_fp_str}')\n",
    "        exit()\n",
    "    img_mean_ndarray = np.load(img_mean_fp_str)\n",
    "    #img_mean = net.layers[0]\n",
    "    #print(f'image mean shape: {img_mean_ndarray.shape} of {type(img_mean_ndarray)}')\n",
    "    img_mean_list = [img_mean_ndarray[0].mean(), img_mean_ndarray[1].mean(), img_mean_ndarray[2].mean()]\n",
    "    img_mean_ndarray = np.float32(img_mean_list)\n",
    "    \n",
    "    # img bounds\n",
    "    img_min_ndarray = -img_mean_ndarray\n",
    "    img_max_ndarray = -img_mean_ndarray + 255.\n",
    "    \n",
    "    img_min0_tuple = (img_min_ndarray[0], img_max_ndarray[0])\n",
    "    img_min0_list = [img_min0_tuple]\n",
    "    img_min0 = np.array(img_min0_list) * num_of_pix / 3\n",
    "    \n",
    "    img_min1_tuple = (img_min_ndarray[1], img_max_ndarray[1])\n",
    "    img_min1_list = [img_min1_tuple]\n",
    "    img_min1 = np.array(img_min1_list) * num_of_pix / 3\n",
    "    \n",
    "    img_min2_tuple = (img_min_ndarray[2], img_max_ndarray[2])\n",
    "    img_min2_list = [img_min2_tuple]\n",
    "    img_min2 = np.array(img_min2_list) * num_of_pix / 3\n",
    "    \n",
    "    img_bounds = img_min0 + img_min1 + img_min2\n",
    "    \n",
    "    # initial image\n",
    "    if initial_image is None:\n",
    "        init_img_x = (img_size_tuple[0], img_size_tuple[1], img_size_tuple[2])\n",
    "        #print(f'init img x {init_img_x} {type(init_img_x)}')\n",
    "        initial_image = np.random.randint(0, 256, init_img_x)\n",
    "        #print(f'created initial image: {initial_image.shape} {type(initial_image)}')\n",
    "    if save_intermediate:\n",
    "        save_fn_str = 'initial_img.png'\n",
    "        save_fp_str = os.path.join(save_intermediate_path, save_fn_str)\n",
    "        save_fp_str = os.path.normpath(save_fp_str)\n",
    "        if not os.path.exists(save_fp_str):\n",
    "            print(f'ERROR: path does not exist: {save_fp_str}')\n",
    "            exit()\n",
    "        init_pil_img = PIL.Image.fromarray(np.uint8(initial_image))\n",
    "        init_pil_img.save(save_fp_str)\n",
    "\n",
    "    # preprocess initial img\n",
    "    #initial_image = img_preprocess(initial_image, img_mean_ndarray)\n",
    "    #print('preprocessing initial image')\n",
    "    img_init_ndarray = tf.keras.applications.vgg19.preprocess_input(initial_image * img_mean_ndarray)\n",
    "    img_init_ndarray = img_init_ndarray.flatten()\n",
    "    #print('preprocessed initial image')\n",
    "\n",
    "    # layer_list\n",
    "    layer_list = features.keys()\n",
    "    \n",
    "    # layer weight\n",
    "    if layer_weight is None:\n",
    "        weights = np.ones(len(layer_list))\n",
    "        weights = np.float32(weights)\n",
    "        weights = weights / weights.sum()\n",
    "        layer_weight = {}\n",
    "        for j, layer in enumerate(layer_list):\n",
    "            layer_weight[layer] = weights[j]\n",
    "    \n",
    "    # feature mask\n",
    "    feature_masks = create_feature_masks(features, masks=mask, channels=channel)\n",
    "    \n",
    "    # optimization params\n",
    "    loss_list = []\n",
    "    opt_params = {\n",
    "        'args': (net, features, feature_masks, layer_weight, loss_fun, save_intermediate, save_intermediate_every, save_intermediate_path, save_intermediate_ext, loss_list),\n",
    "        'method': 'L-BFGS-B',\n",
    "        'jac': True,\n",
    "        'bounds': img_bounds,\n",
    "        'options': {'maxiter': maxiter, 'disp': disp},\n",
    "    }\n",
    "    \n",
    "    # optimization\n",
    "    #res = optimize.minimize(obj_fun, img_init_ndarray, **opt_params)\n",
    "    #opt = tf.keras.optimizers.SGD()\n",
    "    \n",
    "    # recon img\n",
    "    #img = res.x\n",
    "    img = initial_image.reshape(img_size_tuple)\n",
    "    #img = img_deprocess(img, img_mean)\n",
    "\n",
    "    # return img\n",
    "    return img, loss_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce3487bd-428d-4c9e-908f-6ca43fd38462",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### III) Load Input Files & Create Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "5e52c9cb-b8ce-4826-a2e8-03f72292b477",
   "metadata": {},
   "outputs": [],
   "source": [
    "img_mean = np.load(img_mean_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "963afc76-db86-433a-8b38-e23637de4d58",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.applications.vgg19.VGG19(include_top=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c45ba1da-c39a-4d17-95da-bf041e4af05e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature SD estimated from true CNN features of 10000 images\n",
    "feat_std_dict = scipy.io.loadmat(feat_std_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d6a52c1e-c5fd-4cfe-85a4-020112d7fb3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir(bids_root_dirpath_str):\n",
    "    os.makedirs(bids_root_dirpath_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c371d478-07ff-4562-9d6d-353be0205338",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(results_dirpath_str):\n",
    "    os.makedirs(results_dirpath_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b91d39aa-59e4-497e-a6e3-34abf9a33333",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### IV) Initialize Given Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "113c42a9-a12b-4a88-97a7-733e0d3ceaf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[104.00699 116.66877 122.67892]\n"
     ]
    }
   ],
   "source": [
    "# Average image of ImageNet\n",
    "img_mean_ndarray = np.float32([img_mean[0].mean(), img_mean[1].mean(), img_mean[2].mean()])\n",
    "print(img_mean_ndarray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "71fee6bd-575e-4c63-8f13-1c9672ce5fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial input shape: (None, 224, 224, 3)\n",
      "change input shape:  (None, 224, 224, 3)\n",
      "height: 224 and width: 224\n",
      "model reshaped: (None, 224, 224, 3)\n"
     ]
    }
   ],
   "source": [
    "#net = caffe.Classifier(prototxt_file, model_file, mean=img_mean, channel_swap=channel_swap)\n",
    "print(f'initial input shape: {model.input.shape}')\n",
    "#model.layers[0].shape = (10, 3, 224, 224)\n",
    "print(f'change input shape:  {model.input.shape}')\n",
    "\n",
    "#h, w = net.blobs['data'].data.shape[-2:]\n",
    "h, w = model.input.shape[1:3]\n",
    "print(f'height: {h} and width: {w}')\n",
    "\n",
    "#net.blobs['data'].reshape(1, 3, h, w)\n",
    "model.layers[0].shape = (1, 3, h, w)\n",
    "print(f'model reshaped: {model.input.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "28c492b3-455c-4b61-9c5c-9be696fb16f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial image shape: (224, 224, 3)\n"
     ]
    }
   ],
   "source": [
    "# Initial image for the optimization (here we use the mean of ilsvrc_2012_mean.npy as RGB values)\n",
    "img_init_ndarray = np.zeros((h, w, 3), dtype='float32')\n",
    "img_init_ndarray[:, :, 0] = img_mean_ndarray[2].copy()\n",
    "img_init_ndarray[:, :, 1] = img_mean_ndarray[1].copy()\n",
    "img_init_ndarray[:, :, 2] = img_mean_ndarray[0].copy()\n",
    "print(f'initial image shape: {img_init_ndarray.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5711be66-283f-40cf-9535-61c13efcf91b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer 0: input_1\n",
      "layer 1: block1_conv1 is conv1_1\n",
      "layer 2: block1_conv2 is conv1_2\n",
      "layer 3: block1_pool is pool1\n",
      "layer 4: block2_conv1 is conv2_1\n",
      "layer 5: block2_conv2 is conv2_2\n",
      "layer 6: block2_pool is pool2\n",
      "layer 7: block3_conv1 is conv3_1\n",
      "layer 8: block3_conv2 is conv3_2\n",
      "layer 9: block3_conv3 is conv3_3\n",
      "layer 10: block3_conv4 is conv3_4\n",
      "layer 11: block3_pool is pool3\n",
      "layer 12: block4_conv1 is conv4_1\n",
      "layer 13: block4_conv2 is conv4_2\n",
      "layer 14: block4_conv3 is conv4_3\n",
      "layer 15: block4_conv4 is conv4_4\n",
      "layer 16: block4_pool is pool4\n",
      "layer 17: block5_conv1 is conv5_1\n",
      "layer 18: block5_conv2 is conv5_2\n",
      "layer 19: block5_conv3 is conv5_3\n",
      "layer 20: block5_conv4 is conv5_4\n",
      "layer 21: block5_pool is pool5\n",
      "layer 22: Flatten\n",
      "layer 23: fc1 is fc6\n",
      "layer 24: fc2 is fc7\n",
      "layer 25: predictions is fc8\n"
     ]
    }
   ],
   "source": [
    "# CNN Layers (all conv and fc layers)\n",
    "layers = []\n",
    "count = 0\n",
    "pool_count = 1\n",
    "block_count = 1\n",
    "for layer in model.layers:\n",
    "    if layer.__class__.__name__ == 'InputLayer':\n",
    "        print(f'layer {count}: {layer.name}')\n",
    "    elif layer.__class__.__name__ == 'Conv2D':\n",
    "        sections = layer.name.split('_')\n",
    "        block = sections[0]\n",
    "        blockNum = block[-1]\n",
    "        if int(blockNum) > block_count:\n",
    "            block_count = int(blockNum)\n",
    "        layerNum = sections[1][-1]\n",
    "        val = 'conv' + blockNum + '_' + layerNum\n",
    "        print(f'layer {count}: {layer.name} is {val}')\n",
    "        layers.append(val)\n",
    "    elif layer.__class__.__name__ == 'MaxPooling2D':\n",
    "        sections = layer.name.split('_')\n",
    "        block = sections[0]\n",
    "        blockNum = block[-1]\n",
    "        if int(blockNum) > block_count:\n",
    "            block_count = int(blockNum)\n",
    "        val = 'pool' + blockNum\n",
    "        print(f'layer {count}: {layer.name} is {val}')\n",
    "        layers.append(val)\n",
    "        pool_count = pool_count + 1\n",
    "    elif layer.__class__.__name__ == 'Flatten':\n",
    "        print(f'layer {count}: Flatten')\n",
    "    elif layer.__class__.__name__ == 'Dense':\n",
    "        block_count = block_count + 1\n",
    "        val = 'fc' + str(block_count)\n",
    "        print(f'layer {count}: {layer.name} is {val}')\n",
    "        layers.append(val)\n",
    "    else:\n",
    "        print(f'Error finding layer {count}: {layer.name}')\n",
    "    count = count + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "cf4c3c46-0e22-4671-8c07-198ec02c4d2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss_type': 'l2', 'maxiter': 200, 'initial_image': None, 'disp': True}\n"
     ]
    }
   ],
   "source": [
    "opts_dict = {\n",
    "    'loss_type': 'l2', # The loss function type: {'l2','l1','inner','gram'}\n",
    "    'maxiter': max_iteration, # The maximum number of iterations\n",
    "    'initial_image': None, # (setting to None will use random noise as initial image)\n",
    "    'disp': True # Display the information on the terminal or not\n",
    "}\n",
    "print(opts_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b5296552-9ecf-44d1-9ad1-9d85e22e5717",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"loss_type\": \"l2\", \"maxiter\": 200, \"initial_image\": null, \"disp\": true}\n"
     ]
    }
   ],
   "source": [
    "# Save the optional parameters\n",
    "opts_json_str = json.dumps(opts_dict)\n",
    "print(opts_json_str)\n",
    "opts_fp_str = os.path.normpath(os.path.join(results_dirpath_str, 'options.json'))\n",
    "options_file_handler = open(opts_fp_str, 'w')\n",
    "options_file_handler.write(opts_json_str)\n",
    "options_file_handler.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55165e43-b35e-40c9-badb-e75608993b56",
   "metadata": {},
   "source": [
    "### Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "1a15eb4b-438a-4638-839d-0de5a2042e02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subject: S1 ROI: VC label: Img0002\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S1\\VC\\normalized_recon_img_0002.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S1 ROI: VC label: Img0011\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S1\\VC\\normalized_recon_img_0011.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S1 ROI: VC label: Img0045\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S1\\VC\\normalized_recon_img_0045.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S1 ROI: VC label: Img0048\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S1\\VC\\normalized_recon_img_0048.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S2 ROI: VC label: Img0002\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S2\\VC\\normalized_recon_img_0002.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S2 ROI: VC label: Img0011\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S2\\VC\\normalized_recon_img_0011.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S2 ROI: VC label: Img0045\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S2\\VC\\normalized_recon_img_0045.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S2 ROI: VC label: Img0048\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S2\\VC\\normalized_recon_img_0048.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S3 ROI: VC label: Img0002\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S3\\VC\\normalized_recon_img_0002.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S3 ROI: VC label: Img0011\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S3\\VC\\normalized_recon_img_0011.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S3 ROI: VC label: Img0045\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S3\\VC\\normalized_recon_img_0045.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n",
      "Subject: S3 ROI: VC label: Img0048\n",
      "reconstructed image filepath: C:\\Users\\PaulDRP\\source\\repos\\eeg1\\results\\S3\\VC\\normalized_recon_img_0048.jpg\n",
      "clip recon image: (224, 224, 3)\n",
      "norm recon image: (224, 224, 3)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Reconstrucion\n",
    "for subject, roi, image_label in product(subjects_list, rois_list, image_label_list):\n",
    "    print(f'Subject: {subject} ROI: {roi} label: {image_label}')\n",
    "    \n",
    "    sub_save_dirpath_str = os.path.join(results_dirpath_str, subject, roi)\n",
    "    sub_save_dirpath_str = os.path.normpath(sub_save_dirpath_str)\n",
    "    #print(f'sub save dir: {sub_save_dirpath_str}')\n",
    "    if not os.path.exists(sub_save_dirpath_str):\n",
    "        os.makedirs(sub_save_dirpath_str)\n",
    "    \n",
    "    # Load the decoded CNN features\n",
    "    #print('loading decoded CNN features')\n",
    "    features = {}\n",
    "    for layer in layers:\n",
    "        # The file full name depends on the data structure for decoded CNN features\n",
    "        layer_feat_fn_str = '' # PD: added code to use S1 if layer is pool\n",
    "        layer_feat_fp_str = ''\n",
    "        if layer.startswith('pool'):\n",
    "            layer_feat_fn_str = f'{image_type}-{network_name_str}-{layer}-S1-{roi}-{image_label}.mat'\n",
    "            layer_feat_fp_str = os.path.join(decoded_features_dirpath_str, image_type, network_name_str, layer, 'S1', roi, layer_feat_fn_str)\n",
    "        else:\n",
    "            layer_feat_fn_str = f'{image_type}-{network_name_str}-{layer}-{subject}-{roi}-{image_label}.mat'\n",
    "            layer_feat_fp_str = os.path.join(decoded_features_dirpath_str, image_type, network_name_str, layer, subject, roi, layer_feat_fn_str)\n",
    "        layer_feat_fp_str = os.path.normpath(layer_feat_fp_str)\n",
    "        #print(f'loading layer feature file: {layer_feat_fp_str}')\n",
    "        layer_feat_dict = scipy.io.loadmat(layer_feat_fp_str)\n",
    "        layer_feat_ndarray = layer_feat_dict['feat']\n",
    "        if 'fc' in layer:\n",
    "            layer_feat_ndarray = layer_feat_ndarray.reshape(layer_feat_ndarray.size)\n",
    "        \n",
    "        # Correct the norm of the decoded CNN features\n",
    "        feat_std_npf32 = estimate_cnn_feat_std(layer_feat_ndarray)\n",
    "        layer_feat_std_ndarray = feat_std_dict[layer]\n",
    "        feat_ndarray = (layer_feat_ndarray / feat_std_npf32) * layer_feat_std_ndarray\n",
    "        features.update({layer: feat_ndarray})\n",
    "    #print('loaded decoded CNN features')\n",
    "    \n",
    "    # Weight of each layer in the total loss function\n",
    "    \n",
    "    # Norm of the CNN features for each layer\n",
    "    #print('normalizing features for layers')\n",
    "    feat_list = []\n",
    "    for layer in layers:\n",
    "        feat_list.append(np.linalg.norm(features[layer]))\n",
    "    feat_norm = np.array(feat_list, dtype='float32')\n",
    "    #print('normalized features for layers')\n",
    "    \n",
    "    # Use the inverse of the squared norm of the CNN features as the weight for each layer\n",
    "    weights = 1. / (feat_norm ** 2)\n",
    "    \n",
    "    # Normalise the weights such that the sum of the weights = 1\n",
    "    weights = weights / weights.sum()\n",
    "    layer_weight = dict(zip(layers, weights))\n",
    "    \n",
    "    opts_dict.update({'layer_weight': layer_weight})\n",
    "    \n",
    "    # Reconstruction\n",
    "    snapshots_dir = os.path.join(sub_save_dirpath_str, 'snapshots', f'image-{image_label}')\n",
    "    snapshots_dir = os.path.normpath(snapshots_dir)\n",
    "    #print(f'snapshots dir: {snapshots_dir}')\n",
    "    if not os.path.exists(snapshots_dir):\n",
    "        print(f'ERROR: path does not exist: {snapshots_dir}')\n",
    "        exit()\n",
    "    \n",
    "    recon_img_ndarray, loss_list = reconstruct_image(features, model, save_intermediate=True, save_intermediate_path=snapshots_dir, **opts_dict)\n",
    "    #print(f'recon image: {recon_img_ndarray.shape} {type(recon_img_ndarray)}')\n",
    "    \n",
    "    # Save the results\n",
    "    recon_img_fn_str = f'normalized_recon_img_{image_label[3:]}.jpg'\n",
    "    recon_img_fp_str = os.path.normpath(os.path.join(sub_save_dirpath_str, recon_img_fn_str))\n",
    "    print(f'reconstructed image filepath: {recon_img_fp_str}')\n",
    "    \n",
    "    # To better display the image, clip pixels with extreme values\n",
    "    # (0.02% of pixels with extreme low values and 0.02% of the pixels with extreme high values).\n",
    "    # And then normalise the image by mapping the pixel value to be within [0,255].\n",
    "    recon_img_ndarray = clip_extreme_value(recon_img_ndarray, pct=4)\n",
    "    print(f'clip recon image: {recon_img_ndarray.shape}')\n",
    "    \n",
    "    # Normalize Reconstructed Image\n",
    "    recon_img_ndarray = normalise_img(recon_img_ndarray)\n",
    "    print(f'norm recon image: {recon_img_ndarray.shape}')\n",
    "    \n",
    "    # Create Image\n",
    "    recon_pilimage = PIL.Image.fromarray(np.uint8(recon_img_ndarray))\n",
    "    recon_pilimage.save(recon_img_fp_str)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67eb03e3-85b4-4bfd-a0cf-016ed11333d7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a1612aa-8f2a-46d9-951c-316da2035a0c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "872c37fb-2dc5-4f9b-88ae-7938f0cec914",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "382f12ec-e9a8-40e8-bfe2-faa4a277a627",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1339603-4428-41bf-9ab7-96efbe1b6d3f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9d15a09-be64-4d68-bfbb-e49c5a7fe2fa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}