S&P_500_Pairwise_Correlations.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOzF1AEQLeZzmhBFlvL5k8Z",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Farmhouse121/Adventures-in-Financial-Data-Science/blob/main/S%26P_500_Pairwise_Correlations.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sX5OetCL3ZeG",
        "outputId": "a2fb5f37-cea6-46cd-feea-0250c88fedf0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "17:23:41 Starting...\n",
            "17:23:41 Installing yfinance into Google notebook...\n",
            "17:23:47 Installing arch into Google notebook...\n",
            "17:23:57 Initialized.\n"
          ]
        }
      ],
      "source": [
        "from my_library import *"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "display(sp500:=pd.read_html('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "EvIyDKmM3i3h",
        "outputId": "f6ca0702-b76a-46c5-d153-b628ccb48c32"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "    Symbol            Security             GICS Sector  \\\n",
              "0      MMM                  3M             Industrials   \n",
              "1      AOS         A. O. Smith             Industrials   \n",
              "2      ABT              Abbott             Health Care   \n",
              "3     ABBV              AbbVie             Health Care   \n",
              "4      ACN           Accenture  Information Technology   \n",
              "..     ...                 ...                     ...   \n",
              "498    XYL          Xylem Inc.             Industrials   \n",
              "499    YUM         Yum! Brands  Consumer Discretionary   \n",
              "500   ZBRA  Zebra Technologies  Information Technology   \n",
              "501    ZBH       Zimmer Biomet             Health Care   \n",
              "502    ZTS              Zoetis             Health Care   \n",
              "\n",
              "                                GICS Sub-Industry    Headquarters Location  \\\n",
              "0                        Industrial Conglomerates    Saint Paul, Minnesota   \n",
              "1                               Building Products     Milwaukee, Wisconsin   \n",
              "2                           Health Care Equipment  North Chicago, Illinois   \n",
              "3                                   Biotechnology  North Chicago, Illinois   \n",
              "4                  IT Consulting & Other Services          Dublin, Ireland   \n",
              "..                                            ...                      ...   \n",
              "498  Industrial Machinery & Supplies & Components   White Plains, New York   \n",
              "499                                   Restaurants     Louisville, Kentucky   \n",
              "500            Electronic Equipment & Instruments   Lincolnshire, Illinois   \n",
              "501                         Health Care Equipment          Warsaw, Indiana   \n",
              "502                               Pharmaceuticals   Parsippany, New Jersey   \n",
              "\n",
              "     Date added      CIK      Founded  \n",
              "0    1957-03-04    66740         1902  \n",
              "1    2017-07-26    91142         1916  \n",
              "2    1957-03-04     1800         1888  \n",
              "3    2012-12-31  1551152  2013 (1888)  \n",
              "4    2011-07-06  1467373         1989  \n",
              "..          ...      ...          ...  \n",
              "498  2011-11-01  1524472         2011  \n",
              "499  1997-10-06  1041061         1997  \n",
              "500  2019-12-23   877212         1969  \n",
              "501  2001-08-07  1136869         1927  \n",
              "502  2013-06-21  1555280         1952  \n",
              "\n",
              "[503 rows x 8 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-802b462a-c026-4494-8828-5016328394b6\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Symbol</th>\n",
              "      <th>Security</th>\n",
              "      <th>GICS Sector</th>\n",
              "      <th>GICS Sub-Industry</th>\n",
              "      <th>Headquarters Location</th>\n",
              "      <th>Date added</th>\n",
              "      <th>CIK</th>\n",
              "      <th>Founded</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>MMM</td>\n",
              "      <td>3M</td>\n",
              "      <td>Industrials</td>\n",
              "      <td>Industrial Conglomerates</td>\n",
              "      <td>Saint Paul, Minnesota</td>\n",
              "      <td>1957-03-04</td>\n",
              "      <td>66740</td>\n",
              "      <td>1902</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>AOS</td>\n",
              "      <td>A. O. Smith</td>\n",
              "      <td>Industrials</td>\n",
              "      <td>Building Products</td>\n",
              "      <td>Milwaukee, Wisconsin</td>\n",
              "      <td>2017-07-26</td>\n",
              "      <td>91142</td>\n",
              "      <td>1916</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ABT</td>\n",
              "      <td>Abbott</td>\n",
              "      <td>Health Care</td>\n",
              "      <td>Health Care Equipment</td>\n",
              "      <td>North Chicago, Illinois</td>\n",
              "      <td>1957-03-04</td>\n",
              "      <td>1800</td>\n",
              "      <td>1888</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ABBV</td>\n",
              "      <td>AbbVie</td>\n",
              "      <td>Health Care</td>\n",
              "      <td>Biotechnology</td>\n",
              "      <td>North Chicago, Illinois</td>\n",
              "      <td>2012-12-31</td>\n",
              "      <td>1551152</td>\n",
              "      <td>2013 (1888)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ACN</td>\n",
              "      <td>Accenture</td>\n",
              "      <td>Information Technology</td>\n",
              "      <td>IT Consulting &amp; Other Services</td>\n",
              "      <td>Dublin, Ireland</td>\n",
              "      <td>2011-07-06</td>\n",
              "      <td>1467373</td>\n",
              "      <td>1989</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>498</th>\n",
              "      <td>XYL</td>\n",
              "      <td>Xylem Inc.</td>\n",
              "      <td>Industrials</td>\n",
              "      <td>Industrial Machinery &amp; Supplies &amp; Components</td>\n",
              "      <td>White Plains, New York</td>\n",
              "      <td>2011-11-01</td>\n",
              "      <td>1524472</td>\n",
              "      <td>2011</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499</th>\n",
              "      <td>YUM</td>\n",
              "      <td>Yum! Brands</td>\n",
              "      <td>Consumer Discretionary</td>\n",
              "      <td>Restaurants</td>\n",
              "      <td>Louisville, Kentucky</td>\n",
              "      <td>1997-10-06</td>\n",
              "      <td>1041061</td>\n",
              "      <td>1997</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>500</th>\n",
              "      <td>ZBRA</td>\n",
              "      <td>Zebra Technologies</td>\n",
              "      <td>Information Technology</td>\n",
              "      <td>Electronic Equipment &amp; Instruments</td>\n",
              "      <td>Lincolnshire, Illinois</td>\n",
              "      <td>2019-12-23</td>\n",
              "      <td>877212</td>\n",
              "      <td>1969</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>501</th>\n",
              "      <td>ZBH</td>\n",
              "      <td>Zimmer Biomet</td>\n",
              "      <td>Health Care</td>\n",
              "      <td>Health Care Equipment</td>\n",
              "      <td>Warsaw, Indiana</td>\n",
              "      <td>2001-08-07</td>\n",
              "      <td>1136869</td>\n",
              "      <td>1927</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>502</th>\n",
              "      <td>ZTS</td>\n",
              "      <td>Zoetis</td>\n",
              "      <td>Health Care</td>\n",
              "      <td>Pharmaceuticals</td>\n",
              "      <td>Parsippany, New Jersey</td>\n",
              "      <td>2013-06-21</td>\n",
              "      <td>1555280</td>\n",
              "      <td>1952</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>503 rows × 8 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-802b462a-c026-4494-8828-5016328394b6')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-802b462a-c026-4494-8828-5016328394b6 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-802b462a-c026-4494-8828-5016328394b6');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ef8abb94-2d1c-43a4-94cd-f750325577ad\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ef8abb94-2d1c-43a4-94cd-f750325577ad')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ef8abb94-2d1c-43a4-94cd-f750325577ad button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_cf993c1f-ca9d-45fa-b439-062e08fd9f29\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('sp500')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_cf993c1f-ca9d-45fa-b439-062e08fd9f29 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('sp500');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "sp500",
              "summary": "{\n  \"name\": \"sp500\",\n  \"rows\": 503,\n  \"fields\": [\n    {\n      \"column\": \"Symbol\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 503,\n        \"samples\": [\n          \"JCI\",\n          \"BMY\",\n          \"LEN\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Security\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 503,\n        \"samples\": [\n          \"Johnson Controls\",\n          \"Bristol Myers Squibb\",\n          \"Lennar\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"GICS Sector\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 11,\n        \"samples\": [\n          \"Materials\",\n          \"Industrials\",\n          \"Consumer Staples\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"GICS Sub-Industry\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 127,\n        \"samples\": [\n          \"Telecom Tower REITs\",\n          \"Oil & Gas Refining & Marketing\",\n          \"Homebuilding\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Headquarters Location\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 252,\n        \"samples\": [\n          \"Chicago, Illinois; Pittsburgh, Pennsylvania\",\n          \"Arlington, Virginia\",\n          \"Redwood City, California\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Date added\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"num_unique_values\": 376,\n        \"samples\": [\n          \"1984-04-30\",\n          \"1999-06-30\",\n          \"2015-09-18\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"CIK\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 566520,\n        \"min\": 1800,\n        \"max\": 2005951,\n        \"num_unique_values\": 500,\n        \"samples\": [\n          813828,\n          1730168,\n          945841\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Founded\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 199,\n        \"samples\": [\n          \"1848\",\n          \"1908\",\n          \"1909\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "results=pd.DataFrame({\"Sample\":list(range(N:=10))}).set_index(\"Sample\")\n",
        "\n",
        "for i in results.index:\n",
        "    df=download(list(sp500.sample(2,axis=0)['Symbol'])).dropna()\n",
        "\n",
        "    if df.shape[0]<10:\n",
        "        continue\n",
        "\n",
        "    for ticker in (tickers:=list(set(df.columns.get_level_values(1)))):\n",
        "        df[('Return',ticker)]=df[('Adj Close',ticker)].pct_change()*hundred\n",
        "\n",
        "    df.dropna(inplace=True)\n",
        "    df.index=pd.DatetimeIndex(df.index).to_period('D')\n",
        "\n",
        "    for j in range(2):\n",
        "        results.loc[i,j]=tickers[j]\n",
        "\n",
        "    results.loc[i,\"Correlation\"]=df.corr().iloc[0,1]*hundred\n",
        "    results.loc[i,\"Sample\"]=df.shape[0]\n",
        "\n",
        "results.set_index([0,1],inplace=True)\n",
        "results.index.names=[\"Ticker 1\",\"Ticker 2\"]\n",
        "results.loc[\"Mean\",\"Sample\"]=results.shape[0]\n",
        "results.loc[\"Mean\",\"Correlation\"]=results[\"Correlation\"].mean()\n",
        "display(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        },
        "id": "Zh7Xd2Rb4f0c",
        "outputId": "7962109e-2951-499a-f5fb-583de37c8fca"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n",
            "[*********************100%%**********************]  2 of 2 completed\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   Correlation   Sample\n",
              "Ticker 1 Ticker 2                      \n",
              "AEP      ES          98.515296  12977.0\n",
              "CSCO     WAT         87.878976   7227.0\n",
              "NWS      PPL         35.079496   2802.0\n",
              "EG       DOC         74.965680   7260.0\n",
              "FRT      INVH        24.072846   1890.0\n",
              "EA       MTD         89.050317   6723.0\n",
              "NTAP     BWA         60.786732   7225.0\n",
              "JNJ      DAL         76.235856   4344.0\n",
              "ADBE     TFC         80.383119   9571.0\n",
              "VLTO     CRM         34.028350    211.0\n",
              "Mean                 66.099667     10.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5a785614-ed3f-4528-ae46-76404bb50e0a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>Correlation</th>\n",
              "      <th>Sample</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Ticker 1</th>\n",
              "      <th>Ticker 2</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>AEP</th>\n",
              "      <th>ES</th>\n",
              "      <td>98.515296</td>\n",
              "      <td>12977.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CSCO</th>\n",
              "      <th>WAT</th>\n",
              "      <td>87.878976</td>\n",
              "      <td>7227.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NWS</th>\n",
              "      <th>PPL</th>\n",
              "      <td>35.079496</td>\n",
              "      <td>2802.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>EG</th>\n",
              "      <th>DOC</th>\n",
              "      <td>74.965680</td>\n",
              "      <td>7260.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>FRT</th>\n",
              "      <th>INVH</th>\n",
              "      <td>24.072846</td>\n",
              "      <td>1890.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>EA</th>\n",
              "      <th>MTD</th>\n",
              "      <td>89.050317</td>\n",
              "      <td>6723.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NTAP</th>\n",
              "      <th>BWA</th>\n",
              "      <td>60.786732</td>\n",
              "      <td>7225.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>JNJ</th>\n",
              "      <th>DAL</th>\n",
              "      <td>76.235856</td>\n",
              "      <td>4344.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ADBE</th>\n",
              "      <th>TFC</th>\n",
              "      <td>80.383119</td>\n",
              "      <td>9571.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VLTO</th>\n",
              "      <th>CRM</th>\n",
              "      <td>34.028350</td>\n",
              "      <td>211.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Mean</th>\n",
              "      <th></th>\n",
              "      <td>66.099667</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a785614-ed3f-4528-ae46-76404bb50e0a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5a785614-ed3f-4528-ae46-76404bb50e0a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5a785614-ed3f-4528-ae46-76404bb50e0a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0d00661b-15cd-432a-9b22-445f6fc81840\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0d00661b-15cd-432a-9b22-445f6fc81840')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0d00661b-15cd-432a-9b22-445f6fc81840 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_3e35ec9d-8224-4e0e-93f3-f9d4005583ee\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_3e35ec9d-8224-4e0e-93f3-f9d4005583ee button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "results",
              "summary": "{\n  \"name\": \"results\",\n  \"rows\": 11,\n  \"fields\": [\n    {\n      \"column\": \"Correlation\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 24.944960454222066,\n        \"min\": 24.07284631255458,\n        \"max\": 98.51529608941205,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          89.05031737126177,\n          98.51529608941205,\n          34.02834994291611\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Sample\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 4034.6644042033354,\n        \"min\": 10.0,\n        \"max\": 12977.0,\n        \"num_unique_values\": 11,\n        \"samples\": [\n          6723.0,\n          12977.0,\n          211.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "sY3CKvvG8-Qf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}