Linear Regression with Newtons Method.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearRegression:\n",
    "    \n",
    "    def fit(self, X, y, epochs=10, lr=0.1):\n",
    "        X = np.insert(X, 0, 1, axis=1)\n",
    "        self.weights = np.zeros(X.shape[1])\n",
    "        self.loss = []\n",
    "        Xs = X.T.dot(X)\n",
    "        for i in range(epochs):\n",
    "            h = X.dot(self.weights)\n",
    "            self.loss.append(((y - h) ** 2).sum()/len(X))\n",
    "            H = np.linalg.pinv(2 * Xs)\n",
    "            gradient = (h - y).dot(X)\n",
    "            self.weights -= H.dot(gradient)\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return np.insert(X, 0, 1, axis=1).dot(self.weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3XlYVGX7wPHvo+KC4F5YEmBZ7gpipi0mai5YVm+bhkuuKVra7zXTKNc02zOXzMpywSWXbBHNDXPpTVNzy6U0ETVLRU0RFWGe3x8DMsAMzMDMnIG5P9c1FzBz5pxnHuDc59nuo7TWCCGEECWMLoAQQgjPIAFBCCEEIAFBCCFEBgkIQgghAAkIQgghMkhAEEIIAUhAEEIIkUECghBCCEACghBCiAylXH0ApdQs4GHgtNa6QcZzVYBFQAiQADyttT6f376qVaumQ0JCXFZWd7p8+TLly5c3uhgeQerCTOrBTOohi7PqYseOHWe11jflt51ydeoKpVRLIBmYYxEQ3gbOaa0nKaVGAJW11q/kt6+mTZvq7du3u7S87rJhwwZatWpldDE8gtSFmdSDmdRDFmfVhVJqh9a6aX7bubzLSGu9ETiX4+lHgdkZ388GHnN1OYQQQuTNqDGEAK31qYzv/wYCDCqHEEKIDC7vMgJQSoUA31t0GV3QWleyeP281rqyjff2B/oDBAQEhC9cuNDl5XWH5ORk/Pz8jC6GR5C6MJN6MJN6yOKsuoiIiLCry8jlg8o2/KOUukVrfUopdQtw2taGWuuZwEwwjyHk7E+7fv06J06c4OrVq64sr9NVrFiRsmXLGl0MypYtS2BgID4+PoaVQfqMzaQezKQesri7LowKCN8CPYFJGV+/KeiOTpw4gb+/PyEhISilnFU+l7t06RL+/v6GlkFrTVJSEidOnKBmzZqGlkUIYTyXjyEopRYA/wNqK6VOKKX6YA4EDyml/gDaZvxcIFevXqVq1apFKhh4CqUUVatWLXKtKyGEa7i8haC17mrjpTbOOoYEg4KTuhNCZJKVykIIIQAJCE6hlKJbt243fk5LS+Omm27i4Ycfdmg/ISEhnD17ttDbCCFyi42FkBAoUcL8NTbW6BJ5HgkITlC+fHn27dvHlStXAFizZg01atQwuFRCiEyxsdC/Pxw7Blqbv/bvX7CgUJwDiwQEJ4mMjGTFihUALFiwgK5ds4ZOzp07x2OPPUajRo1o3rw5e/bsASApKYl27dpRv359+vbti+WakHnz5tGsWTNCQ0N5/vnnSU9Pd+8HEqIYiYmBlJTsz6WkmJ93hDMDiycyatqpSwxdNZRdf+9y6j5Dq4fyYYcP892uS5cujBs3jocffpg9e/bQu3dvNm3aBMDo0aMJCwtj+fLlrF+/nh49erBp0ybGjh3L/fffz6hRo1ixYgWff/45AAcOHGDRokVs2bIFHx8foqOjiY2NpUePHk79bEJ4i8REx563xVZg6dbN/NqECRAVVbAyeoJiFRCM1KhRIxISEliwYAGRkZHZXtu8eTNLly4FoHXr1iQlJXHx4kU2btzIsmXLAOjUqROVK5sXa69bt44dO3Zw9913A3DlyhVuvvlmN34aIYqXoCDz1by15x2RVwDJbC1A0Q0KxSog2HMl70qdO3dm2LBhbNiwgaSkpALvR2tNz549efPNN51YOiG814QJ5pO15dW9r6/5+dhY89V9YqI5QOR1lW8rsGTK7IYqqgFBxhCcqHfv3owePZqGDRtme/6BBx4gNqOTccOGDVSrVo0KFSrQsmVL5s+fD8DKlSs5f958S4g2bdqwZMkSTp82Z/Q4d+4cx/L6KxRC5CkqCmbOhOBgUMr8deZM82s5xwS6dzdvY23AeMIEcyDJS2YroigOPherFoLRAgMDefHFF3M9P2bMGHr37k2jRo3w9fVl9mxz5u/Ro0fTtWtX6tevz7333ktQRvu1Xr16vPHGG7Rr1w6TyYSPjw/Tpk0jODjYrZ9HiOIkKir3lXtISO4xgcy5Hda6gDK/xsTYbikEBWUNPmfu21Z3kmXrpEoV83PnzmW1VNw+WVFrXWQe4eHhOqf9+/fneq4ouHjxotFFuMHoOoyPjzf0+J5C6sHMnfWglNbmEGD7ERxs/b3z5mnt65t9W19f8/PBwfnvy9r7c+5r6dJ4p3xOYLu24xwrXUZCCK9lz6CyrYFkW91QUVH2zWqyNmPJUkoKnDyZf/mcSQKCEMJr2TMmkFfQiIqChAQwmcxfM7uDbL3H8nl7prympua/jTNJQBBCeC3Lq3wwX+lbypyJ5ChrgSbnvuxpnZQu7fixC0MCghDCq2Ve5WsNc+da7wIqyD4tu5NeqBrLP+VCiOqeNeUov9aJr6/7B5VllpEQQmSwNhPJEVbXNJB7ylFK9/7EaShXNYpy5cwzi6zNMsp8zl2khSCEEE5gK89R8pDco8e+OoUJxJCUBFeumFsmZ8+aHyaTORjExMCOHe5dwyABoZCSkpIIDQ0lNDSU6tWrU6NGjRs/p9o5ItSrVy8OHTpk9zFPnTpFZGQkjRs3pl69enTu3DnP7c+dO8eMGTPs3r8QwvGFZbbyHPkmWR89DiLxxjaWSfZiY6HvyP0ce6A9Z0wn3JpAT7qMCqlq1ars2mVOqDdmzBj8/PwYNmxYtm1uzPEtYT3+fvHFFw4d87XXXqNTp04MGjQI4Eb2VFsyA8KAAQMcOo4Q3srehWWWbE41JYgQcq9iSyRrVDnzvZdTLxO9bDxXn3sPUv05a4oA3JcSw+taCO5aTn748GHq1atHVFQU9evX59SpU/Tv35+mTZtSv359Jk3Kuo30/fffz65du0hLS6NSpUqMGDGCxo0b06JFixvpKyydOnWKwMDAGz83atToxveTJk2iWbNmNGrUiHHjxgEwYsQIDh06RGhoKCNGjHDNBxaiGClIumxbs4ber5p79PgyvrxK1pSj24I0yw8up970elxs9Bbs6Q5TDlG3VPMb2ziambUgvCoguDuX+cGDB3nppZfYv38/NWrUYNKkSWzfvp3du3cTHx/P/v37c73n33//5cEHH2T37t20aNGCWbNm5dpm8ODB9OzZk9atWzNx4kROnToFQFxcHImJiWzdupVdu3bx008/8dNPPzFp0iRq167Nrl27sgUiIYR1BUmXbWuq6T2Ts6YcaRSJKph+zGQB5sv9srf8SZVBj/D4osepUKYCASs2wTezIOWmbPtyNDNrQXhVQHDWTTLsdccdd9C0adMbPy9YsIAmTZrQpEkTDh06ZDUglCtXjo4dOwIQHh5OQkJCrm0iIyM5cuQIffr0Yf/+/YSFhZGUlMTq1atZuXIlYWFhNGnShMOHD/P777+75sMJUYzZs7Asp7xWLmfObVXaxKa5CfwUHAWlrlGx8xukD6jP4bQfea/de+zsv5P3htyf7xoGV/GqMQRn3STDXuXLl7/x/R9//MHkyZPZtm0blSpV4plnnuHq1au53lPaYiVKyZIlSUtLs7rvqlWrEhUVRVRUFB06dGDz5s1orXnttdfo06dPtm0PHz7spE8khHfIK112XuyZthoVBTc3X8OguEH8ce4Pnqz7JB+0/4DACoE3XoesC9XgYPfdeMerWggFifrOcvHiRfz9/alQoQKnTp1i3bp1Bd7XunXrbty/+eLFixw9epSgoCDat2/P559/zuXLlwE4ceIEZ8+exd/fn0uXLjnlcwhRlBR0zDDPq/1C+OvSX3RZ0oV289qh0ayKWsXipxbfCAaWx09IgPDw7CkxXM2rWggFjfrO0KRJE+rVq0edOnUIDg6mefPm+b/Jhl9++YXBgwfj4+ODyWRi4MCBhIWFERYWxsGDB2/s29/fn/nz5xMSEkJ4eDgNGzakU6dOMo4gvEJBZgpZKuwiNUtppjSmbpvKqPhRpKanMubBMbxy/yuULVXWOQdwFntSonrKwxnprzNT0ypl/jpvnkNvdxpJf51F0j6bST2YOase7ElB7Q5bErfoRh830oxBd5jXQR9OOmz3e51VF9iZ/tqrWgjg3KgvhPBc7h4zzOlsyllGrB3B579+TmCFQJY8tYT/1P0PKmcGPQ/iVWMIQohiynKwoFo1qFaNNF2Co4TQlewDB5l3NHPVeiSTNvHZzs+oPbU2s3fP5uV7X+bAoAM8Ue8Jjw4GYHBAUEq9pJT6TSm1Tym1QCnlYR1qQgij5XvyzrnAKCkJkpIogSaEY3xK/xtBwdcXIiNdtx5p19+7uG/WffT7rh8Nbm7Arud38fZDb+NX2q/wO3cDwwKCUqoG8CLQVGvdACgJdDGqPEIIz2PXYtJ8bj1WnhQmEnNjplBcnOPrkfILShevXWToqqGEzwznyLkjzHlsDht6bqD+zfUd/ciGMnoMoRRQTil1HfAF/jK4PEIID5LXYtIbY4F2DAqEqEQy13h27259G1u7yWu20rPPahb9toj/++H/+Dv5bwY2Hcgbrd+gcrnK+ZbJEynzALRBB1dqCDABuAKs1lrnGu5VSvUH+gMEBASEL1y4MNvrFStWpFatWm4orXOlp6dTsmRJo4sBmBeu/fvvv4YdPzk5GT+/otGkdiWpBzPLetixw/Z24eEZ3+zdm/+9JkuXhoYN89zcYpNsbG1/vmQiK9Vkdl7YyV1+d/HSnS9Rp0KdvMvhIGf9TUREROzQWjfNd0N7piK54gFUBtYDNwE+wHKgW17vcca0U2c7e/asbty4sW7cuLEOCAjQt956642fr127ZvN9Oaedfv755/rUqVNWt928ebNu1qyZbty4sa5Tp44eN25cnmXasWOHXrlypd2fweg6lOmWZlIPZpb1YNfU0XnztPb1tb4hmF+zmF9ubfMcm2SjVI5d+lzWtI7RvO6jK75ZUU/fNl2npae5oiq8atppW+Co1voMgFJqGXAvMM/AMjnMnvTX9pg1axZNmjShevXquV7r2bMny5cvp0GDBqSnp+d774SdO3eyb98+OnTo4HA5hPAkdi0mtcz1kJho/dZjFnPNc25uZZNsgoLM3UQA3PU9dHwBKidQ/nB3Dn38DgF+AU75rJ7AyFlGiUBzpZSvMs/FagMccPlR3ZX/Gpg9ezbNmjUjNDSU6OhoTCYTaWlpdO/enebNm9OgQQM++ugjFi1axK5du3jmmWes3ljnzJkzNwJFyZIlqVevHmBuTj733HM0a9aMsLAwvvvuO65cucK4ceOIjY0lNDSUJUuWuOzzCVFo+fw/2p1CIjPXg8mU/dZjNvI+WG6eX2qICROgbPUE6PIoPPsIXPelzIINfNJhTrEKBmDgoLLWeqtSagmwE0gDfgVmuvSghV3L7oB9+/bx9ddf89NPP1GqVCn69+/PwoULueOOOzh79iw///wz/v7+XLhwgUqVKjFlyhSmTp1KaGhorn0NHTqUO++8k4iICDp27EiPHj0oU6YM48aNo0OHDnz55ZecP3+ee+65hz179jBq1Cj27dvHhx9+6NTPJIRT2fp/nDs322ZGLiZNTU/lePD7mAaOQ6Uq9Jq3CPprKBPHly6WC1wNXYegtR6tta6jtW6gte6utb7m0gO6Mf/12rVr+eWXX2jatCmhoaH8+OOPHDlyhFq1anHo0CFefvllfvjhBypWrJjvvsaOHcsvv/xC27ZtmTNnDp06dQJg9erVTJgwgdDQUCIiIrh69SqJ7lqGKURh2fp/PHnSmPLksP7oehrPaMzIdSN5uE5HEl4+gN48nGN/Fs9gAMZPO3UvN65l11rTu3dvxo8fn+u1PXv2sGzZMqZNm8bSpUuZOTP/hlGtWrWoVasWffv2pVq1avz7779orVm+fDl33HFHtm03btzotM8hRH5iY+3vj8/G1v+dnfcid5W/k/9m2OphxO6N5fbKt7Pi2RVE3hlpaJncxbtSV7gx/3Xbtm356quvOHv2LABJSUkkJiZy5swZtNY8/vjjjBs3jp07dwLkmaJ6xYoVmTOz+OOPPyhTpgz+/v60b9+eKVOm3Nju119/zXdfQjhToe5CaOv/zuKeIO6UZkpjytYp1J5am8X7FzOq5Sj2DdznNcEAvC0g2LrHnQvyXzds2JDRo0fTtm1bGjVqRLt27fjnn384fvw4LVu25L777qNXr15MnDgRgF69etG3b1+rg8pffvklderUITQ0lOeee4758+dTokQJRo8ezeXLl2nYsCH169dnzJgxALRu3Zrdu3cTFhYmg8rCpQrVC2vr/7FGDXfO/QBg64mtNPu0GS+uepHmgc3ZN3AfYyPGUs6nnGsP7GnsmZvqKQ+nrEPwkPzXkv46i8y/NyuK9ZBrjn7GQyk7d2Dl/3Hp0nib6wSc/e+blJKk+3/bX6sxSt/63q36q31faZPJVLidOpE3rUMwhuS/FsJpss3Rz/G8Xaz8P56cssFqq2PIELhyJfukpO7doVs3x28zadImZu+azfC1wzl/5TwvNX+JMa3G4F/G386CF0/e1WUkhHAqa70+SplP1gXt6rE1ppyUlLt7KmNozaGxiz3/7KHlFy3p/W1valetzc7nd/Je+/e8PhhAMQkIOvOvQjhM6k4UhuXCMTAHg4KcpC0VdEw5v7GLS9cuMWz1MJp80oRDSYf44tEv2NhrI40CGhXsgMVQkQ8IZcuWJSkpSU5sBaC1JikpibJl5TYUouAyV/0GB2cFg0wpKeYuHcvWQn4DxjVqWB9rrlo1/7JYm8mqtWbxb4upM60O7/3vPXqH9ebgoIM8F/ocJVSRPwU6VZEfQwgMDOTEiROcOXPG6KI45OrVqx5xIi5btiyBgYFGF0MUA3kt58lsLWzZArNn550soEoVc6sj59oGyJ3XKKecYxd/JP3B4JWDWX1kNaHVQ1n69FKaBzYv2Af0AkU+IPj4+FCzZk2ji+GwDRs2EBYWZnQxhHAaWwPMmVJSzCf69PTcz2e7vwF5z/2IiTEfx7J7CrLPIL9y/QqTNk9i0pZJlC1VlskdJhN9dzSlShT5U55LSXtJCOEU1gaYc8oZDDLZmywgs3tKa3PKI2tJ71b+sZIGHzdg3MZxPFnvSQ4OOsiL97wowcAOEhCEEI6zMhCQc4DZGlv3hCpIsoCcGUtbPnycJ756gsj5kfiU8GFdj3XE/ieWW/xvcXznXkoCghDCMXnkq8g8Sc+bZ31guH9/5ycLuJ5+nXe2vEPdaXVZ+cdKJraeyJ6Be2hds3XBd+qlJCAIIRxjR74KW/cxmD7dzvsb2GnjsY2EfRLG8LXDaV2zNfsH7WfkAyMpXdKYfEhFnXSqCSEcY2fWYFsDw85IFnD68mleXvMyc3bPIbhiMN90+YbOtTsXbqdCAoIQwkGFzldRcOmmdGbumMmr61/lcuplXr3/VWJaxuDrk89otrCLdBkJIRxTgKzBzsheuv2v7TT/vDnRcdGEVQ9jz8A9TGgzQYKBE0kLQQjhGAfvUl/YO9eev3KemPUxzNg+gwC/AOb/Zz5dGnTBfCt24UwSEIQQjnNgICCvMei8dqG1Zu6euQxbPYykK0kMbjaY8RHjqVg2/9vOioKRgCCEcKmC3Ln2t9O/ER0XzcZjG2ke2JwfIn8g7BZZ2e9qEhCEEC7lyBh0cmoyn/z5CUs2LaFCmQrMfHgmfZr0kSR0biK1LIRwKXvGoLXWLDuwjLrT6rLw+EJ6Nu7JocGH6BfeT4KBG0lNCyFcytYitczxgz/P/8nDCx7mia+e4J+EKgwuN4W1L37GD19XM7bgXki6jIQQLmdtDPpq2lXe2fIOEzdPhPRS+Kz7gOtbBhPy9maHZyIJ55AWghDC7VYfWU2jjxsxasMoOtfuTNWFB7m+aSiYsq5R87sDmnA+CQhCCLc5efEkTy9+mvbz2gOwuttqFj25iL8O1rC6vb1psYVzGBoQlFKVlFJLlFIHlVIHlFItjCyPEMI1rqdf5/3/vU+daXX47vfvGB8xnr0D9/LQHQ8BtrNeuCEbhrBg9BjCZGCV1vpJpVRpQNagC1HMbEncwsAVA9l7ei+Rd0YypeMUbq98e7ZtJkzIfXvMwqbFFo4zLCAopSoCLYHnALTWqUCqUeURQjjXmctneGXtK3yx6wtuq3Aby55exmN1HrOacsIyGwaYZyLlkQ1DuIiRLYSawBngC6VUY2AHMERrfdnAMgkhCsmkTXy28zNGrB3BpdRLvHLfK7ze8nXKly6f5/syZyJt2GC+yY5wP6Ut71LtzgMr1RT4GbhPa71VKTUZuKi1fj3Hdv2B/gABAQHhCxcudH9hXSA5ORk/Pz+ji+ERpC7MikM9/H7pdz7840MOXDpA44qNGXrnUELKhzi0j+JQD87irLqIiIjYobVumu+GWmtDHkB1IMHi5weAFXm9Jzw8XBcX8fHxRhfBY0hdmBXlerhw5YJ+Ie4FXWJsCX3zOzfrObvmaJPJVKB9FeV6cDZn1QWwXdtxXjZslpHW+m/guFKqdsZTbYD9RpVHCGFFPjcy0Fozf+986kyrw9RtUxnYdCCHBh+ie+Pukp66CDJ6HcILQKxSag8QCkw0uDxCFG+O3Kkm80YGx46B1lk3Msh4z8GzB2kzpw1Ry6K4rcJtbOu3jamRU6lUtpJbPopwPkOnnWqtdwH592sJIQrP0TvV2LiRQcqokbxR/Tfe/eldypcuz8edPqZfk36ULFHSteUXLmd0C0EI4S553anGGivLhL+tDfU6H+fNzW/ybPnmHBp8iAFNB0gwKCYkIAjhLRy9U43FMuGEStC5KzzaFfxS4ccv4MvXd3Dz8jUuKKgwigQEIbxFHvkhrA4tTJjANf9yTHwA6g2C9TXhndXw6wxoeQzJPlcMGZ26QgjhLjbyQ2yOnGB1aGHfh9X5ekQlDl2/whP74YNVcNvFHPuU7HPFirQQhPAWNu5U0y0uKvvQgt8pUjo+y6S/2pLm50vcs3Es2RqcOxiAZJ8rZiQgCOFNoqLMeSFMJmInJBASE5V1v+MSaXDPR/BCbai7DDaMZl/0Pjre2dG++2CKIk+6jITwQjlnoBL4M3QaCLfsgsPtIW4qwf61KJt5hrDMPpeYaG4ZSPa5YkcCghBe6MYM1HJJ0HYkhH8KF2vAV1/B/ifx9VVM+CjHm6zdB1MUKxIQhPBCxxJNEPYlPDQcyl6An/4LG0ZDqr+knvZiMoYghJfZ/fduygy8Hx7tA2fqwoxfYfW72YJBTIx92S1E8SItBCG8xMVrFxkdP5op26bgW70y+rsvSf2lB2BOQufrC5GRjmW3EMWLtBCEKOa01nz121fUnVaXyVsn07dJXxKGHWLWkJ4EByvLGajExTmW3UIUL9JCEKIY+z3pdwbHDWbNn2tocksTvn7ma5rVaAZYHyPu3t36fmT9mXeQgCBEMXRl7hdMXPYSbzf4l3Lpiik1ejCw76x8k9AFBZG1LiHH86L4ky4jIYqZFTOHUX9nH94I/Zenf4ODUzSDhy+h5IL8bz8r68+8mwQEIYqJxH8TeXzR4zx86j3KXNes/xLmfg3Vk7F7IMBGdgsZUPYSEhCEKGJyZiadPS+Vtza/Rd1pdVl9ZDWT1sLuGRCRkP19pmOJdk0jtchuQUKCBANvImMIQhQhuW56xo/02hqNPrKfx+o8xoftPyR45oOQnnsgIJEgmUYq8iQtBCGKkBspJ8r/A493h16t0CVTuGnNd3z9zNcEVwq2OhBwGV9exTwQINNIhS3SQhCiCDl2PB3ungFtYsAnBX58DTaP5GyaRQCwSERnOpZIIkG8ygQWkNUkkGmkwhoJCEIUEdtObqN09EBSq+2EP9vAimmQVBuAoOAcG2csMrg9RKaRCvtJl5EQHu78lfMM+H4AzT9rjm/AKUp/uxDmrLkRDPKaFirTSIUjJCAI4aG01szeNZvaU2vz6c5PGXLPEI69fJBZ//dMrpQTtgaIZRqpcIR0GQnhgfad3kf0img2JW6iRWALVndaTWj1UMDx2xLIbQyEvSQgCOFBklOTGbthLB/8/AEVy1bks0c+o1dYL0ooacwL15OAIIQH0Frz45kf6T6tOycunqBvWF/ebPsm1XyrGV004UVsBgSlVBwQrbVOcF9xhPA+h88d5oWVL7Dq8CoaBzTmqye/osVtLYwulvBCebVDvwBWK6VilFI+riqAUqqkUupXpdT3rjqGEJ7oatpVxmwYQ4PpDdiSuIXBdwxme//tVoNBznQVchcz4Qo2Wwha68VKqZXA68B2pdRcwGTx+vtOKsMQ4ABQwUn7E8LjrTq8isFxgzly/ghdGnTh/Xbvc2jHIUqVyP0vmStdhaSfEC6S30hVKnAZKAP453gUmlIqEOgEfOaM/Qnh6U5cPMFTi5+iY2xHSpUoxZrua1jwxAJu8b/F5ntupKuwIOknhCvkNYbQAXgf+BZoorVOsbVtIXwIDMdJAUYIT3U9/Tofbf2I0RtGk67TeSPiDYbdO4wypcrk+15baSYk/YRwNqW1tv6CUpuAAVrr31xyYKUeBiK11tFKqVbAMK31w1a26w/0BwgICAhfuDD/m3wUBcnJyfj5+RldDI9Q3Oti7797+eCPDzh6+SjNqzTnxVovcku53C0CW/Wwdy+kpubeb+nS0LChK0psrOL+9+AIZ9VFRETEDq1103w31Fob8gDeBE4ACcDfQAowL6/3hIeH6+IiPj7e6CJ4jOJaF6eTT+vnlj+nGYMO+iBILz+wXJtMJpvb26qHefO09vXVGrIevr7m54uj4vr3UBDOqgtgu7bjvGzYahet9UitdaDWOgToAqzXWnczqjxCOEu6KZ1Ptn9C7am1mbdnHiPuG8H+6P08WudRlFJ27ydzZlH37lCuHFStKuknhGvJwjQhnGjHXzuIjotm28lttAppxbTIadS7qZ7D+8k5sygpyZyUbu5cCQTCdTxiPbzWeoO2Mn4gRFFx4eoFBscNptlnzTh24RjzHp/H+h7rCxQMQGYWCWNIC0GIQtBaM3/vfP67+r+cSTlDdNNoxrceT6WylQq1X5lZJIwgAUGIAjpw5gDRcdFsSNhAsxrNiIuKo8ktTZyy76AgubGNcD+P6DISoii5nHqZkWtH0mhGI3b/vZsZnWbwvz7/c1owALmxjTCGBAQh7KS15puD31Bvej0mbZlE90bdOTT4EM83fd7p6antvbGN5DgSziRdRkLY4ej5o7y46kXDvY//AAAZN0lEQVS+//17GtzcgE29NnF/0P0uPWZ+N7aRHEfC2aSFIEQerqVd442Nb1Bvej02JGzgvXbvsbP/TpcHA3vITCThbNJCEMKGtX+uZVDcIH5P+p0n6z3JB+0/ILBCoNHFukFmIglnkxaCEDn8dekvuizpwkNzH8KkTayKWsXipxbnGQyM6Mu3NeNIZiKJgpKAIESGNFMaH/78IXWm1mH5weWMbTWWvQP30r5W+zzfl9mXf+yYOdNQZl9+XkHBGQFEZiIJZ5OAIATw0/GfCJ8Zzks/vMR9QffxW/RvjHpwFGVLlc33vY725dsKIOfOORYo7J2JJIS9ZAxBeLWzKWcZsXYEn//6OYEVAln69FIer/O4Q0noHO3LtxVAjh+HV191bNZQfjORhHCEtBCEVzJpE5/t/IzaU2sze/dshrUYxoFBB/hP3f84FAzA8b58W4EiLU1mDQljSUAQXmfX37u4f9b99PuuH/Vvqs+vz//KO+3ewa90wW5E4mhfvqODvjJrSLiLBAThNS5eu8jQVUMJnxnO4XOH+fLRL/nxuR9pcHODQu3X0b58WwGklI0OXJk1JNxFxhBEsae1ZtFvi/i/H/6Pv5P/ZkDTAUxoPYHK5So77RiO9OVnbhcTY776DwoyB4ly5cyBwbLbSGYNCXeSFoIo1g6dPcRDcx+i69Ku3Op/K1v7bmV6p+lODQYFERUFCQlgMpm/RkVBlSoya0gYSwKCKJZSrqfw2vrXaPhxQ7b/tZ2pHaeyte9W7q5xt+sPXohFBtYChRsOKwQgXUaiGPru0He8uOpFEi4k0KNxD95u+zYBfgHuObhBGeck0Z1wBmkhiGLj2IVjPLrwUTov7Iyvjy8bem5g9mOz3RcMIM9Vaq68gpdEd8IZpIUgirzU9FTe++k9xm8cj1KKt9u+zdDmQ/Ep6eP+wtiYI6qPJbr0Cl4S3QlnkBaCKNLWH11P4xmNeXX9q3So1YEDgw7w8n0vGxMMwOYc0ZMlg1x6BS+J7oQzSEBwERngc62/k/+m27JutJnThtT0VFY8u4JlzywjqKLBZ0AbiwxeSbc+d9RZV/CS6E44gwQEFyhI9kthn3RTOlO3TaX21Nos3r+Y11u+zr6B+4i8M9LoopnZWKW2Jdh6v5CzruAl0Z1wBhlDcIG8BvjkH7Tgtp7YSnRcNDtP7eSh2x9iWuQ07qx6p9HFys3KKrUJZJ8FBM6/gpdEd6KwpIXgAjLA51znrpxjwPcDaPF5C/5O/ptFTy7ih24/OD0YuLKbT67gRVEgLQQXCAoydxNZe17Yz6RNzN41m+Frh3P+ynmGNh/KmFZjqFCmgtOP5Y55/HIFLzydtBBcQAb4Cm/PP3to+UVLen/bm7uq3sWO/jt4v/37Tg8Gma2Cbt1kHr8QhgUEpdRtSql4pdR+pdRvSqkhRpXF2aR7oOAuXbvEf3/4L00+acLBsweZ1XkWm3ptonH1xk4/luXgvy3SzSe8iZFdRmnAf7XWO5VS/sAOpdQarfV+A8vkNNI94BitNUv2L2HoD0M5dekU/Zr0Y2KbiVT1reqyY1ob/M9JuvmENzGshaC1PqW13pnx/SXgAFDDqPII45xIOUGH2A48veRpAsoH8FOfn/jkkU9cGgwg76v/rsRyTIVw9JgsJBHeQ2mtjS4DSqkQYCPQQGt9Mcdr/YH+AAEBAeELFy50e/lcITk5GT+/gt2hq7i4ln6N+cfnMz9xPmVKlKF3zd48euujlFQlXXrcc+fg5ElITbX+ehXOEcwxSmDKerJECXPfX5UqVvdTujTUqJHtZbu3ydwo+eab8Tt92sZG3kP+N7I4qy4iIiJ2aK2b5ruh1trQB+AH7AD+k9+24eHhuriIj493+D3z5mkdHKy1Uuav8+Y5u1TuE/d7nL598u2aMejW01vrvy7+5Zbjzpunta+v1uYlg7kfvr5aX6oabP3F4OA89+Prm/13Ys82lhvFv/uujY28S0H+N4orZ9UFsF3bcT42dJaRUsoHWArEaq2XGVkWd8mc1bJjh2M9EcVl9fPxf4/zxFdPEDk/Ep8SPqzrsY7X677OLf63uOX4eY0bZA7++53LfyGJPdlF7cpAKmlKhQcxcpaRAj4HDmit3zeqHO6Uc1aLIyf1on7euJ5+nXe2vEPdaXVZ+cdKJrSewO4Bu2lds7Vby2Fr3EApixvS5JEpLjOg25qZZLl/uxYoyipG4UGMbCHcB3QHWiuldmU8PCQhjWsU5qRe2POGkcn2Nh7bSNgnYQxfO5w2t7dh/6D9vPrAq5QpVcZ9hchgV1ZQKwtJUpQvzx6bQPfueU9TtdyPXceSNKXCgxg5y2iz1lpprRtprUMzHnFGlccdCnNSL8x5wxndTQUJKP8k/0PP5T158MsHSU5N5psu3/BNl28IqRRi/4GdzK5FgxYLSTSKRBVMXz2TBUSR1xyMnPux61iyilF4EnsGGjzl4ZZBZReO3AYHZw0uvvtuvLWxyjyLle8ApR3HtTFG6tRjp6Wn6enbputKkyppn3E++tW1r+rLqZdt7t8dg4iWv9aqVc0Pe37FturOWl1a249df04ZG8W/+27Rny3gBDKonMXdg8qGn+QdeTgzIFj9Ry3MWdfOY2buPjMg5Lf7gp7ILN+cjtJHCdZdmZftoyllX7kdCSi/nPxFN53Z1Dx7aHZrfeDMgXw/27vvxrv0PFiYX6tS9gUDZ5AToZnUQxYJCG4ICLZOEPZMN3TGse09CRYqPll5czK+2YKCvR/L1knRMqCcSzmno7+P1mqM0tXfra4X7F2gTSaTXcWzNzgWVGFaSPm1EJxZZjkRmkk9ZJGA4IaAYOufPB07znxOYs8vulBdPTbefJRgh09keZXDZDLpObvm6JvfuVmXGFtCvxj3or5w5UK+XSWF6T5zlD0BzRZrQTlzf85u1ciJ0EzqIYtXrUMwis3BXQowcluI6Tv5vbVQM4tsbBREosPJ9myNew4c9RutZreix/Ie1KxUk+39tjO542S+X1ox30Fsd862LMyAvLVEhXPnmj/XjWmqQhQX9kQNT3m4uoXwQlUH+2gK0aezdGl8vm8tSAsh88r8KAV4cx4sr/irVL+kyz4yXPN6KV1iZGXde9onOt2U7lC53dlCcPHQkNPIlbGZ1EMW6TJyQ0DI8wThyCyjQvTpfPRRfL5vdfREZrl9V+bpZJx7FjSZTHrIjKVa/V+gZgyazr01vqdz7daeLhprYwiu6orJPJ6np/2QE6GZ1EMWCQhuCAhaO+kEUYjOacur4rzeWpj41JV5+ijmWUa23mzv/g8nHdYd53U0B4IBjTS3bbbr6j+vOGk5wJ6zKj3xCt7V5ERoJvWQRQKCmwKCU7i4heAoR+OTPS2QK9ev6LEbxuoy48tov4l+muYfaEpct/vq354TvCvqoiiSE6GZ1EMWGVQ2goMDw5mbRx2bQIoq2CrTGjWcv0DV0cHT/FJprD6ymoYfN2T0htE8VucxDg46SPCpoWDKfV8ly2M4esc4W2mocw4w2/o1Ofq8EMIGe6KGpzxc0kJw8HI25+ZdmaePqWBtyqNbxpr4+Hin92s7emVuc9FVhRP6qa+e0oxB3/nRnXr14dUFPoY9CjOeMnCgY897cjeUXBmbST1kkS4jdwcEB7t9CpsGIpOr/ugLNSZe4rqm+fuakX5avVZWP/nReH31+tVCHcMeOWdcWQuytuq9ZEkbQc3Gw5O7oeREaCb1kMXdAcHIeyp7BgcnxHt6tmJH7uU8YYJ5fUBKCnDbFugUDdX3wB8d0XFTibt2O0uq5N6fs+8XXaWKuUspJgbuOxbLp6o/vjqjLytjEcN9KXCM3AdNT3fsWJ7yexLCE8kYgoMd7x6TrdgJC+K6d4cylc9Q5une0Od+KHceFi6D2BVw/na33m8hKsq80Cs2OCYrGGRKSeGtks4piGSVFsI27woI1k6ieaQfdnBzhw5b6M9RwHzWN96aaEI3mcn5brW5VnsubB4O0/bDwccBdWN7t19R2zhgjfTEXPXuKMkqLUTevCcg2DqJgtUpMbFEObK5zS4UW4c9d64Qn6UQd9qJiYGUijuhz73wyPPwTyOYsZuS8W9Bau6bebv9itrGAVVw0I16LwhH03UI4ZXsGWjwlEehBpXdMXhsZbTV1n4++ii+4J/FxvSgdFSeg7wXrlzQdHxBM6qEZtjNmkZzNJiyzcIxYlZOtoEzO6Yx2arTqlWL3swiSzKYaib1kEXWIbiKqwePbTQF7jtmvRvH1tz7zF3l2cVk4yo6kSCrvUdaa+bvnU+daXWg2VTYPgCmHoI93cnsHsq8gra35eNIuR3qMrNjEYOtbrvJkwv/GYTwavZEDU95eHQLwcYbjpcMdqiFYNc8fwfudbD/9H4d8WWEZgz67pl36/GfbXfqVXRmowhyN1zsXQ9QkKugopCbyFFyZWwm9ZBF1iG4KiAUcgFavidOG904JpTV/SxdGp/reHndjCVXIMrnbmj4XNYj147UPuN8dKVJlfT0bdN1WnpatmMV9mRqrY7sXSdg+Xny+qMvjid+W+REaCb1kEUCgqsCgtYOn10c2jyPJoW1/Vj+ou05sdrKR2T1sLW/0SX/G6wZg+75dU/9T/I/9teRA+y933B+n8fWH31RSVvtLHIiNJN6yCIBwZUBwZUcPHtZ/qLtObHa6qrKdthKRzVdH9GMQQdOrK9/TPjR6R/Tkj33G85rJbG14GjJWavCiwo5EZpJPWSRQeWiytGMbhbym+uf1/z5qCiY9sk1Kj0yEQbVQ92+ni5V3+bP4b/SMrhlAT6I/fKbkurrax7gtrV+IL8puJ6+KlyI4kYCgjNlLrc1mRy6v2JeJ9b84sq6P9cx6UJjLoTH8GTjTiQOP8iCwS/jU9LH4eI7ytpsH5Wxpi2z3NOnk+f6gZQUOHnS+msesypcCC8hAcFOrkylbGsa5bx5tuPKqUuneHbps7Sd25Y0Uxpxz8ax+KnFBFYIdF7B8mHv/YYz46RS1vdjawpuQVaFCyEKTpLb2SFziUFK9nxrgHPmuGfuIybG3B0SFGQ+6Vnbd5opjem/TOe19a+Rmp7KmAfH8Mr9r1C2VNnCF6QAHEl0FxRkrrucSpe2vW+wr16EEIVnaAtBKdVBKXVIKXVYKTXCyLLkpRCZIuxmT2/Tzyd+5u5P72bIqiHce9u97Ivex+hWo10WDJzdKrJ1xV+jhu33FLAXTghRAIYFBKVUSWAa0BGoB3RVStUzqjx5MXpwMykliX7f9qPF5y04c/kMi59azMqoldSqUstlxyxE/jybbI27V6nivHILIQrOyBZCM+Cw1vpPrXUqsBB41MDy2GTU4KZJm5j16yxqT63NF7u+YFiLYRwYdIAn6z2JstUh7ySuahXJFb8QnsvIgFADOG7x84mM5zyOEYObe/7ZwwNfPECfb/tQp1odfn3+V95p9w7+Zfxdd1ALRreKhBDup8xrFgw4sFJPAh201n0zfu4O3KO1Hpxju/5Af4CAgIDwhQsXur2sYJ4rf/KkeUZM6dLmfu/CdHUkJyfj55c73fTltMt8mfAly04uw9/HnwG3D6BdQDtKKPfG7r17rc/+KV0aGjZ07rFs1YW3kXowk3rI4qy6iIiI2KG1bprvhvasXnPFA2gB/GDx80hgZF7v8eiVyg7KuQLRZDLpRfsW6Vvfu1WrMUo//93zOiklyZjCafemjZCVqWZSD2ZSD1m86Z7KvwB3KqVqAieBLsCzBpbHML8n/c7guMGs+XMNYdXDWPb0Mu4JvMfQMsmUTyG8j2EBQWudppQaDPwAlARmaa1/M6o8Rrhy/Qpvbn6Tt7a8RblS5ZjacSoDmg6gZImSRhcNcGyNgRCi6DN0YZrWOg6IM7IMRvlf0v/oPb03Ry8cJaphFO+2e5fqftWNLpYQwovJSmU3S/w3kSGrhrD84HLqVqtLfM94WoW0MrpYQgghAcFdUtNT+eB/HzBu4zgA+tXsx9SoqZQuaSNvgxBCuJkEBDf4MeFHouOi2X9mP4/VeYwP23/I0V1HJRgIITyKZDt1oX+S/6H7191pNbsVKddT+K7rd3z9zNcEV7KRC1oIIQwkLQQXSDelM2P7DGLWx5ByPYXXHniNkQ+MxNfHxp1ihBDCA0hAcLJfTv7CwBUD2XFqB21qtmFa5DRqV6ttdLGEECJfEhCc5PyV87y67lU+2fEJ1f2qs+CJBTxT/xmXJ6ETQghnkYBQSFpr5uyew8trXibpShJD7hnC2IixVChTweiiCSGEQyQgFMK+0/uIXhHNpsRNtAhswepOqwmtHmp0sYQQokAkIBRAcmoyYzeM5YOfP6BS2Up89shn9Arr5faMpEII4UwSEBygtWbZgWUMWTWEk5dO0jesL5PaTqKqb1WjiyaEEIUmAcFOh88d5oWVL7Dq8CoaBzRm8VOLaXFbC6OLJYQQTiMBIR9X067y1ua3eHPzm5QuWZoP2n/A4GaDKVVCqk4IUbzIWS0Pqw6vYnDcYI6cP0KXBl14r9173Op/q9HFEkIIl5CAYMWJiyd46YeXWLJ/CXdVvYs13dfQ9va2RhdLCCFcSgKChevp15m8dTJjNowhXafzRsQbDLt3GGVKlTG6aEII4XISEDJsOraJ6Lho9p3eR6c7OzGl4xRqVq5pdLGEEMJtvD4gnL58muFrhjN792yCKgax/JnldK7dWVJOCCG8jtcGhHRTOp/u/JSR60ZyOfUyI+8fScwDMZQvXd7oogkhhCG8MiDsPLWTgSsGsu3kNiJCIpgWOY26N9U1ulhCCGEorwsIo+JHMWHTBG7yvYnY/8TStUFX6R4SQgi8MCDUrFSTQXcPYnzEeCqWrWh0cYQQwmN4XUDoFdaLXvQyuhhCCOFxJD2nEEIIQAKCEEKIDBIQhBBCABIQhBBCZDAkICil3lFKHVRK7VFKfa2UqmREOYQQQmQxqoWwBmigtW4E/A6MNKgcQgghMhgSELTWq7XWaRk//gwEGlEOIYQQWTxhDKE3sNLoQgghhLdz2cI0pdRaoLqVl2K01t9kbBMDpAGxeeynP9A/48dkpdQhZ5fVINWAs0YXwkNIXZhJPZhJPWRxVl0E27OR0lo74ViOU0o9BzwPtNFapxhSCAMppbZrrZsaXQ5PIHVhJvVgJvWQxd11YUjqCqVUB2A48KA3BgMhhPBERo0hTAX8gTVKqV1KqRkGlUMIIUQGQ1oIWutaRhzXw8w0ugAeROrCTOrBTOohi1vrwrAxBCGEEJ7FE6adCiGE8AASEFxMKdVBKXVIKXVYKTXCyuv/p5Tan5HGY51Syq7pYUVNfvVgsd0TSimtlCq2s0zsqQul1NMZfxe/KaXmu7uM7mDH/0aQUipeKfVrxv9HpBHldDWl1Cyl1Gml1D4bryul1EcZ9bRHKdXEZYXRWsvDRQ+gJHAEuB0oDewG6uXYJgLwzfh+ILDI6HIbUQ8Z2/kDGzGvXm9qdLkN/Ju4E/gVqJzx881Gl9ugepgJDMz4vh6QYHS5XVQXLYEmwD4br0diXryrgObAVleVRVoIrtUMOKy1/lNrnQosBB613EBrHa+zpt4W1zQe+dZDhvHAW8BVdxbOzeypi37ANK31eQCt9Wk3l9Ed7KkHDVTI+L4i8Jcby+c2WuuNwLk8NnkUmKPNfgYqKaVucUVZJCC4Vg3guMXPJzKes6UPxTONR771kNEMvk1rvcKdBTOAPX8TdwF3KaW2KKV+zli3U9zYUw9jgG5KqRNAHPCCe4rmcRw9jxSY191T2VMppboBTYEHjS6LuymlSgDvA88ZXBRPUQpzt1ErzC3GjUqphlrrC4aWyv26Al9qrd9TSrUA5iqlGmitTUYXrLiSFoJrnQRus/g5MOO5bJRSbYEYoLPW+pqbyuZO+dWDP9AA2KCUSsDcT/ptMR1Ytudv4gTwrdb6utb6KOYU8Xe6qXzuYk899AG+AtBa/w8oizm3j7ex6zziDBIQXOsX4E6lVE2lVGmgC/Ct5QZKqTDgE8zBoDj2FUM+9aC1/ldrXU1rHaK1DsE8ltJZa73dmOK6VL5/E8ByzK0DlFLVMHch/enOQrqBPfWQCLQBUErVxRwQzri1lJ7hW6BHxmyj5sC/WutTrjiQdBm5kNY6TSk1GPgB86yKWVrr35RS44DtWutvgXcAP2CxUgogUWvd2bBCu4Cd9eAV7KyLH4B2Sqn9QDrwstY6ybhSO5+d9fBf4FOl1EuYB5if0xnTbooTpdQCzBcA1TLGS0YDPgBa6xmYx08igcNACtDLZWUphvUrhBCiAKTLSAghBCABQQghRAYJCEIIIQAJCEIIITJIQBBCCAFIQBCiwJRStymljiqlqmT8XDnj5xBjSyZEwUhAEKKAtNbHgY+BSRlPTQJmaq0TDCuUEIUg6xCEKASllA+wA5iFOUtpqNb6urGlEqJgZKWyEIWgtb6ulHoZWAW0k2AgijLpMhKi8DoCpzAn6BOiyJKAIEQhKKVCgYcwZ2h9yVU3LhHCHSQgCFFAypyN8GNgqNY6EXOiwneNLZUQBScBQYiC64c5O+2ajJ+nA3WVUl53kyNRPMgsIyGEEIC0EIQQQmSQgCCEEAKQgCCEECKDBAQhhBCABAQhhBAZJCAIIYQAJCAIIYTIIAFBCCEEAP8PHXBvHv+OchYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Generating our data.\n",
    "import matplotlib.pyplot as plt\n",
    "X = np.linspace(1.0, 10.0, 100)[:, np.newaxis]\n",
    "Y = np.sin(X) + 0.1 * np.power(X, 2) + 0.5 * np.random.randn(100, 1)\n",
    "X /= np.max(X)\n",
    "\n",
    "X = np.hstack((np.ones_like(X), X))\n",
    "\n",
    "order = np.random.permutation(len(X))\n",
    "portion = 20\n",
    "\n",
    "\"\"\"\n",
    "    Split your data into train and test using the order and permutation variable\n",
    "\"\"\"\n",
    "test_idx = order[:portion]\n",
    "train_idx = order[portion:]\n",
    "\n",
    "train_x = X[train_idx]\n",
    "train_y = Y[train_idx]\n",
    "\n",
    "test_x = X[test_idx]\n",
    "test_y = Y[test_idx]\n",
    "\n",
    "train_y = train_y.reshape(-1,)\n",
    "\n",
    "model = LinearRegression().fit(train_x, train_y, epochs=6)\n",
    "\n",
    "plt.plot(X[:,1], model.predict(X), c='g', label='Model')\n",
    "plt.scatter(train_x[:,1], train_y, c='b', label='Train Set')\n",
    "plt.scatter(test_x[:,1], test_y, c='r', label='Test Set')\n",
    "plt.grid()\n",
    "plt.legend(loc='best')\n",
    "plt.xlabel('X')\n",
    "plt.ylabel('Y')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f98e9ab5890>]"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAHBlJREFUeJzt3Xt0lfWd7/H3d+/sJOQCBBIw5MZF8C4XA0m9QL1WvFTtxSLG6UzbY89MXWOt6/S068yc5Vkza6anHbWdHqcdWzu1A8horVZbrahYkVYCCfebgiiQEEjCPSH3/Tt/ZAMBwYRk7/3s/ezPa62svffzPMn+7Hb5eR5+z7OfnznnEBGR5BfwOoCIiESHCl1ExCdU6CIiPqFCFxHxCRW6iIhPqNBFRHxChS4i4hMqdBERn1Chi4j4RFo83yw/P9+NHz8+nm8pIpL0amtrm51zBf1tF9dCHz9+PDU1NfF8SxGRpGdmOweynYZcRER8QoUuIuITKnQREZ/ot9DNrMTM3jKzzWa2ycwejCx/xMzqzWxt5OeW2McVEZGzGchJ0W7gYefcajPLBWrN7PXIusedc/8Su3giIjJQ/Ra6c64BaIg8P2pmW4CiWAcTEZFzc05j6GY2HpgOVEcWPWBm683sF2aWF+VsIiJyDgZc6GaWAzwPfNM5dwT4CTAJmEbvEfyjZ/m9+82sxsxqmpqaBhVy+bZm/u2P2wf1uyIiqWJAhW5mIXrLfKFz7jcAzrl9zrke51wY+Bkw60y/65x70jlX7pwrLyjo94tOZ/TOtiYeXfI+ew+3D+r3RURSwUCucjHgKWCLc+6xPssL+2x2F7Ax+vF6za8opSfsWLxqV6zeQkQk6Q3kCP0q4D7gutMuUfy+mW0ws/XAtcBDsQpZNjqb2VMKWLxyN9094Vi9jYhIUhvIVS7LATvDqleiH+fsqipKuf8/a3ljSyM3X3pePN9aRCQpJM03Ra+7cAyFIzJZWD2ge9SIiKScpCn0tGCAe2aV8s62Zj5sbvU6johIwkmaQgeYN7OEtICxSEfpIiIfk1SFPmZ4JjddMpbnauto7+rxOo6ISEJJqkIHqKoo49CxLn6/vsHrKCIiCSXpCv1Tk0YzsSCbBRp2ERE5RdIVuplxb0UZa3YdYtOew17HERFJGElX6ABfmFFMZijAghX65qiIyHFJWegjskLcfvk4fru2nqPtXV7HERFJCElZ6ABVlWUc6+zhhTX1XkcREUkISVvoU0tGclnRCBas2Ilzzus4IiKeS9pCB6iqLOX9fS2s+uig11FERDyX1IV++9Rx5GamsWCFLmEUEUnqQs9KT+PzM4p5dWMDzS0dXscREfFUUhc69A67dPU4nq3Z7XUUERFPJX2hnz8ml8qJo1hUvYuesE6OikjqSvpCh95LGOsOtrHs/cFNQi0i4ge+KPSbLj6P/JwMnRwVkZTmi0JPTwswb2YJS99rpO7gMa/jiIh4wheFDnBPRSkGPLNS93cRkdTkm0IvGjmM6y4cw3+t2k1nd9jrOCIiceebQge4t7KM5pZOXtu01+soIiJx56tCnzO5gJJRw3RyVERSkq8KPRAw5s8qo/rDA2zbd9TrOCIiceWrQge4u7yY9GCAhdU6OSoiqcV3hT46J4O5l53H87V1HOvs9jqOiEjc+K7Qofebo0c7unlp7R6vo4iIxI0vC728LI8LxuayoFqTX4hI6vBloZsZVZWlbKw/wrq6w17HERGJC18WOsCd04vISg/qEkYRSRm+LfTczBB3Ti/i5XV7OHSs0+s4IiIx59tCB6iqKKOjO8yva+u8jiIiEnO+LvSLxw1nRulIFlXv0slREfE9Xxc69F7CuKO5lT9/sN/rKCIiMdVvoZtZiZm9ZWabzWyTmT0YWT7KzF43s22Rx7zYxz13t1xWSF5WSCdHRcT3BnKE3g087Jy7GKgEvmFmFwPfAd50zk0G3oy8TjiZoSBfLC9hyeZ97DvS7nUcEZGY6bfQnXMNzrnVkedHgS1AEXAH8HRks6eBO2MVcqjmzyqlJ+xYvHK311FERGLmnMbQzWw8MB2oBsY65xoiq/YCY8/yO/ebWY2Z1TQ1eTOJ8/j8bK6ZnM8zK3fR3aPJL0TEnwZc6GaWAzwPfNM5d6TvOtd7CckZLyNxzj3pnCt3zpUXFBQMKexQVFWWsfdIO29ubfQsg4hILA2o0M0sRG+ZL3TO/SayeJ+ZFUbWFwIJ3ZTXXziGwhGZOjkqIr41kKtcDHgK2OKce6zPqpeAL0eefxn4bfTjRU9aMMC8maW8s62Zj5pbvY4jIhJ1AzlCvwq4D7jOzNZGfm4BvgfcaGbbgBsirxPavFklBAPGopWa/EJE/Cetvw2cc8sBO8vq66MbJ7bGDs/kpovH8lzNbr514xQyQ0GvI4mIRI3vvyl6uqrKMg4e6+KVDQ39bywikkRSrtCvnDSaifnZOjkqIr6TcoVuZsyvKGX1rkNs3nOk/18QEUkSKVfoAF+4opiMtAALqnWULiL+kZKFPjIrndunjuPFNfUcbe/yOo6ISFSkZKFD78nRY509vLim3usoIiJRkbKFPrV4BJcWDWfBCk1+ISL+kLKFbmZUVZTx3r6j1Ow86HUcEZEhS9lCB/jstHHkZqbpEkYR8YWULvSs9DQ+P6OYVzfsZX9Lh9dxRESGJKULHeDeilI6e8I8W1PndRQRkSFJ+UKfPDaXigmjWLRyJ+GwTo6KSPJK+UKH3ksYdx9o4+1t3syoJCISDSp04DOXnEd+TgYLdXJURJKYCh1ITwvwpZnFLN3aSP2hNq/jiIgMigo94p5ZpTjgmWpNfiEiyUmFHlGcl8V1F4xh8arddHaHvY4jInLOVOh9VFWW0dzSwZLNe72OIiJyzlTofcyeUkBx3jB9c1REkpIKvY9goHfyixU7DrC98ajXcUREzokK/TR3l5cQChoLVujkqIgkFxX6afJzMph7aSHPr67jWGe313FERAZMhX4GVZVlHG3v5uV1e7yOIiIyYCr0M5g5Po8pY3M07CIiSUWFfgZmRlVlGRvqD7Nu9yGv44iIDIgK/Szuml5EVnpQlzCKSNJQoZ9FbmaIO6cX8fL6PRw+1uV1HBGRfqnQP0FVRRntXWF+vVqTX4hI4lOhf4KLxw1nRulIFlbvxDlNfiEiiU2F3o+qyjJ2NLXy7gf7vY4iIvKJVOj9uOWyQvKyQiyo1slREUlsKvR+ZIaCfLG8hCWb9tF4pN3rOCIiZ6VCH4D5s0rpDjsWr9rtdRQRkbPqt9DN7Bdm1mhmG/sse8TM6s1sbeTnltjG9Nb4/GyumZzPMyt30d2jyS9EJDEN5Aj9l8DNZ1j+uHNuWuTnlejGSjxVlWU0HG5n6dZGr6OIiJxRv4XunFsGHIhDloR2/YVjKByRyQLNOSoiCWooY+gPmNn6yJBMXtQSJai0YIB5M0tZ9n4TO/e3eh1HRORjBlvoPwEmAdOABuDRs21oZvebWY2Z1TQ1NQ3y7RLDvFklBAPGIh2li0gCGlShO+f2Oed6nHNh4GfArE/Y9knnXLlzrrygoGCwORPC2OGZ3HTxWJ6t2U17V4/XcURETjGoQjezwj4v7wI2nm1bv6mqLOPgsS5e3djgdRQRkVMM5LLFZ4B3gQvMrM7Mvgp838w2mNl64FrgoRjnTBhXThrNxPxsTX4hIgknrb8NnHP3nGHxUzHIkhTMjPkVpfzj77ewpeEIFxUO9zqSiAigb4oOyheuKCYjLaDJL0QkoajQB2FkVjq3Tx3Hi2vqaeno9jqOiAigQh+0qsoyWjt7eGFNvddRREQAFfqgTS0ewaVFw1m4QpNfiEhiUKEPkplRVVHG1r1Hqd150Os4IiIq9KH47LRx5Gam6eSoiCQEFfoQZKWn8fkZxbyyYS/7Wzq8jiMiKU6FPkT3VpTS2RPmudo6r6OISIpToQ/R5LG5VEwYxcLqnYTDOjkqIt5RoUdBVWUZuw+08fa25L6bpIgkNxV6FHzmkvPIz8lgoU6OioiHVOhRkJ4W4Eszi1m6tZH6Q21exxGRFKVCj5J7ZpXigGc0+YWIeESFHiXFeVlcd8EYFq/aTWd32Os4IpKCVOhRVFVZRnNLB0s27/U6ioikIBV6FM2eUkBx3jB9c1REPKFCj6JgoHfyixU7DrC98ajXcUQkxajQo+zu8hJCQdMUdSISdyr0KMvPyWDupYU8v7qOY52a/EJE4keFHgNVlWUcbe/m5XV7vI4iIilEhR4DM8fnMWVsjoZdRCSuVOgxYGZUVZaxof4w63Yf8jqOiKQIFXqM3DW9iKz0oC5hFJG4UaHHSG5miDumFfHy+j0cPtbldRwRSQEq9BiqqiylvSvMr1dr8gsRiT0VegxdMm4E00tHsrB6J85p8gsRiS0VeoxVVZSxo6mVdz/Y73UUEfE5FXqM3Xp5ISOzQiyo1slREYktFXqMZYaCfPGKYpZs2kfjkXav44iIj6nQ42B+RRndYcfiVbu9jiIiPqZCj4MJ+dlcMzmfZ1buortHk1+ISGyo0OPk3ooyGg63s3Rro9dRRMSnVOhxcsNFYzhveCYLNOeoiMRIv4VuZr8ws0Yz29hn2Sgze93MtkUe82IbM/mlBQPMm1XCsveb2Lm/1es4IuJDAzlC/yVw82nLvgO86ZybDLwZeS39mDezlGDAWKSjdBGJgX4L3Tm3DDhw2uI7gKcjz58G7oxyLl86b0QmN140lmdrdtPe1eN1HBHxmcGOoY91zjVEnu8FxkYpj+9VVZZx8FgXr25s6H9jEZFzMOSToq73JiVnvVGJmd1vZjVmVtPU1DTUt0t6V04azYT8bE1+ISJRN9hC32dmhQCRx7Nei+ece9I5V+6cKy8oKBjk2/lHIGDcW1FK7c6DbGk44nUcEfGRwRb6S8CXI8+/DPw2OnFSwxeuKCYjLaDJL0QkqgZy2eIzwLvABWZWZ2ZfBb4H3Ghm24AbIq9lgEZmpXPb5eN4cU09LR3dXscREZ9I628D59w9Z1l1fZSzpJSqylKeX13HC2vqua+yzOs4IuID+qaoR6aVjOSSccNZuEKTX4hIdKjQPWJmVFWWsXXvUWp3HvQ6joj4gArdQ3dMG0duRppOjopIVKjQPZSVnsbnZhTxyoa97G/p8DqOiCQ5FbrH7q0so7MnzHO1dV5HEZEkp0L32JSxucyaMIpF1bsIh3VyVEQGT4WeAKoqy9h14BjLtunWCCIyeCr0BHDzJeeRn5Ou+7uIyJCo0BNAelqAu8tLWLp1H/WH2ryOIyJJSoWeIO6ZVYoDFq/UUbqIDI4KPUGUjMri2gvGsHjVbrp6wl7HEZEkpEJPIFWVpTQd7WDJpn1eRxGRJKRCTyBzpoyhaOQwfXNURAZFhZ5AggFjfkUp7+7Yz/bGFq/jiEiSUaEnmC/NLCEUNBZW6yhdRM6NCj3B5OdkcPOlhTxfW0dbZ4/XcUQkiajQE1BVRSlH2rt5ed0er6OISBJRoSegWRNGMWVsDv/3D1v57dp6TYAhIgOiQk9AZsaP75lBUd4wHly8lnt/Xq2TpCLSLxV6grrgvFxe+Jur+Ic7L2VD/WHm/mgZP3htq8bVReSsVOgJLBgw7qssY+nDn+b2qeN44q0PuOGxt3ljs754JCIfp0JPAgW5GTx29zQW319JVnqQr/2qhq89XcPuA8e8jiYiCUSFnkQqJ47mlQev4btzL+RP25u58fG3eeKt7XR2694vIqJCTzqhYICvz5nEGw/P4dNTxvCD195j7o+W8eftzV5HExGPqdCTVNHIYfz0viv4j7+cSVePY/7Pq3lw8Roaj7R7HU1EPKJCT3LXXjiGJQ/N5m+vn8yrG/Zy/aNv88s/fUi3bsErknJU6D6QGQryrRun8NpDs5lWOpJHXt7MHU/8iTW7DnodTUTiSIXuIxPys/nVV2bxxPwZNLd08Lmf/Jnv/mYDh451eh1NROJAhe4zZsatlxfyxrfm8JWrJvBszW6ue/Rtnq3ZTTisWwiI+JkK3adyM0P8/W0X8/IDVzMhP5tv/3o9d//7u2xpOOJ1NBGJERW6z108bjjPff1TfP/zl/NBUwu3/Xg5//i7zbR0dHsdTUSiTIWeAgIB4+6ZJSx9+NPcXV7Mz5d/yPWP/pHfr2/QnRxFfESFnkLystP5589dzm/+5kpGZ2fwjUWr+YtfrOTD5lavo4lIFKjQU9CM0jxeeuAqHrn9YtbuOsRnHl/GY6+/T3uX7uQoksyGVOhm9pGZbTCztWZWE61QEntpwQB/edUE3nx4DnMvO49/fXMbn/nhMt56r9HraCIySNE4Qr/WOTfNOVcehb8lcTZmeCY/mjedRV+rIBgw/uo/VvHXC2rZc6jN62gico405CIAXHl+Pq8+eA3/4zMX8NZ7jdzw2Ns8uewDunQLAZGkMdRCd8ASM6s1s/ujEUi8k5EW5BvXns/rD83hykmj+adXtnLrv77Dyg8PeB1NRAZgqIV+tXNuBjAX+IaZzT59AzO738xqzKymqalpiG8n8VAyKouff3kmP/uLclo7erj739/l4WfX0dzS4XU0EfkEFq3rkM3sEaDFOfcvZ9umvLzc1dTo3GkyOdbZzf9bup2fvbODYaEg3775Qu6ZVUowYF5HE0kZZlY7kPOUgz5CN7NsM8s9/hy4Cdg42L8niSkrPY1v33whrz54DZeMG8HfvbiRz/3bn9hQd9jraCJymqEMuYwFlpvZOmAl8Hvn3B+iE0sSzfljcln03yr40bxp1B9q57NPLOd//3Yjh9u6vI4mIhFRG3IZCA25+MOR9i4eW/I+v3r3I0Zlp/O/br2IO6cVYaZhGJFYiPmQi6Su4ZkhHvnsJbz0wNUU5WXx0H+tY96TK9i276jX0URSmgpdBu3SohG88NdX8k93XcbWvUeZ+6N3+N6rWznWqTs5inhBhS5DEggY8ytKWfrwHO6aXsRP3/6AGx9bxmub9upOjiJxpkKXqBidk8EPvjiV5/77p8jJSOPr/1nLV5+uYdf+Y15HE0kZKnSJqpnjR/G7v72av7v1Iqp37OfGx9/mx29uo6Nbd3IUiTUVukRdKBjga9dM5I2H53DDRWN59PX3mfvDd1i+rdnraCK+pkKXmCkcMYwn7p3B01+ZRdg5qp6q5oFFq9l3pN3raCK+pEKXmJszpYA/fHM2D90whSWb93H9o2/z1PIP6dadHEWiSoUucZEZCvLgDZN5/aHZXFGWxz/8bjO3/Xg5tTt1J0eRaFGhS1yVjc7ml381k59WzeBwWxef/8m7/M9fr+dAa6fX0USSngpd4s7MuPnSQt741hy+Pnsiz6+u47pH/8jilbsIh3Xtushg6V4u4rn39h7l71/cyMqPDjBiWIiJBdlMzM9hYkE2kwqymViQQ9noLDLSgl5HFfHEQO/lokKXhOCc43frG1ixYz87mlrZ0dzCviMnJ9QIGBTnZZ1S9r2Fn8OY3AzdGEx8baCFnhaPMCL9MTNunzqO26eOO7GspaObDyPl/kFTKzuaWtjR1Er1jgO0dZ38olJORhoT8rM/VvYT83MYlq6jekkdKnRJWDkZaVxWPILLikecsjwcduw90n7iSH5HUysfNLVQ89FBXlq3h77/6Bw3IpOJBZGSz88+8XzciGEENOuS+IwKXZJOIGCMGzmMcSOHcfXk/FPWtXf18GFza2/ZN7Wwo7n38YXV9RztOHkXyMxQgPGje4ds+h7RTyzIJjczFO+PJBIVKnTxlcxQkIsKh3NR4fBTljvnaGrpiBT9ybLftOcwr25soO/FNQW5GSeO5if1KfvivGGkBXVhmCQuFbqkBDNjTG4mY3IzqZw4+pR1nd1hdh1ojYzTnyz7P2xs4OCxk1PshYJG2ehTh24mRco+Lzs93h9J5GNU6JLy0tMCnD8ml/PH5H5s3cHWzj4nZU+W/VvvNdLVc/KwPi8r1Fvyp5V96ahs0tN0VC/xoUIX+QR52elckT2KK8pGnbK8uydM3cG2Pidle8v+j+838Vxt3YntggGjJG/Yx8p+YkE2BTm63FKiS4UuMghpwQDj87MZn5/NdReeuu5Ie9eJyy2Pj9l/0NTCn7Y309F98oZkuRlpTCzIZlR2OpmhIMNCQTLTI4+hQOQxyLD0IJlpvY/Hl2WGAideDwsFyYg8hoKmnUQKU6GLRNnwzBBTS0YytWTkKcvDYceew22nDN182NzK/tZO2jp7aO/uoa0zTHtXD21dPfQM4jYIwYCd2CEc30kc3yH07iwCfXYKfXcSZ96BnNjR9NmBZIaCZKQFtONIQCp0kTgJBIzivCyK87KYPaWg3+27esK0dfXQ3tlb8O1dva/bOnto7+o5UfzHl3V0h2nrPLms/cROovf14bYuGo+c3L6tq4eOrjCdg7iNsRmn/KshI7JDOL4DyTix7uM7kMy0AMFggKAZAev93yUQeR4M9P4L4/R1wUDvie3AANZZ5O/0ty4QoM/fMyzAied915mRNDsvFbpIggoFA4SCAYbH+Lr47p4w7ZGdwSk7is6eU5a39Vn3sZ1Mnx1Ia0c3zS2dH/tbfYebko3Z2cs+EDj+vHcHEjjLdv/8ucuZNWFU/282BCp0kRSXFgyQEwyQkxHbOgiHHe3dvTuB7nAY56An7Ag7RzhM7+OJn0GuC0OPczjn6BnounDv3zz++sS605YfX9f7++eyrvezZ2fE/jYUKnQRiYtAwMhKTyNLl+zHjC6QFRHxCRW6iIhPqNBFRHxChS4i4hMqdBERn1Chi4j4hApdRMQnVOgiIj5hzp37DYAG/WZmTcDOQf56PtAcxTjJQJ85Negzp4ahfOYy51y/NwCKa6EPhZnVOOfKvc4RT/rMqUGfOTXE4zNryEVExCdU6CIiPpFMhf6k1wE8oM+cGvSZU0PMP3PSjKGLiMgnS6YjdBER+QRJUehmdrOZvWdm283sO17niTUz+4WZNZrZRq+zxIOZlZjZW2a22cw2mdmDXmeKNTPLNLOVZrYu8pn/j9eZ4sXMgma2xsx+53WWeDCzj8xsg5mtNbOamL5Xog+5mFkQeB+4EagDVgH3OOc2exoshsxsNtAC/Mo5d6nXeWLNzAqBQufcajPLBWqBO33+/7EB2c65FjMLAcuBB51zKzyOFnNm9i2gHBjunLvN6zyxZmYfAeXOuZhfd58MR+izgO3OuR3OuU5gMXCHx5liyjm3DDjgdY54cc41OOdWR54fBbYARd6mii3XqyXyMhT5Seyjqygws2LgVuDnXmfxo2Qo9CJgd5/Xdfj8P/ZUZmbjgelAtbdJYi8y9LAWaARed875/jMDPwS+DSTvjNHnzgFLzKzWzO6P5RslQ6FLijCzHOB54JvOuSNe54k151yPc24aUAzMMjNfD6+Z2W1Ao3Ou1usscXa1c24GMBf4RmRINSaSodDrgZI+r4sjy8RHIuPIzwMLnXO/8TpPPDnnDgFvATd7nSXGrgI+GxlTXgxcZ2YLvI0Ue865+shjI/ACvcPIMZEMhb4KmGxmE8wsHZgHvORxJomiyAnCp4AtzrnHvM4TD2ZWYGYjI8+H0XvSf6u3qWLLOfdd51yxc248vf8dL3XOVXkcK6bMLDtyoh8zywZuAmJ29VrCF7pzrht4AHiN3pNlzzrnNnmbKrbM7BngXeACM6szs696nSnGrgLuo/eIbW3k5xavQ8VYIfCWma2n96DldedcSlzGl2LGAsvNbB2wEvi9c+4PsXqzhL9sUUREBibhj9BFRGRgVOgiIj6hQhcR8QkVuoiIT6jQRUR8QoUuIuITKnQREZ9QoYuI+MT/BxGwYlirL0qSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogisticRegression:\n",
    "    \n",
    "    def fit(self, X, y, epochs=10, lr=0.1):\n",
    "        X = np.insert(X, 0, 1, axis=1)\n",
    "        self.weights = np.zeros(X.shape[1])\n",
    "        self.loss = []\n",
    "        Xs = X.T.dot(X)\n",
    "        for i in range(epochs):\n",
    "            h = self.sigmoid(X.dot(self.weights))\n",
    "            self.loss.append(self.get_loss(h,y))\n",
    "            invH =  np.linalg.pinv(Xs * h.dot(1-h))\n",
    "            gradient = (h - y).dot(X)\n",
    "            self.weights -= invH.dot(gradient)\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return self.sigmoid(np.insert(X, 0, 1, axis=1).dot(self.weights))\n",
    "    \n",
    "    def get_loss(self, h, y):\n",
    "        return np.abs(y.dot(np.log(h)) + (1 - y).dot(np.log(1 - h)))\n",
    "    \n",
    "    def sigmoid(self, z):\n",
    "        return 1/(1 + np.exp(-1 * z))\n",
    "    \n",
    "    def predict_classes(self, X):\n",
    "        return (self.predict(X) >= 0.5) * 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.46901795 0.47771344]\n",
      "[0 0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.metrics import accuracy_score\n",
    "X, y = load_iris(return_X_y=True)\n",
    "clf = LogisticRegression().fit(X, (y !=0 )*1, lr=0.01, epochs=10)\n",
    "print(clf.predict(X[:2]))\n",
    "print(clf.predict_classes(X[:2]))\n",
    "pre = clf.predict_classes(X[:])\n",
    "accuracy_score((y !=0 )*1, pre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}