Taiwan ETF Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用Gradient Boosting Regressor 來預測ETF價格"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "這個使用了GBR 來預測 18檔 ETF的價格\n",
    "一次要預測一週的價格\n",
    "\n",
    "這裡的思路是以下模式：\n",
    "1. 將每日的開盤、收盤、最高、最低、交易量建立各自的特徵\n",
    "2. 根據這些特徵去預測隔天的開盤、收盤、最高、最低、交易量\n",
    "3. 反覆重複以上過程去預測連續日期的價格\n",
    "\n",
    "運作分四個區塊：\n",
    "1. 讀取資料\n",
    "2. 特徵工程\n",
    "3. 用演算法進行學習\n",
    "4. 執行預測"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "#import library\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%pylab inline\n",
    "\n",
    "#import machine learning model\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "#pandas output setting\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('expand_frame_repr', True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#set the path of file\n",
    "tetfp_path = 'C:/Users/AdamChang/Documents/Documents/PFD/Porfolio/T-Brain ETF Compitition/TBrain_Round2_DataSet_20180331/tetfp.csv'\n",
    "\n",
    "# read the data\n",
    "tetfp = pd.read_csv(tetfp_path, encoding='big5',\n",
    "                    names=['Code','Date','Company','Open','High','Low','Close','Volumn'],skiprows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Code</th>\n",
       "      <th>Date</th>\n",
       "      <th>Company</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volumn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50</td>\n",
       "      <td>20130102</td>\n",
       "      <td>元大台灣50</td>\n",
       "      <td>54.00</td>\n",
       "      <td>54.65</td>\n",
       "      <td>53.90</td>\n",
       "      <td>54.40</td>\n",
       "      <td>16,487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>20130103</td>\n",
       "      <td>元大台灣50</td>\n",
       "      <td>54.90</td>\n",
       "      <td>55.05</td>\n",
       "      <td>54.65</td>\n",
       "      <td>54.85</td>\n",
       "      <td>29,020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50</td>\n",
       "      <td>20130104</td>\n",
       "      <td>元大台灣50</td>\n",
       "      <td>54.85</td>\n",
       "      <td>54.85</td>\n",
       "      <td>54.40</td>\n",
       "      <td>54.50</td>\n",
       "      <td>9,837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>20130107</td>\n",
       "      <td>元大台灣50</td>\n",
       "      <td>54.55</td>\n",
       "      <td>54.55</td>\n",
       "      <td>53.90</td>\n",
       "      <td>54.25</td>\n",
       "      <td>8,910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50</td>\n",
       "      <td>20130108</td>\n",
       "      <td>元大台灣50</td>\n",
       "      <td>54.00</td>\n",
       "      <td>54.20</td>\n",
       "      <td>53.65</td>\n",
       "      <td>53.90</td>\n",
       "      <td>12,507</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Code      Date           Company   Open   High    Low  Close       Volumn\n",
       "0    50  20130102  元大台灣50            54.00  54.65  53.90  54.40       16,487\n",
       "1    50  20130103  元大台灣50            54.90  55.05  54.65  54.85       29,020\n",
       "2    50  20130104  元大台灣50            54.85  54.85  54.40  54.50        9,837\n",
       "3    50  20130107  元大台灣50            54.55  54.55  53.90  54.25        8,910\n",
       "4    50  20130108  元大台灣50            54.00  54.20  53.65  53.90       12,507"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tetfp.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Preprocess(tetfp1):\n",
    "    #Change the Date type\n",
    "    tetfp1['Date'] = pd.to_datetime(tetfp1['Date'],format='%Y%m%d')\n",
    "\n",
    "    #Change the Code datatype\n",
    "    tetfp1['Code'] = tetfp1['Code'].astype('object')\n",
    "\n",
    "    #Process the datatype of the volumn data\n",
    "    for i in range(tetfp1['Volumn'].count()):\n",
    "        a = str(tetfp1.loc[i,'Volumn']).split(',')\n",
    "        b=''\n",
    "        for j in a:\n",
    "            b=b+j\n",
    "        b = b.lstrip()\n",
    "        b = float(b)\n",
    "        tetfp1.loc[i,'Volumn']=b\n",
    "    tetfp1['Volumn'] = tetfp1['Volumn'].astype('float64')\n",
    "\n",
    "    #Get the date part of the date\n",
    "    tetfp1['Year'] = tetfp1['Date'].dt.year\n",
    "    tetfp1['Month'] = tetfp1['Date'].dt.month\n",
    "    tetfp1['Day'] = tetfp1['Date'].dt.day\n",
    "    tetfp1['Dayofwork'] = tetfp1['Date'].dt.dayofweek\n",
    "    tetfp1['Week'] = tetfp1['Date'].dt.week\n",
    "\n",
    "    #Get the season\n",
    "    tetfp1['Season'] = 'nan'\n",
    "    for i in range(tetfp1.Season.count()):\n",
    "        if tetfp1.loc[i,'Month'] in [1,2,3]:\n",
    "            tetfp1.loc[i,'Season'] = 1\n",
    "        elif tetfp1.loc[i,'Month'] in [4,5,6]:\n",
    "            tetfp1.loc[i,'Season'] = 2\n",
    "        elif tetfp1.loc[i,'Month'] in [7,8,9]:\n",
    "            tetfp1.loc[i,'Season'] = 3\n",
    "        else:\n",
    "            tetfp1.loc[i,'Season'] = 4\n",
    "\n",
    "    #Shift the date of price\n",
    "    tetfp1['Open1DayShift'] = tetfp1['Open'].shift(1)\n",
    "    tetfp1['High1DayShift'] = tetfp1['High'].shift(1)\n",
    "    tetfp1['Low1DayShift'] = tetfp1['Low'].shift(1)\n",
    "    tetfp1['Close1DayShift'] = tetfp1['Close'].shift(1)\n",
    "    tetfp1['Volumn1DayShift'] = tetfp1['Volumn'].shift(1)\n",
    "\n",
    "    #Calculate the range and fluctuation\n",
    "    tetfp1['Close2DayShift'] = tetfp1['Close'].shift(2)\n",
    "    tetfp1['Fluctuation'] = tetfp1['Close1DayShift'] - tetfp1['Close2DayShift']\n",
    "    #Calculate the range yesterday\n",
    "    tetfp1['YesterdayRange'] = tetfp1['High1DayShift'] - tetfp1['Low1DayShift']\n",
    "\n",
    "    #fill the shift nan data\n",
    "    tetfp1 = tetfp1.fillna(method = 'bfill')\n",
    "\n",
    "    #Calculate the fluctuation\n",
    "    tetfp1['RiseOrDown'] = 'nan'\n",
    "    for i in range(tetfp1['Fluctuation'].count()):\n",
    "        if tetfp1.loc[i,'Fluctuation']==0:\n",
    "            tetfp1.loc[i,'RiseOrDown'] = 0\n",
    "        elif tetfp1.loc[i,'Fluctuation']>0:\n",
    "            tetfp1.loc[i,'RiseOrDown'] = 1\n",
    "        else:\n",
    "            tetfp1.loc[i,'RiseOrDown']=-1\n",
    "\n",
    "    #Select history information\n",
    "    pmax = tetfp1.loc[0,'Close1DayShift']\n",
    "    pmin = tetfp1.loc[0,'Close1DayShift']\n",
    "    tetfp1['HistoryMax'] = 'nan'\n",
    "    tetfp1['HistoryMin'] = 'nan'\n",
    "    for i in range(tetfp1['HistoryMax'].count()):\n",
    "        if tetfp1.loc[i,'Close1DayShift'] > pmax:\n",
    "            pmax = tetfp1.loc[i,'Close1DayShift']\n",
    "        tetfp1.loc[i,'HistoryMax'] = pmax\n",
    "        if tetfp1.loc[i,'Close1DayShift'] < pmin:\n",
    "            pmin = tetfp1.loc[i,'Close1DayShift']\n",
    "        tetfp1.loc[i,'HistoryMin'] = pmin\n",
    "\n",
    "    #Calculate the history range\n",
    "    tetfp1['HistoryRange'] = tetfp1['HistoryMax'] - tetfp1['HistoryMin']\n",
    "\n",
    "    #Overview the history range\n",
    "    tetfp1['HistoryRangeRise'] = 0\n",
    "    for i in range(tetfp1['HistoryRange'].count()):\n",
    "        if (i-1)>0:\n",
    "            if tetfp1.loc[i,'HistoryRange']>tetfp1.loc[i-1,'HistoryRange']:\n",
    "                tetfp1.loc[i,'HistoryRangeRise'] = 1\n",
    "\n",
    "    #Overview the yesterday range\n",
    "    tetfp1['YesterdayRangeRise'] = 0\n",
    "    for i in range(tetfp1['YesterdayRange'].count()):\n",
    "        if (i-1)>0:\n",
    "            if tetfp1.loc[i,'YesterdayRange']>tetfp1.loc[i-1,'YesterdayRange']:\n",
    "                tetfp1.loc[i,'YesterdayRangeRise'] = 1\n",
    "            elif tetfp1.loc[i,'YesterdayRange']<tetfp1.loc[i-1,'YesterdayRange']:\n",
    "                tetfp1.loc[i,'YesterdayRangeRise'] = -1\n",
    "\n",
    "    #Calculate the volumn change\n",
    "    tetfp1['Volumn2DayShift'] = tetfp1['Volumn'].shift(2)\n",
    "    tetfp1['VolumnFluctuation'] = tetfp1['Volumn1DayShift'] - tetfp1['Volumn2DayShift']\n",
    "\n",
    "    #Calculate the volumn rise or down\n",
    "    tetfp1 = tetfp1.fillna(method='bfill')\n",
    "    tetfp1['VolumnRise'] = 0\n",
    "    for i in range(tetfp1['VolumnRise'].count()):\n",
    "        if tetfp1.loc[i,'VolumnFluctuation'] > 0:\n",
    "            tetfp1.loc[i,'VolumnRise'] = 1\n",
    "        else:\n",
    "            tetfp1.loc[i,'VolumnRise'] = -1\n",
    "\n",
    "    #Moving Average - Open\n",
    "    tetfp1['Open1DayShiftMA5'] = pd.rolling_mean(tetfp1['Open1DayShift'],5)\n",
    "    tetfp1['Open1DayShiftMA10'] = pd.rolling_mean(tetfp1['Open1DayShift'],10)\n",
    "    tetfp1['Open1DayShiftMA20'] = pd.rolling_mean(tetfp1['Open1DayShift'],20)\n",
    "    tetfp1['Open1DayShiftMA60'] = pd.rolling_mean(tetfp1['Open1DayShift'],60)\n",
    "    tetfp1['Open1DayShiftMA120'] = pd.rolling_mean(tetfp1['Open1DayShift'],120)\n",
    "\n",
    "    #Moving Average - High\n",
    "    tetfp1['High1DayShiftMA5'] = pd.rolling_mean(tetfp1['High1DayShift'],5)\n",
    "    tetfp1['High1DayShiftMA10'] = pd.rolling_mean(tetfp1['High1DayShift'],10)\n",
    "    tetfp1['High1DayShiftMA20'] = pd.rolling_mean(tetfp1['High1DayShift'],20)\n",
    "    tetfp1['High1DayShiftMA60'] = pd.rolling_mean(tetfp1['High1DayShift'],60)\n",
    "    tetfp1['High1DayShiftMA120'] = pd.rolling_mean(tetfp1['High1DayShift'],120)\n",
    "\n",
    "    #Moving Average - Low\n",
    "    tetfp1['Low1DayShiftMA5'] = pd.rolling_mean(tetfp1['Low1DayShift'],5)\n",
    "    tetfp1['Low1DayShiftMA10'] = pd.rolling_mean(tetfp1['Low1DayShift'],10)\n",
    "    tetfp1['Low1DayShiftMA20'] = pd.rolling_mean(tetfp1['Low1DayShift'],20)\n",
    "    tetfp1['Low1DayShiftMA60'] = pd.rolling_mean(tetfp1['Low1DayShift'],60)\n",
    "    tetfp1['Low1DayShiftMA120'] = pd.rolling_mean(tetfp1['Low1DayShift'],120)\n",
    "\n",
    "    #Moving Average - Close\n",
    "    tetfp1['Close1DayShiftMA5'] = pd.rolling_mean(tetfp1['Close1DayShift'],5)\n",
    "    tetfp1['Close1DayShiftMA10'] = pd.rolling_mean(tetfp1['Close1DayShift'],10)\n",
    "    tetfp1['Close1DayShiftMA20'] = pd.rolling_mean(tetfp1['Close1DayShift'],20)\n",
    "    tetfp1['Close1DayShiftMA60'] = pd.rolling_mean(tetfp1['Close1DayShift'],60)\n",
    "    tetfp1['Close1DayShiftMA120'] = pd.rolling_mean(tetfp1['Close1DayShift'],120)\n",
    "\n",
    "    #Moving Average - Volumn\n",
    "    tetfp1['Volumn1DayShiftMA5'] = pd.rolling_mean(tetfp1['Volumn1DayShift'],5)\n",
    "    tetfp1['Volumn1DayShiftMA10'] = pd.rolling_mean(tetfp1['Volumn1DayShift'],10)\n",
    "    tetfp1['Volumn1DayShiftMA20'] = pd.rolling_mean(tetfp1['Volumn1DayShift'],20)\n",
    "    tetfp1['Volumn1DayShiftMA60'] = pd.rolling_mean(tetfp1['Volumn1DayShift'],60)\n",
    "    tetfp1['Volumn1DayShiftMA120'] = pd.rolling_mean(tetfp1['Volumn1DayShift'],120)\n",
    "\n",
    "    #Moving Sum - Volumn\n",
    "    tetfp1['Volumn1DayShiftMS5'] = pd.rolling_sum(tetfp1['Volumn1DayShift'],5)\n",
    "    tetfp1['Volumn1DayShiftMS10'] = pd.rolling_sum(tetfp1['Volumn1DayShift'],10)\n",
    "    tetfp1['Volumn1DayShiftMS20'] = pd.rolling_sum(tetfp1['Volumn1DayShift'],20)\n",
    "    tetfp1['Volumn1DayShiftMS60'] = pd.rolling_sum(tetfp1['Volumn1DayShift'],60)\n",
    "    tetfp1['Volumn1DayShiftMS120'] = pd.rolling_sum(tetfp1['Volumn1DayShift'],120)\n",
    "\n",
    "    #Fill the nan data\n",
    "    tetfp1=tetfp1.fillna(method='bfill')\n",
    "\n",
    "    #Change the datatype\n",
    "    tetfp1['Code'] = tetfp1['Code'].astype('object')\n",
    "    tetfp1['Month'] = tetfp1['Month'].astype('object')\n",
    "    tetfp1['Dayofwork'] = tetfp1['Dayofwork'].astype('object')\n",
    "    tetfp1['Season'] = tetfp1['Season'].astype('object')\n",
    "    tetfp1['RiseOrDown'] = tetfp1['RiseOrDown'].astype('object')\n",
    "    tetfp1['HistoryRangeRise'] = tetfp1['HistoryRangeRise'].astype('object')\n",
    "    tetfp1['YesterdayRangeRise'] = tetfp1['YesterdayRangeRise'].astype('object')\n",
    "    tetfp1['VolumnRise'] = tetfp1['VolumnRise'].astype('object')\n",
    "    \n",
    "    return tetfp1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learning(tetfp1):\n",
    "    #define prediction list\n",
    "    keylst = ['Open','High','Low','Close','Volumn']\n",
    "    \n",
    "    #define prediction parameter\n",
    "    OpenPrice = 0\n",
    "    HighPrice = 0\n",
    "    LowPrice = 0\n",
    "    ClosePrice = 0\n",
    "    Volumn = 0\n",
    "    \n",
    "    #select the column from the dataframe\n",
    "    for key in keylst:\n",
    "        responce = tetfp1[key]\n",
    "        variable = tetfp1[['Month','Dayofwork','Season','Open1DayShift','High1DayShift',\n",
    "                           'Low1DayShift', 'Close1DayShift', 'Volumn1DayShift',\n",
    "               'Close2DayShift', 'Fluctuation', 'YesterdayRange', 'RiseOrDown',\n",
    "               'HistoryMax', 'HistoryMin', 'HistoryRange', 'HistoryRangeRise',\n",
    "               'YesterdayRangeRise', 'Volumn2DayShift', 'VolumnFluctuation',\n",
    "               'VolumnRise', 'Open1DayShiftMA5', 'Open1DayShiftMA10',\n",
    "               'Open1DayShiftMA20', 'Open1DayShiftMA60', 'Open1DayShiftMA120',\n",
    "               'High1DayShiftMA5', 'High1DayShiftMA10', 'High1DayShiftMA20',\n",
    "               'High1DayShiftMA60', 'High1DayShiftMA120', 'Low1DayShiftMA5',\n",
    "               'Low1DayShiftMA10', 'Low1DayShiftMA20', 'Low1DayShiftMA60',\n",
    "               'Low1DayShiftMA120', 'Close1DayShiftMA5', 'Close1DayShiftMA10',\n",
    "               'Close1DayShiftMA20', 'Close1DayShiftMA60', 'Close1DayShiftMA120',\n",
    "               'Volumn1DayShiftMA5', 'Volumn1DayShiftMA10', 'Volumn1DayShiftMA20',\n",
    "               'Volumn1DayShiftMA60', 'Volumn1DayShiftMA120', 'Volumn1DayShiftMS5',\n",
    "               'Volumn1DayShiftMS10', 'Volumn1DayShiftMS20', 'Volumn1DayShiftMS60',\n",
    "               'Volumn1DayShiftMS120']]\n",
    "        \n",
    "        #use one hot encoding\n",
    "        variable_dummy = pd.get_dummies(variable)\n",
    "        \n",
    "        #get the last data for prediction\n",
    "        x = variable_dummy[:-1]\n",
    "        px = variable_dummy[-1:]\n",
    "        responce = responce[:-1]\n",
    "        y = tetfp1[-1:]\n",
    "        \n",
    "        #seperate the training set and testing set\n",
    "        xtrain,xtest,ytrain,ytest = train_test_split(x, responce)\n",
    "        \n",
    "        #define machine learning model\n",
    "        gbr = GradientBoostingRegressor().fit(xtrain,ytrain)\n",
    "        \n",
    "        #assign different prediction\n",
    "        if key == 'Open':\n",
    "            OpenPrice = gbr.predict(px)\n",
    "        if key == 'High':\n",
    "            HighPrice = gbr.predict(px)\n",
    "        if key == 'Low':\n",
    "            LowPrice = gbr.predict(px)\n",
    "        if key == 'Close':\n",
    "            ClosePrice = gbr.predict(px)\n",
    "        if key == 'Volumn':\n",
    "            Volumn = gbr.predict(px)\n",
    "        #print('Key:',key,'\\t price:',gbr.predict(px))\n",
    "        \n",
    "    return OpenPrice, HighPrice, LowPrice, ClosePrice, Volumn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Code</th>\n",
       "      <th>Date</th>\n",
       "      <th>Company</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volumn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16940</td>\n",
       "      <td>6208</td>\n",
       "      <td>20130102</td>\n",
       "      <td>富邦台50</td>\n",
       "      <td>30.78</td>\n",
       "      <td>31.16</td>\n",
       "      <td>30.77</td>\n",
       "      <td>31.02</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16941</td>\n",
       "      <td>6208</td>\n",
       "      <td>20130103</td>\n",
       "      <td>富邦台50</td>\n",
       "      <td>31.29</td>\n",
       "      <td>31.30</td>\n",
       "      <td>31.21</td>\n",
       "      <td>31.28</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16942</td>\n",
       "      <td>6208</td>\n",
       "      <td>20130104</td>\n",
       "      <td>富邦台50</td>\n",
       "      <td>31.25</td>\n",
       "      <td>31.25</td>\n",
       "      <td>31.03</td>\n",
       "      <td>31.03</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16943</td>\n",
       "      <td>6208</td>\n",
       "      <td>20130107</td>\n",
       "      <td>富邦台50</td>\n",
       "      <td>30.85</td>\n",
       "      <td>30.95</td>\n",
       "      <td>30.78</td>\n",
       "      <td>30.95</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16944</td>\n",
       "      <td>6208</td>\n",
       "      <td>20130108</td>\n",
       "      <td>富邦台50</td>\n",
       "      <td>30.75</td>\n",
       "      <td>30.77</td>\n",
       "      <td>30.60</td>\n",
       "      <td>30.73</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  Code      Date            Company   Open   High    Low  Close  \\\n",
       "0  16940  6208  20130102  富邦台50              30.78  31.16  30.77  31.02   \n",
       "1  16941  6208  20130103  富邦台50              31.29  31.30  31.21  31.28   \n",
       "2  16942  6208  20130104  富邦台50              31.25  31.25  31.03  31.03   \n",
       "3  16943  6208  20130107  富邦台50              30.85  30.95  30.78  30.95   \n",
       "4  16944  6208  20130108  富邦台50              30.75  30.77  30.60  30.73   \n",
       "\n",
       "        Volumn  \n",
       "0           79  \n",
       "1           61  \n",
       "2           48  \n",
       "3           17  \n",
       "4           15  "
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:110: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=5).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:111: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=10).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:112: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=20).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:113: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=60).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:114: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=120).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:117: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=5).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:118: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=10).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:119: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=20).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:120: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=60).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:121: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=120).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:124: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=5).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:125: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=10).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:126: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=20).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:127: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=60).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:128: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=120).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:131: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=5).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:132: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=10).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:133: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=20).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:134: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=60).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:135: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=120).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:138: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=5).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:139: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=10).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:140: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=20).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:141: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=60).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:142: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=120).mean()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:145: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=5).sum()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:146: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=10).sum()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:147: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=20).sum()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:148: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=60).sum()\n",
      "C:\\Users\\AdamChang\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:149: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=120).sum()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Code = 56\n",
      "Close price is: [25.149999999999999, 25.197905531768576, 25.105450261746576, 25.157972571518854, 25.118924527116619]\n",
      "Code = 6204\n",
      "Close price is: [52.100000000000001, 52.16513082512899, 52.276534499656549, 52.216842881413967, 52.247126799774293]\n",
      "Code = 51\n",
      "Close price is: [32.109999999999999, 31.830856829046297, 32.085822053738788, 31.867167776633554, 31.968137555730401]\n",
      "Code = 6208\n",
      "Close price is: [46.0, 45.913084894532425, 46.080591697817695, 45.898887957994212, 46.156849075038501]\n",
      "Code = 52\n",
      "Close price is: [53.200000000000003, 53.601715536822802, 53.758953255389528, 53.823450675283937, 54.148436177470103]\n",
      "Code = 53\n",
      "Close price is: [34.200000000000003, 34.967027898807203, 34.182755872996523, 34.932740590145883, 34.435494641041082]\n",
      "Code = 54\n",
      "Close price is: [23.09, 23.12867931009686, 23.189181886393659, 23.192880481768562, 23.189874014988366]\n",
      "Code = 55\n",
      "Close price is: [17.039999999999999, 16.988064072621512, 16.958937122945226, 16.879376967618185, 16.944903898215927]\n",
      "Code = 6203\n",
      "Close price is: [36.82, 37.089229300344158, 36.938432244997003, 37.149190690956466, 36.994058234334048]\n",
      "Code = 57\n",
      "Close price is: [48.740000000000002, 48.790703983793222, 48.85539584731783, 48.876513577195027, 48.84679445617126]\n",
      "Code = 6201\n",
      "Close price is: [13.26, 13.298065289560592, 13.34701920328904, 13.343620225507788, 13.341512852489782]\n",
      "Code = 58\n",
      "Close price is: [45.060000000000002, 45.330696782354075, 45.1603436242805, 45.438430796335645, 45.503511224486786]\n",
      "Code = 59\n",
      "Close price is: [41.969999999999999, 41.732841006017466, 41.636407573401776, 41.455351015931171, 41.592806543653737]\n",
      "Code = 50\n",
      "Close price is: [79.200000000000003, 79.420437155649211, 79.230967442564662, 79.655676945660787, 79.246900781616858]\n",
      "Code = 690\n",
      "Close price is: [21.539999999999999, 21.591588066615575, 21.573380268302447, 21.599492763042122, 21.70082108982055]\n",
      "Code = 692\n",
      "Close price is: [20.84, 20.887223214563672, 20.797201648306551, 20.93693729516653, 20.857625546909148]\n",
      "Code = 701\n",
      "Close price is: [20.859999999999999, 21.02939390832778, 20.856477097139344, 21.078398397247177, 20.937664304397892]\n",
      "Code = 713\n",
      "Close price is: [30.120000000000001, 30.063028229545726, 30.124802640102036, 30.033786178689351, 30.193109733766079]\n"
     ]
    }
   ],
   "source": [
    "#define the etf list and the prediction date\n",
    "CodeData = tetfp['Code'].value_counts()\n",
    "Codelst = [list(CodeData.index)]\n",
    "Datelst =['20180430','20180502','20180503','20180504']\n",
    "\n",
    "#start learning\n",
    "for CodeKey in Codelst[0]:\n",
    "    #get the training data\n",
    "    df = tetfp[tetfp.Code == CodeKey].copy().reset_index()\n",
    "    #define the date list and volume list for prediction\n",
    "    Datelst2 = []\n",
    "    Vol = []\n",
    "    \n",
    "    #define all prediction output\n",
    "    Indexlst = [CodeData.loc[CodeKey]]\n",
    "    Openlst = [df.loc[max(df.index),'Open']]\n",
    "    Highlst = [df.loc[max(df.index),'High']]\n",
    "    Lowlst = [df.loc[max(df.index),'Low']]\n",
    "    Closelst = [df.loc[max(df.index),'Close']]\n",
    "    Volumnlst = [str(df.loc[max(df.index),'Volumn'])] \n",
    "    \n",
    "    #start prediction and learning\n",
    "    for i in range(len(Datelst)):        \n",
    "        Datelst2.append(Datelst[i])\n",
    "        \n",
    "        #for first prediction\n",
    "        if i == 0:\n",
    "            new = pd.DataFrame({'index':Indexlst,\n",
    "                               'Code':[CodeKey],\n",
    "                               'Date':Datelst2,\n",
    "                               'Company':['predict'],\n",
    "                               'Open':Openlst,\n",
    "                               'High':Highlst,\n",
    "                               'Low':Lowlst,\n",
    "                               'Close':Closelst,\n",
    "                                'Volumn':Volumnlst}\n",
    "                               ,index=Indexlst)\n",
    "        #for prediction from the second\n",
    "        else:\n",
    "            new = pd.DataFrame({'index':Indexlst,\n",
    "                               'Code':[CodeKey],\n",
    "                               'Date':Datelst2,\n",
    "                               'Company':['predict'],\n",
    "                               'Open':Openlst,\n",
    "                               'High':Highlst,\n",
    "                               'Low':Lowlst,\n",
    "                               'Close':Closelst,\n",
    "                                'Volumn':Vol}\n",
    "                               ,index=Indexlst)\n",
    "\n",
    "        tetfp_run = df\n",
    "        tetfp_run = tetfp_run.append(new)\n",
    "        \n",
    "        #data preprocessing\n",
    "        data = Preprocess(tetfp_run)\n",
    "        if i == 0:\n",
    "            Vol.append(str(data.loc[max(data.index),'Volumn']))\n",
    "        \n",
    "        #start learning\n",
    "        O, H, L, C, V = learning(data)\n",
    "        \n",
    "        #store the output\n",
    "        Indexlst.append(Indexlst[i]+1)\n",
    "        Openlst.append(O[0])\n",
    "        Highlst.append(H[0])\n",
    "        Lowlst.append(L[0])\n",
    "        Closelst.append(C[0])\n",
    "        Vol.append(V[0])\n",
    "        \n",
    "    print('Code =',CodeKey)\n",
    "    print('Close price is:',Closelst)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}