new version

Author: wabyking

config/grid_search_cnn.ini

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+[COMMON]
+model = lstm;basic_cnn;kim_cnn;multi_cnn;inception_cnn;fasttext;rcnn;bilstm
+keep_dropout=0.8;0.9
+batch_size=64;32;128
+learning_rate=0.01;0.001
+optimizer = adam;rmsprop
+dataset = imdb

config/imdb.ini

@@ -1,3 +1,3 @@
 [COMMON]
-dataset = imdb
+dataset = imdb;sst
 

dataHelper.py

@@ -9,7 +9,6 @@
 import random
 import time
 from utils import log_time_delta
-from tqdm import tqdm
 from dataloader import Dataset
 import torch
 from torch.autograd import Variable

main.py

@@ -4,80 +4,145 @@
 from __future__ import division
 from __future__ import print_function
 
-import torch
-from torch.autograd import Variable
-import torch.optim as optim
 import numpy as np
-
+import pandas as pd
 from six.moves import cPickle
+import time,os,random
+import itertools
 
-import opts
-import models
+import torch
+from torch.autograd import Variable
+import torch.optim as optim
 import torch.nn as nn
-import utils
 import torch.nn.functional as F
-from torchtext import data
-from torchtext import datasets
-from torchtext.vocab import Vectors, GloVe, CharNGram, FastText
 from torch.nn.modules.loss import NLLLoss,MultiLabelSoftMarginLoss,MultiLabelMarginLoss,BCELoss
-import dataHelper
-import time,os
-
-
-from_torchtext = False
-
-opt = opts.parse_opt()
-#opt.proxy="http://xxxx.xxxx.com:8080"
-
-
-if "CUDA_VISIBLE_DEVICES" not in os.environ.keys():
-    os.environ["CUDA_VISIBLE_DEVICES"] =opt.gpu
-#opt.model ='lstm'
-#opt.model ='capsule'
 
-if from_torchtext:
-    train_iter, test_iter = utils.loadData(opt)
-else:
-    import dataHelper as helper
-    train_iter, test_iter = dataHelper.loadData(opt)
-
-opt.lstm_layers=2
-
-model=models.setup(opt)
-if torch.cuda.is_available():
-    model.cuda()
-model.train()
-print("# parameters:", sum(param.numel() for param in model.parameters() if param.requires_grad))
-optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=opt.learning_rate)
-optimizer.zero_grad()
-loss_fun = F.cross_entropy
-
-#batch = next(iter(train_iter))
-
-#x=batch.text[0]
-
-#x=batch.text[0] #64x200
-
-#print(utils.evaluation(model,test_iter))
-for i in range(opt.max_epoch):
-    for epoch,batch in enumerate(train_iter):
-        start= time.time()
- 
-        text = batch.text[0] if from_torchtext else batch.text
-        predicted = model(text)
+import opts
+import models
+import utils
 
-        loss= loss_fun(predicted,batch.label)
 
-        loss.backward()
-        utils.clip_gradient(optimizer, opt.grad_clip)
-        optimizer.step()
-        if epoch% 100==0:
-            if  torch.cuda.is_available():
-                print("%d iteration %d epoch with loss : %.5f in %.4f seconds" % (i,epoch,loss.cpu().item(),time.time()-start))
-            else:
-                print("%d iteration %d epoch with loss : %.5f in %.4f seconds" % (i,epoch,loss.data.numpy()[0],time.time()-start))
+timeStamp = time.strftime("%Y%m%d%H%M%S", time.localtime(int(time.time()) ))
+performance_log_file =  os.path.join("log","result"+timeStamp+ ".csv") 
+if not os.path.exists(performance_log_file):
+    with open(performance_log_file,"w") as f:
+        f.write("argument\n")
+        f.close() 
+      
+        
+def train(opt,train_iter, test_iter,verbose=True):
+    global_start= time.time()
+    logger = utils.getLogger()
+    model=models.setup(opt)
+    if torch.cuda.is_available():
+        model.cuda()
+    params = [param for param in model.parameters() if param.requires_grad] #filter(lambda p: p.requires_grad, model.parameters())
+    
+    model_info =";".join( [str(k)+":"+ str(v)  for k,v in opt.__dict__.items() if type(v) in (str,int,float,list,bool)])  
+    logger.info("# parameters:" + str(sum(param.numel() for param in params)))
+    logger.info(model_info)
+    
+    
+    model.train()
+    optimizer = utils.getOptimizer(params,name=opt.optimizer, lr=opt.learning_rate,scheduler= utils.get_lr_scheduler(opt.lr_scheduler))
+    optimizer.zero_grad()
+    loss_fun = F.cross_entropy
+
+    filename = None
+    percisions=[]
+    for i in range(opt.max_epoch):
+        for epoch,batch in enumerate(train_iter):
+            start= time.time()
+            
+            text = batch.text[0] if opt.from_torchtext else batch.text
+            predicted = model(text)
+    
+            loss= loss_fun(predicted,batch.label)
+    
+            loss.backward()
+            utils.clip_gradient(optimizer, opt.grad_clip)
+            optimizer.step()
+            
+            if verbose:
+                if  torch.cuda.is_available():
+                    logger.info("%d iteration %d epoch with loss : %.5f in %.4f seconds" % (i,epoch,loss.cpu().data.numpy(),time.time()-start))
+                else:
+                    logger.info("%d iteration %d epoch with loss : %.5f in %.4f seconds" % (i,epoch,loss.data.numpy()[0],time.time()-start))
 
-    percision=utils.evaluation(model,test_iter,from_torchtext)
-    print("%d iteration with percision %.4f" % (i,percision))
-
-
+        percision=utils.evaluation(model,test_iter,opt.from_torchtext)
+        if verbose:
+            logger.info("%d iteration with percision %.4f" % (i,percision))
+        if len(percisions)==0 or percision > max(percisions):
+            if filename:
+                os.remove(filename)
+            filename = model.save(metric=percision)
+        percisions.append(percision)
+            
+#    while(utils.is_writeable(performance_log_file)):
+    df = pd.read_csv(performance_log_file,index_col=0,sep="\t")
+    df.loc[model_info,opt.dataset] =  max(percisions) 
+    df.to_csv(performance_log_file,sep="\t")    
+    logger.info(model_info +" with time :"+ str( time.time()-global_start)+" ->" +str( max(percisions) ) )
+    print(model_info +" with time :"+ str( time.time()-global_start)+" ->" +str( max(percisions) ) )
+
+def main():
+    from_torchtext = False
+    if "CUDA_VISIBLE_DEVICES" not in os.environ.keys():
+        os.environ["CUDA_VISIBLE_DEVICES"] =opt.gpu
+    #opt.model ='lstm'
+    #opt.model ='capsule'    
+    if from_torchtext:
+        train_iter, test_iter = utils.loadData(opt)
+    else:
+        import dataHelper 
+        train_iter, test_iter = dataHelper.loadData(opt)
+
+    model=models.setup(opt)
+    print(opt.model)
+    if torch.cuda.is_available():
+        model.cuda()
+    
+
+    
+    train(opt,train_iter, test_iter)
+        
+if __name__=="__main__": 
+    parameter_pools = utils.parse_grid_parameters("config/grid_search_cnn.ini")
+    
+#    parameter_pools={
+#            "model":["lstm","cnn","fasttext"],
+#            "keep_dropout":[0.8,0.9,1.0],
+#            "batch_size":[32,64,128],
+#            "learning_rate":[100,10,1,1e-1,1e-2,1e-3],
+#            "optimizer":["adam"],
+#            "lr_scheduler":[None]            
+#                        }    
+    opt = opts.parse_opt()
+    if "CUDA_VISIBLE_DEVICES" not in os.environ.keys():
+        os.environ["CUDA_VISIBLE_DEVICES"] =opt.gpu
+    train_iter, test_iter = utils.loadData(opt)
+#    if from_torchtext:
+#        train_iter, test_iter = utils.loadData(opt)
+#    else:
+#        import dataHelper 
+#        train_iter, test_iter = dataHelper.loadData(opt)
+    if False:
+        model=models.setup(opt)
+        print(opt.model)
+        if torch.cuda.is_available():
+            model.cuda()
+        train(opt,train_iter, test_iter)
+    else:
+        
+        pool =[ arg for arg in itertools.product(*parameter_pools.values())]
+        random.shuffle(pool)
+        args=[arg for i,arg in enumerate(pool) if i%opt.gpu_num==opt.gpu]
+        
+        for arg in args:
+            olddataset = opt.dataset
+            for k,v in zip(parameter_pools.keys(),arg):
+                opt.__setattr__(k,v)
+            if "dataset" in parameter_pools and olddataset != opt.dataset:
+                train_iter, test_iter = utils.loadData(opt)
+            train(opt,train_iter, test_iter,verbose=False)
+   

models/BERTFast.py

@@ -3,16 +3,18 @@
 import numpy as np
 from torch import nn
 from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM
-
-class BERTFast(nn.Module): 
+from models.BaseModel import BaseModel
+class BERTFast(BaseModel): 
     def __init__(self, opt ):
-        super(BERTFast, self).__init__()
+        super(BERTFast, self).__init__(opt)
         self.model_name = 'bert'
         self.opt=opt
 
         self.fc = nn.Linear(768, opt.label_size)
 
-        self.bert_model = BertModel.from_pretrained('bert-base-uncased')
+        self.bert_model = BertModel.from_pretrained('bert-base-uncased')  
+        for param in self.bert_model.parameters():
+            param.requires_grad=self.opt.bert_trained
         self.content_fc = nn.Sequential(
             nn.Linear(768,100),
             nn.BatchNorm1d(100),
@@ -22,12 +24,16 @@ def __init__(self, opt ):
             # nn.ReLU(inplace=True),
             nn.Linear(100,opt.label_size)
         )
+        self.hidden2label = nn.Linear(768, opt.label_size)
+        self.properties.update(
+                {"bert_trained":self.opt.bert_trained
+                })
 
 
     def forward(self,  content):
         encoded, _ = self.bert_model(content)
         encoded_doc = t.mean(encoded[-1],dim=1)
-        logits = self.content_fc(encoded_doc)
+        logits = self.hidden2label(encoded_doc)
         return logits
 
 import argparse

models/BaseModel.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+import torch as t
+
+import numpy as np
+from torch import nn
+from collections import OrderedDict
+import os
+class BaseModel(nn.Module):
+    def __init__(self, opt ):
+        super(BaseModel, self).__init__()
+        self.model_name = 'BaseModel'
+        self.opt=opt
+        
+        self.encoder = nn.Embedding(opt.vocab_size,opt.embedding_dim)
+        if opt.__dict__.get("embeddings",None) is not None:
+            self.encoder.weight=nn.Parameter(opt.embeddings,requires_grad=opt.embedding_training)
+        self.fc = nn.Linear(opt.embedding_dim, opt.label_size)
+        
+        self.properties = {"model_name":self.__class__.__name__,
+                "embedding_dim":self.opt.embedding_dim,
+                "embedding_training":self.opt.embedding_training,
+                "max_seq_len":self.opt.max_seq_len,
+                "batch_size":self.opt.batch_size,
+                "learning_rate":self.opt.learning_rate,
+                "keep_dropout":self.opt.keep_dropout,
+                }
+ 
+    def forward(self,content):
+        content_=t.mean(self.encoder(content),dim=1)
+        out=self.fc(content_.view(content_.size(0),-1))
+        return out
+    
+
+    
+    def save(self,save_dir="saved_model",metric=None):
+        if not os.path.exists(save_dir):
+            os.mkdir(save_dir)
+        self.model_info = "__".join([k+"_"+str(v) if type(v)!=list else k+"_"+str(v)[1:-1].replace(",","_").replace(",","")  for k,v in self.properties.items() ])
+        if metric:
+            path = os.path.join(save_dir, str(metric) +"__"+ self.model_info)
+        else:
+            path = os.path.join(save_dir,self.model_info)
+        t.save(self,path)
+        return path
+    
+
+        
+if __name__ == '__main__':
+    import sys
+    sys.path.append(r"..")
+    import opts
+    opt=opts.parse_opt()
+    opt.vocab_size=2501
+    opt.embedding_dim=300
+    opt.label_size=3
+    m = BaseModel(opt)
+
+    content = t.autograd.Variable(t.arange(0,2500).view(10,250)).long()
+    o = m(content)
+    print(o.size())
+    path = m.save()