diff --git a/.gitignore b/.gitignore index b7fa6fb..284a6b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ *.pyc *.swp* + +**/*.json +**/*.zip \ No newline at end of file diff --git a/Annotations/download.sh b/Annotations/download.sh new file mode 100644 index 0000000..c86c8e0 --- /dev/null +++ b/Annotations/download.sh @@ -0,0 +1,13 @@ +URL=https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa +TRAIN=v2_Annotations_Train_mscoco.zip +VAL=v2_Annotations_Val_mscoco.zip + +if [ ! -f $TRAIN ]; then + wget $URL/$TRAIN +fi +if [ ! -f $VAL ]; then + wget $URL/$VAL +fi + +unzip $TRAIN +unzip $VAL diff --git a/Images/download.sh b/Images/download.sh new file mode 100644 index 0000000..30c8691 --- /dev/null +++ b/Images/download.sh @@ -0,0 +1,18 @@ +URL=http://images.cocodataset.org/zips +TRAIN=train2014.zip +VAL=val2014.zip +TEST=test2015.zip + +if [ ! -f $TRAIN ]; then + wget $URL/$TRAIN +fi +if [ ! -f $VAL ]; then + wget $URL/$VAL +fi +if [ ! -f $TEST ]; then + wget $URL/$TEST +fi + +unzip $TRAIN +unzip $VAL +unzip $TEST diff --git a/PythonEvaluationTools/vqaEvalDemo.py b/PythonEvaluationTools/vqaEvalDemo.py deleted file mode 100644 index 07ca21d..0000000 --- a/PythonEvaluationTools/vqaEvalDemo.py +++ /dev/null @@ -1,89 +0,0 @@ -# coding: utf-8 - -import sys -dataDir = '../../VQA' -sys.path.insert(0, '%s/PythonHelperTools/vqaTools' %(dataDir)) -from vqa import VQA -from vqaEvaluation.vqaEval import VQAEval -import matplotlib.pyplot as plt -import skimage.io as io -import json -import random -import os - -# set up file names and paths -versionType ='v2_' # this should be '' when using VQA v2.0 dataset -taskType ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 -dataType ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. -dataSubType ='train2014' -annFile ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType) -quesFile ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType) -imgDir ='%s/Images/%s/%s/' %(dataDir, dataType, dataSubType) -resultType ='fake' -fileTypes = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] - -# An example result json file has been provided in './Results' folder. - -[resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['%s/Results/%s%s_%s_%s_%s_%s.json'%(dataDir, versionType, taskType, dataType, dataSubType, \ -resultType, fileType) for fileType in fileTypes] - -# create vqa object and vqaRes object -vqa = VQA(annFile, quesFile) -vqaRes = vqa.loadRes(resFile, quesFile) - -# create vqaEval object by taking vqa and vqaRes -vqaEval = VQAEval(vqa, vqaRes, n=2) #n is precision of accuracy (number of places after decimal), default is 2 - -# evaluate results -""" -If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function -By default it uses all the question ids in annotation file -""" -vqaEval.evaluate() - -# print accuracies -print "\n" -print "Overall Accuracy is: %.02f\n" %(vqaEval.accuracy['overall']) -print "Per Question Type Accuracy is the following:" -for quesType in vqaEval.accuracy['perQuestionType']: - print "%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType]) -print "\n" -print "Per Answer Type Accuracy is the following:" -for ansType in vqaEval.accuracy['perAnswerType']: - print "%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType]) -print "\n" -# demo how to use evalQA to retrieve low score result -evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35] #35 is per question percentage accuracy -if len(evals) > 0: - print 'ground truth answers' - randomEval = random.choice(evals) - randomAnn = vqa.loadQA(randomEval) - vqa.showQA(randomAnn) - - print '\n' - print 'generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval]) - ann = vqaRes.loadQA(randomEval)[0] - print "Answer: %s\n" %(ann['answer']) - - imgId = randomAnn[0]['image_id'] - imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' - if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - -# plot accuracy for various question types -plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center') -plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10) -plt.title('Per Question Type Accuracy', fontsize=10) -plt.xlabel('Question Types', fontsize=10) -plt.ylabel('Accuracy', fontsize=10) -plt.show() - -# save evaluation results to ./Results folder -json.dump(vqaEval.accuracy, open(accuracyFile, 'w')) -json.dump(vqaEval.evalQA, open(evalQAFile, 'w')) -json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w')) -json.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w')) - diff --git a/PythonEvaluationTools/vqaEvaluation/__init__.py b/PythonEvaluationTools/vqaEvaluation/__init__.py deleted file mode 100644 index 148424d..0000000 --- a/PythonEvaluationTools/vqaEvaluation/__init__.py +++ /dev/null @@ -1 +0,0 @@ -author='aagrawal' diff --git a/PythonEvaluationTools/vqaEvaluation/vqaEval.py b/PythonEvaluationTools/vqaEvaluation/vqaEval.py deleted file mode 100644 index 120382b..0000000 --- a/PythonEvaluationTools/vqaEvaluation/vqaEval.py +++ /dev/null @@ -1,185 +0,0 @@ -# coding=utf-8 - -__author__='aagrawal' - -# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: -# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py). -import sys -import re - -class VQAEval: - def __init__(self, vqa, vqaRes, n=2): - self.n = n - self.accuracy = {} - self.evalQA = {} - self.evalQuesType = {} - self.evalAnsType = {} - self.vqa = vqa - self.vqaRes = vqaRes - self.params = {'question_id': vqa.getQuesIds()} - self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", \ - "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", \ - "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", \ - "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", \ - "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", \ - "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", \ - "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", \ - "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", \ - "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", \ - "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", \ - "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", \ - "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", \ - "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", \ - "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", \ - "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", \ - "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", \ - "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", \ - "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", \ - "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", \ - "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", \ - "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", \ - "youll": "you'll", "youre": "you're", "youve": "you've"} - self.manualMap = { 'none': '0', - 'zero': '0', - 'one': '1', - 'two': '2', - 'three': '3', - 'four': '4', - 'five': '5', - 'six': '6', - 'seven': '7', - 'eight': '8', - 'nine': '9', - 'ten': '10' - } - self.articles = ['a', - 'an', - 'the' - ] - - - self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") - self.commaStrip = re.compile("(\d)(\,)(\d)") - self.punct = [';', r"/", '[', ']', '"', '{', '}', - '(', ')', '=', '+', '\\', '_', '-', - '>', '<', '@', '`', ',', '?', '!'] - - - def evaluate(self, quesIds=None): - if quesIds == None: - quesIds = [quesId for quesId in self.params['question_id']] - gts = {} - res = {} - for quesId in quesIds: - gts[quesId] = self.vqa.qa[quesId] - res[quesId] = self.vqaRes.qa[quesId] - - # ================================================= - # Compute accuracy - # ================================================= - accQA = [] - accQuesType = {} - accAnsType = {} - print "computing accuracy" - step = 0 - for quesId in quesIds: - resAns = res[quesId]['answer'] - resAns = resAns.replace('\n', ' ') - resAns = resAns.replace('\t', ' ') - resAns = resAns.strip() - resAns = self.processPunctuation(resAns) - resAns = self.processDigitArticle(resAns) - gtAcc = [] - gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']] - if len(set(gtAnswers)) > 1: - for ansDic in gts[quesId]['answers']: - ansDic['answer'] = self.processPunctuation(ansDic['answer']) - for gtAnsDatum in gts[quesId]['answers']: - otherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum] - matchingAns = [item for item in otherGTAns if item['answer']==resAns] - acc = min(1, float(len(matchingAns))/3) - gtAcc.append(acc) - quesType = gts[quesId]['question_type'] - ansType = gts[quesId]['answer_type'] - avgGTAcc = float(sum(gtAcc))/len(gtAcc) - accQA.append(avgGTAcc) - if quesType not in accQuesType: - accQuesType[quesType] = [] - accQuesType[quesType].append(avgGTAcc) - if ansType not in accAnsType: - accAnsType[ansType] = [] - accAnsType[ansType].append(avgGTAcc) - self.setEvalQA(quesId, avgGTAcc) - self.setEvalQuesType(quesId, quesType, avgGTAcc) - self.setEvalAnsType(quesId, ansType, avgGTAcc) - if step%100 == 0: - self.updateProgress(step/float(len(quesIds))) - step = step + 1 - - self.setAccuracy(accQA, accQuesType, accAnsType) - print "Done computing accuracy" - - def processPunctuation(self, inText): - outText = inText - for p in self.punct: - if (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None): - outText = outText.replace(p, '') - else: - outText = outText.replace(p, ' ') - outText = self.periodStrip.sub("", - outText, - re.UNICODE) - return outText - - def processDigitArticle(self, inText): - outText = [] - tempText = inText.lower().split() - for word in tempText: - word = self.manualMap.setdefault(word, word) - if word not in self.articles: - outText.append(word) - else: - pass - for wordId, word in enumerate(outText): - if word in self.contractions: - outText[wordId] = self.contractions[word] - outText = ' '.join(outText) - return outText - - def setAccuracy(self, accQA, accQuesType, accAnsType): - self.accuracy['overall'] = round(100*float(sum(accQA))/len(accQA), self.n) - self.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} - self.accuracy['perAnswerType'] = {ansType: round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} - - def setEvalQA(self, quesId, acc): - self.evalQA[quesId] = round(100*acc, self.n) - - def setEvalQuesType(self, quesId, quesType, acc): - if quesType not in self.evalQuesType: - self.evalQuesType[quesType] = {} - self.evalQuesType[quesType][quesId] = round(100*acc, self.n) - - def setEvalAnsType(self, quesId, ansType, acc): - if ansType not in self.evalAnsType: - self.evalAnsType[ansType] = {} - self.evalAnsType[ansType][quesId] = round(100*acc, self.n) - - def updateProgress(self, progress): - barLength = 20 - status = "" - if isinstance(progress, int): - progress = float(progress) - if not isinstance(progress, float): - progress = 0 - status = "error: progress var must be float\r\n" - if progress < 0: - progress = 0 - status = "Halt...\r\n" - if progress >= 1: - progress = 1 - status = "Done...\r\n" - block = int(round(barLength*progress)) - text = "\rFinshed Percent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), int(progress*100), status) - sys.stdout.write(text) - sys.stdout.flush() - diff --git a/PythonHelperTools/vqaDemo.py b/PythonHelperTools/vqaDemo.py deleted file mode 100644 index 406b596..0000000 --- a/PythonHelperTools/vqaDemo.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding: utf-8 - -from vqaTools.vqa import VQA -import random -import skimage.io as io -import matplotlib.pyplot as plt -import os - -dataDir ='../../VQA' -versionType ='v2_' # this should be '' when using VQA v2.0 dataset -taskType ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 -dataType ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. -dataSubType ='train2014' -annFile ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType) -quesFile ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType) -imgDir = '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType) - -# initialize VQA api for QA annotations -vqa=VQA(annFile, quesFile) - -# load and display QA annotations for given question types -""" -All possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder. -""" -annIds = vqa.getQuesIds(quesTypes='how many'); -anns = vqa.loadQA(annIds) -randomAnn = random.choice(anns) -vqa.showQA([randomAnn]) -imgId = randomAnn['image_id'] -imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' -if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - -# load and display QA annotations for given answer types -""" -ansTypes can be one of the following -yes/no -number -other -""" -annIds = vqa.getQuesIds(ansTypes='yes/no'); -anns = vqa.loadQA(annIds) -randomAnn = random.choice(anns) -vqa.showQA([randomAnn]) -imgId = randomAnn['image_id'] -imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' -if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - -# load and display QA annotations for given images -""" -Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[]) -Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types. -""" -ids = vqa.getImgIds() -annIds = vqa.getQuesIds(imgIds=random.sample(ids,5)); -anns = vqa.loadQA(annIds) -randomAnn = random.choice(anns) -vqa.showQA([randomAnn]) -imgId = randomAnn['image_id'] -imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' -if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - diff --git a/PythonHelperTools/vqaTools/__init__.py b/PythonHelperTools/vqaTools/__init__.py deleted file mode 100644 index 072d8d9..0000000 --- a/PythonHelperTools/vqaTools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'aagrawal' diff --git a/PythonHelperTools/vqaTools/vqa.py b/PythonHelperTools/vqaTools/vqa.py deleted file mode 100644 index fa0abeb..0000000 --- a/PythonHelperTools/vqaTools/vqa.py +++ /dev/null @@ -1,178 +0,0 @@ -__author__ = 'aagrawal' -__version__ = '0.9' - -# Interface for accessing the VQA dataset. - -# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: -# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py). - -# The following functions are defined: -# VQA - VQA class that loads VQA annotation file and prepares data structures. -# getQuesIds - Get question ids that satisfy given filter conditions. -# getImgIds - Get image ids that satisfy given filter conditions. -# loadQA - Load questions and answers with the specified question ids. -# showQA - Display the specified questions and answers. -# loadRes - Load result file and create result object. - -# Help on each function can be accessed by: "help(COCO.function)" - -import json -import datetime -import copy - -class VQA: - def __init__(self, annotation_file=None, question_file=None): - """ - Constructor of VQA helper class for reading and visualizing questions and answers. - :param annotation_file (str): location of VQA annotation file - :return: - """ - # load dataset - self.dataset = {} - self.questions = {} - self.qa = {} - self.qqa = {} - self.imgToQA = {} - if not annotation_file == None and not question_file == None: - print 'loading VQA annotations and questions into memory...' - time_t = datetime.datetime.utcnow() - dataset = json.load(open(annotation_file, 'r')) - questions = json.load(open(question_file, 'r')) - print datetime.datetime.utcnow() - time_t - self.dataset = dataset - self.questions = questions - self.createIndex() - - def createIndex(self): - # create index - print 'creating index...' - imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} - qa = {ann['question_id']: [] for ann in self.dataset['annotations']} - qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} - for ann in self.dataset['annotations']: - imgToQA[ann['image_id']] += [ann] - qa[ann['question_id']] = ann - for ques in self.questions['questions']: - qqa[ques['question_id']] = ques - print 'index created!' - - # create class members - self.qa = qa - self.qqa = qqa - self.imgToQA = imgToQA - - def info(self): - """ - Print information about the VQA annotation file. - :return: - """ - for key, value in self.datset['info'].items(): - print '%s: %s'%(key, value) - - def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): - """ - Get question ids that satisfy given filter conditions. default skips that filter - :param imgIds (int array) : get question ids for given imgs - quesTypes (str array) : get question ids for given question types - ansTypes (str array) : get question ids for given answer types - :return: ids (int array) : integer array of question ids - """ - imgIds = imgIds if type(imgIds) == list else [imgIds] - quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] - ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] - - if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: - anns = self.dataset['annotations'] - else: - if not len(imgIds) == 0: - anns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],[]) - else: - anns = self.dataset['annotations'] - anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] - anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] - ids = [ann['question_id'] for ann in anns] - return ids - - def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): - """ - Get image ids that satisfy given filter conditions. default skips that filter - :param quesIds (int array) : get image ids for given question ids - quesTypes (str array) : get image ids for given question types - ansTypes (str array) : get image ids for given answer types - :return: ids (int array) : integer array of image ids - """ - quesIds = quesIds if type(quesIds) == list else [quesIds] - quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] - ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] - - if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: - anns = self.dataset['annotations'] - else: - if not len(quesIds) == 0: - anns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa],[]) - else: - anns = self.dataset['annotations'] - anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] - anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] - ids = [ann['image_id'] for ann in anns] - return ids - - def loadQA(self, ids=[]): - """ - Load questions and answers with the specified question ids. - :param ids (int array) : integer ids specifying question ids - :return: qa (object array) : loaded qa objects - """ - if type(ids) == list: - return [self.qa[id] for id in ids] - elif type(ids) == int: - return [self.qa[ids]] - - def showQA(self, anns): - """ - Display the specified annotations. - :param anns (array of object): annotations to display - :return: None - """ - if len(anns) == 0: - return 0 - for ann in anns: - quesId = ann['question_id'] - print "Question: %s" %(self.qqa[quesId]['question']) - for ans in ann['answers']: - print "Answer %d: %s" %(ans['answer_id'], ans['answer']) - - def loadRes(self, resFile, quesFile): - """ - Load result file and return a result object. - :param resFile (str) : file name of result file - :return: res (obj) : result api object - """ - res = VQA() - res.questions = json.load(open(quesFile)) - res.dataset['info'] = copy.deepcopy(self.questions['info']) - res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) - res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) - res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype']) - res.dataset['license'] = copy.deepcopy(self.questions['license']) - - print 'Loading and preparing results... ' - time_t = datetime.datetime.utcnow() - anns = json.load(open(resFile)) - assert type(anns) == list, 'results is not an array of objects' - annsQuesIds = [ann['question_id'] for ann in anns] - assert set(annsQuesIds) == set(self.getQuesIds()), \ - 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' - for ann in anns: - quesId = ann['question_id'] - if res.dataset['task_type'] == 'Multiple Choice': - assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' - qaAnn = self.qa[quesId] - ann['image_id'] = qaAnn['image_id'] - ann['question_type'] = qaAnn['question_type'] - ann['answer_type'] = qaAnn['answer_type'] - print 'DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds()) - - res.dataset['annotations'] = anns - res.createIndex() - return res diff --git a/Questions/download.sh b/Questions/download.sh new file mode 100644 index 0000000..b00ce0d --- /dev/null +++ b/Questions/download.sh @@ -0,0 +1,18 @@ +URL=https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa +TRAIN=v2_Questions_Train_mscoco.zip +VAL=v2_Questions_Val_mscoco.zip +TEST=v2_Questions_Test_mscoco.zip + +if [ ! -f $TRAIN ]; then + wget $URL/$TRAIN +fi +if [ ! -f $VAL ]; then + wget $URL/$VAL +fi +if [ ! -f $TEST ]; then + wget $URL/$TEST +fi + +unzip $TRAIN +unzip $VAL +unzip $TEST diff --git a/README.md b/README.md index 439d59d..4255971 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ There are two types of tasks - Multiple-choice task (18 choices per question) ## Requirements ## -- python 2.7 +- python 3.5+ - scikit-image (visit [this page](http://scikit-image.org/docs/dev/install.html) for installation) - matplotlib (visit [this page](http://matplotlib.org/users/installing.html) for installation) diff --git a/vqa/__init__.py b/vqa/__init__.py new file mode 100644 index 0000000..2631dbb --- /dev/null +++ b/vqa/__init__.py @@ -0,0 +1,2 @@ +from .vqa import VQA +from .vqaEval import VQAEval diff --git a/vqa/vqa.py b/vqa/vqa.py new file mode 100644 index 0000000..8b3865d --- /dev/null +++ b/vqa/vqa.py @@ -0,0 +1,188 @@ +__author__ = 'aagrawal' +__version__ = '0.9' + +# Interface for accessing the VQA dataset. + +# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: +# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py). + +# The following functions are defined: +# VQA - VQA class that loads VQA annotation file and prepares data structures. +# getQuesIds - Get question ids that satisfy given filter conditions. +# getImgIds - Get image ids that satisfy given filter conditions. +# loadQA - Load questions and answers with the specified question ids. +# showQA - Display the specified questions and answers. +# loadRes - Load result file and create result object. + +# Help on each function can be accessed by: "help(COCO.function)" + +import json +import datetime +import copy + + +class VQA: + def __init__(self, annotation_file=None, question_file=None): + """ + Constructor of VQA helper class for reading and visualizing questions and answers. + :param annotation_file (str): location of VQA annotation file + :return: + """ + # load dataset + self.dataset = {} + self.questions = {} + self.qa = {} + self.qqa = {} + self.imgToQA = {} + if not annotation_file == None and not question_file == None: + print('loading VQA annotations and questions into memory...') + time_t = datetime.datetime.utcnow() + dataset = json.load(open(annotation_file, 'r')) + questions = json.load(open(question_file, 'r')) + print(datetime.datetime.utcnow() - time_t) + self.dataset = dataset + self.questions = questions + self.createIndex() + + def createIndex(self): + # create index + print('creating index...') + imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} + qa = {ann['question_id']: [] for ann in self.dataset['annotations']} + qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} + for ann in self.dataset['annotations']: + imgToQA[ann['image_id']] += [ann] + qa[ann['question_id']] = ann + for ques in self.questions['questions']: + qqa[ques['question_id']] = ques + print('index created!') + + # create class members + self.qa = qa + self.qqa = qqa + self.imgToQA = imgToQA + + def info(self): + """ + Print information about the VQA annotation file. + :return: + """ + for key, value in self.dataset['info'].items(): + print('%s: %s' % (key, value)) + + def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): + """ + Get question ids that satisfy given filter conditions. default skips that filter + :param + imgIds (int array) : get question ids for given imgs + quesTypes (str array) : get question ids for given question types + ansTypes (str array) : get question ids for given answer types + :return: ids (int array) : integer array of question ids + """ + imgIds = imgIds if type(imgIds) == list else [imgIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + anns = sum([self.imgToQA[imgId] + for imgId in imgIds if imgId in self.imgToQA], []) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ + ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ + ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['question_id'] for ann in anns] + return ids + + def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): + """ + Get image ids that satisfy given filter conditions. default skips that filter + :param quesIds (int array) : get image ids for given question ids + quesTypes (str array) : get image ids for given question types + ansTypes (str array) : get image ids for given answer types + :return: ids (int array) : integer array of image ids + """ + quesIds = quesIds if type(quesIds) == list else [quesIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(quesIds) == 0: + anns = sum([self.qa[quesId] + for quesId in quesIds if quesId in self.qa], []) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ + ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ + ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['image_id'] for ann in anns] + return ids + + def loadQA(self, ids=[]): + """ + Load questions and answers with the specified question ids. + :param ids (int array) : integer ids specifying question ids + :return: qa (object array) : loaded qa objects + """ + if type(ids) == list: + return [self.qa[id] for id in ids] + elif type(ids) == int: + return [self.qa[ids]] + + def showQA(self, anns): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + for ann in anns: + quesId = ann['question_id'] + print("Question: %s" % (self.qqa[quesId]['question'])) + for ans in ann['answers']: + print("Answer %d: %s" % (ans['answer_id'], ans['answer'])) + + def loadRes(self, resFile, quesFile): + """ + Load result file and return a result object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = VQA() + res.questions = json.load(open(quesFile)) + res.dataset['info'] = copy.deepcopy(self.questions['info']) + res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) + res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) + res.dataset['data_subtype'] = copy.deepcopy( + self.questions['data_subtype']) + res.dataset['license'] = copy.deepcopy(self.questions['license']) + + print('Loading and preparing results... ') + time_t = datetime.datetime.utcnow() + anns = json.load(open(resFile)) + assert type(anns) == list, 'results is not an array of objects' + annsQuesIds = [ann['question_id'] for ann in anns] + assert set(annsQuesIds) == set(self.getQuesIds()), \ + 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' + for ann in anns: + quesId = ann['question_id'] + if res.dataset['task_type'] == 'Multiple Choice': + assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' + qaAnn = self.qa[quesId] + ann['image_id'] = qaAnn['image_id'] + ann['question_type'] = qaAnn['question_type'] + ann['answer_type'] = qaAnn['answer_type'] + print('DONE (t=%0.2fs)' % + ((datetime.datetime.utcnow() - time_t).total_seconds())) + + res.dataset['annotations'] = anns + res.createIndex() + return res diff --git a/vqa/vqaEval.py b/vqa/vqaEval.py new file mode 100644 index 0000000..6d61986 --- /dev/null +++ b/vqa/vqaEval.py @@ -0,0 +1,187 @@ +# coding=utf-8 + +__author__ = 'aagrawal' + +# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: +# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py). +import sys +import re + + +class VQAEval: + def __init__(self, vqa, vqaRes, n=2): + self.n = n + self.accuracy = {} + self.evalQA = {} + self.evalQuesType = {} + self.evalAnsType = {} + self.vqa = vqa + self.vqaRes = vqaRes + self.params = {'question_id': vqa.getQuesIds()} + self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", + "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", + "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", + "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", + "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", + "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", + "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", + "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", + "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", + "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", + "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", + "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", + "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", + "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", + "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", + "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", + "youll": "you'll", "youre": "you're", "youve": "you've"} + self.manualMap = {'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10' + } + self.articles = ['a', + 'an', + 'the' + ] + + self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") + self.commaStrip = re.compile("(\d)(\,)(\d)") + self.punct = [';', r"/", '[', ']', '"', '{', '}', + '(', ')', '=', '+', '\\', '_', '-', + '>', '<', '@', '`', ',', '?', '!'] + + def evaluate(self, quesIds=None): + if quesIds == None: + quesIds = [quesId for quesId in self.params['question_id']] + gts = {} + res = {} + for quesId in quesIds: + gts[quesId] = self.vqa.qa[quesId] + res[quesId] = self.vqaRes.qa[quesId] + + # ================================================= + # Compute accuracy + # ================================================= + accQA = [] + accQuesType = {} + accAnsType = {} + print("computing accuracy") + step = 0 + for quesId in quesIds: + resAns = res[quesId]['answer'] + resAns = resAns.replace('\n', ' ') + resAns = resAns.replace('\t', ' ') + resAns = resAns.strip() + resAns = self.processPunctuation(resAns) + resAns = self.processDigitArticle(resAns) + gtAcc = [] + gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']] + if len(set(gtAnswers)) > 1: + for ansDic in gts[quesId]['answers']: + ansDic['answer'] = self.processPunctuation( + ansDic['answer']) + for gtAnsDatum in gts[quesId]['answers']: + otherGTAns = [item for item in gts[quesId] + ['answers'] if item != gtAnsDatum] + matchingAns = [ + item for item in otherGTAns if item['answer'] == resAns] + acc = min(1, float(len(matchingAns))/3) + gtAcc.append(acc) + quesType = gts[quesId]['question_type'] + ansType = gts[quesId]['answer_type'] + avgGTAcc = float(sum(gtAcc))/len(gtAcc) + accQA.append(avgGTAcc) + if quesType not in accQuesType: + accQuesType[quesType] = [] + accQuesType[quesType].append(avgGTAcc) + if ansType not in accAnsType: + accAnsType[ansType] = [] + accAnsType[ansType].append(avgGTAcc) + self.setEvalQA(quesId, avgGTAcc) + self.setEvalQuesType(quesId, quesType, avgGTAcc) + self.setEvalAnsType(quesId, ansType, avgGTAcc) + if step % 100 == 0: + self.updateProgress(step/float(len(quesIds))) + step = step + 1 + + self.setAccuracy(accQA, accQuesType, accAnsType) + print("Done computing accuracy") + + def processPunctuation(self, inText): + outText = inText + for p in self.punct: + if (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None): + outText = outText.replace(p, '') + else: + outText = outText.replace(p, ' ') + outText = self.periodStrip.sub("", + outText, + re.UNICODE) + return outText + + def processDigitArticle(self, inText): + outText = [] + tempText = inText.lower().split() + for word in tempText: + word = self.manualMap.setdefault(word, word) + if word not in self.articles: + outText.append(word) + else: + pass + for wordId, word in enumerate(outText): + if word in self.contractions: + outText[wordId] = self.contractions[word] + outText = ' '.join(outText) + return outText + + def setAccuracy(self, accQA, accQuesType, accAnsType): + self.accuracy['overall'] = round( + 100*float(sum(accQA))/len(accQA), self.n) + self.accuracy['perQuestionType'] = {quesType: round( + 100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} + self.accuracy['perAnswerType'] = {ansType: round( + 100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} + + def setEvalQA(self, quesId, acc): + self.evalQA[quesId] = round(100*acc, self.n) + + def setEvalQuesType(self, quesId, quesType, acc): + if quesType not in self.evalQuesType: + self.evalQuesType[quesType] = {} + self.evalQuesType[quesType][quesId] = round(100*acc, self.n) + + def setEvalAnsType(self, quesId, ansType, acc): + if ansType not in self.evalAnsType: + self.evalAnsType[ansType] = {} + self.evalAnsType[ansType][quesId] = round(100*acc, self.n) + + def updateProgress(self, progress): + barLength = 20 + status = "" + if isinstance(progress, int): + progress = float(progress) + if not isinstance(progress, float): + progress = 0 + status = "error: progress var must be float\r\n" + if progress < 0: + progress = 0 + status = "Halt...\r\n" + if progress >= 1: + progress = 1 + status = "Done...\r\n" + block = int(round(barLength*progress)) + text = "\rFinshed Percent: [{0}] {1}% {2}".format( + "#"*block + "-"*(barLength-block), int(progress*100), status) + sys.stdout.write(text) + sys.stdout.flush() diff --git a/vqaDemo.py b/vqaDemo.py new file mode 100644 index 0000000..337115b --- /dev/null +++ b/vqaDemo.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +from vqa import VQA +import random +import skimage.io as io +import matplotlib.pyplot as plt +import os + +dataDir = '../../VQA' +versionType = 'v2_' # this should be '' when not using VQA v2.0 dataset +# 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +taskType = 'OpenEnded' +# 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. +dataType = 'mscoco' +dataSubType = 'train2014' +annFile = '%s/Annotations/%s%s_%s_annotations.json' % ( + dataDir, versionType, dataType, dataSubType) +quesFile = '%s/Questions/%s%s_%s_%s_questions.json' % ( + dataDir, versionType, taskType, dataType, dataSubType) +imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) + +# initialize VQA api for QA annotations +vqa = VQA(annFile, quesFile) + +# load and display QA annotations for given question types +""" +All possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder. +""" +annIds = vqa.getQuesIds(quesTypes='how many') +anns = vqa.loadQA(annIds) +randomAnn = random.choice(anns) +vqa.showQA([randomAnn]) +imgId = randomAnn['image_id'] +imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' +if os.path.isfile(imgDir + imgFilename): + I = io.imread(imgDir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() + +# load and display QA annotations for given answer types +""" +ansTypes can be one of the following +yes/no +number +other +""" +annIds = vqa.getQuesIds(ansTypes='yes/no') +anns = vqa.loadQA(annIds) +randomAnn = random.choice(anns) +vqa.showQA([randomAnn]) +imgId = randomAnn['image_id'] +imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' +if os.path.isfile(imgDir + imgFilename): + I = io.imread(imgDir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() + +# load and display QA annotations for given images +""" +Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[]) +Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types. +""" +ids = vqa.getImgIds() +annIds = vqa.getQuesIds(imgIds=random.sample(ids, 5)) +anns = vqa.loadQA(annIds) +randomAnn = random.choice(anns) +vqa.showQA([randomAnn]) +imgId = randomAnn['image_id'] +imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' +if os.path.isfile(imgDir + imgFilename): + I = io.imread(imgDir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() diff --git a/vqa_evaluate.py b/vqa_evaluate.py new file mode 100644 index 0000000..86a524e --- /dev/null +++ b/vqa_evaluate.py @@ -0,0 +1,123 @@ +# coding: utf-8 + +import sys +import os.path as osp +from vqa import VQA, VQAEval +import matplotlib.pyplot as plt +import skimage.io as io +import json +import random +import os +import argparse +import pprint + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--results") + parser.add_argument("--annotations") + parser.add_argument("--questions") + parser.add_argument("--image_dir") + parser.add_argument("--data_dir", default=".") + parser.add_argument("--version", default="v2") + parser.add_argument("--task", default="OpenEnded") + parser.add_argument("--data", default="mscoco") + parser.add_argument("--data_subtype", default="val2014") + + return parser.parse_args() + + +args = parse_args() + +pprint.pprint(vars(args)) + +# set up file names and paths +data_dir = args.data_dir +version = args.version # this should be 'v2' when using VQA v2.0 dataset +# 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +task = args.task +# 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. +data = args.data +data_subtype = args.data_subtype + +ann_file = args.annotations or '{args.data_dir}/Annotations/{args.version}_{args.data}_{args.data_subtype}_annotations.json'.format( + args=args) +ques_file = args.questions or '{args.data_dir}/Questions/{args.version}_{args.task}_{args.data}_{args.data_subtype}_questions.json'.format( + args=args) + +img_dir = args.image_dir or '{args.data_dir}/Images/{args.data}/{args.data_subtype}/'.format( + args=args) +file_types = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] + +# An example result json file has been provided in './Results' folder. + +file_template = "{data_dir}/Results/{version}_{task}_{data}_{data_subtype}_{file_type}.json" +[res_file, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = [file_template.format( + data_dir=data_dir, version=version, task=task, data=data, data_subtype=data_subtype, file_type=file_type) for file_type in file_types] + +res_file = args.results or res_file + +# create vqa object and vqaRes object +vqa = VQA(ann_file, ques_file) +vqaRes = vqa.loadRes(res_file, ques_file) + +# create vqaEval object by taking vqa and vqaRes +# n is precision of accuracy (number of places after decimal), default is 2 +vqaEval = VQAEval(vqa, vqaRes, n=3) + +# evaluate results +""" +If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function +By default it uses all the question ids in annotation file +""" +vqaEval.evaluate() + +# print accuracies +print("\n") +print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) +print("Per Question Type Accuracy is the following:") +for quesType in vqaEval.accuracy['perQuestionType']: + print("%s : %.02f" % + (quesType, vqaEval.accuracy['perQuestionType'][quesType])) +print("\n") +print("Per Answer Type Accuracy is the following:") +for ansType in vqaEval.accuracy['perAnswerType']: + print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) +print("\n") +# demo how to use evalQA to retrieve low score result +# 35 is per question percentage accuracy +evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId] < 35] +if len(evals) > 0: + print('ground truth answers') + randomEval = random.choice(evals) + randomAnn = vqa.loadQA(randomEval) + vqa.showQA(randomAnn) + + print('\n') + print('generated answer (accuracy %.02f)' % (vqaEval.evalQA[randomEval])) + ann = vqaRes.loadQA(randomEval)[0] + print("Answer: %s\n" % (ann['answer'])) + + imgId = randomAnn[0]['image_id'] + imgFilename = 'COCO_' + data_subtype + '_' + str(imgId).zfill(12) + '.jpg' + if os.path.isfile(img_dir + imgFilename): + I = io.imread(img_dir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() + +# plot accuracy for various question types +plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), + vqaEval.accuracy['perQuestionType'].values(), align='center') +plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), + vqaEval.accuracy['perQuestionType'].keys(), rotation='0', fontsize=10) +plt.title('Per Question Type Accuracy', fontsize=10) +plt.xlabel('Question Types', fontsize=10) +plt.ylabel('Accuracy', fontsize=10) +plt.show() + +# save evaluation results to ./Results folder +json.dump(vqaEval.accuracy, open(accuracyFile, 'w')) +json.dump(vqaEval.evalQA, open(evalQAFile, 'w')) +json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w')) +json.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w'))