From 1579c351146748248b3ce29a4ea3f44e1bbda59d Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Tue, 29 May 2018 13:07:44 -0400 Subject: [PATCH 1/7] updated to work with Python 3 and fixed inconsistent tabs --- .../vqaEvaluation/vqaEval.py | 331 +++++++++--------- PythonHelperTools/vqaTools/vqa.py | 299 ++++++++-------- 2 files changed, 314 insertions(+), 316 deletions(-) diff --git a/PythonEvaluationTools/vqaEvaluation/vqaEval.py b/PythonEvaluationTools/vqaEvaluation/vqaEval.py index 120382b..50c38ce 100644 --- a/PythonEvaluationTools/vqaEvaluation/vqaEval.py +++ b/PythonEvaluationTools/vqaEvaluation/vqaEval.py @@ -8,178 +8,175 @@ import re class VQAEval: - def __init__(self, vqa, vqaRes, n=2): - self.n = n - self.accuracy = {} - self.evalQA = {} - self.evalQuesType = {} - self.evalAnsType = {} - self.vqa = vqa - self.vqaRes = vqaRes - self.params = {'question_id': vqa.getQuesIds()} - self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", \ - "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", \ - "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", \ - "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", \ - "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", \ - "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", \ - "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", \ - "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", \ - "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", \ - "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", \ - "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", \ - "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", \ - "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", \ - "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", \ - "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", \ - "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", \ - "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", \ - "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", \ - "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", \ - "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", \ - "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", \ - "youll": "you'll", "youre": "you're", "youve": "you've"} - self.manualMap = { 'none': '0', - 'zero': '0', - 'one': '1', - 'two': '2', - 'three': '3', - 'four': '4', - 'five': '5', - 'six': '6', - 'seven': '7', - 'eight': '8', - 'nine': '9', - 'ten': '10' - } - self.articles = ['a', - 'an', - 'the' - ] + def __init__(self, vqa, vqaRes, n=2): + self.n = n + self.accuracy = {} + self.evalQA = {} + self.evalQuesType = {} + self.evalAnsType = {} + self.vqa = vqa + self.vqaRes = vqaRes + self.params = {'question_id': vqa.getQuesIds()} + self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", \ + "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", \ + "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", \ + "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", \ + "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", \ + "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", \ + "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", \ + "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", \ + "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", \ + "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", \ + "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", \ + "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", \ + "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", \ + "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", \ + "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", \ + "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", \ + "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", \ + "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", \ + "youll": "you'll", "youre": "you're", "youve": "you've"} + self.manualMap = { 'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10' + } + self.articles = ['a', + 'an', + 'the' + ] - self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") - self.commaStrip = re.compile("(\d)(\,)(\d)") - self.punct = [';', r"/", '[', ']', '"', '{', '}', - '(', ')', '=', '+', '\\', '_', '-', - '>', '<', '@', '`', ',', '?', '!'] + self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") + self.commaStrip = re.compile("(\d)(\,)(\d)") + self.punct = [';', r"/", '[', ']', '"', '{', '}', + '(', ')', '=', '+', '\\', '_', '-', + '>', '<', '@', '`', ',', '?', '!'] - - def evaluate(self, quesIds=None): - if quesIds == None: - quesIds = [quesId for quesId in self.params['question_id']] - gts = {} - res = {} - for quesId in quesIds: - gts[quesId] = self.vqa.qa[quesId] - res[quesId] = self.vqaRes.qa[quesId] - - # ================================================= - # Compute accuracy - # ================================================= - accQA = [] - accQuesType = {} - accAnsType = {} - print "computing accuracy" - step = 0 - for quesId in quesIds: - resAns = res[quesId]['answer'] - resAns = resAns.replace('\n', ' ') - resAns = resAns.replace('\t', ' ') - resAns = resAns.strip() - resAns = self.processPunctuation(resAns) - resAns = self.processDigitArticle(resAns) - gtAcc = [] - gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']] - if len(set(gtAnswers)) > 1: - for ansDic in gts[quesId]['answers']: - ansDic['answer'] = self.processPunctuation(ansDic['answer']) - for gtAnsDatum in gts[quesId]['answers']: - otherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum] - matchingAns = [item for item in otherGTAns if item['answer']==resAns] - acc = min(1, float(len(matchingAns))/3) - gtAcc.append(acc) - quesType = gts[quesId]['question_type'] - ansType = gts[quesId]['answer_type'] - avgGTAcc = float(sum(gtAcc))/len(gtAcc) - accQA.append(avgGTAcc) - if quesType not in accQuesType: - accQuesType[quesType] = [] - accQuesType[quesType].append(avgGTAcc) - if ansType not in accAnsType: - accAnsType[ansType] = [] - accAnsType[ansType].append(avgGTAcc) - self.setEvalQA(quesId, avgGTAcc) - self.setEvalQuesType(quesId, quesType, avgGTAcc) - self.setEvalAnsType(quesId, ansType, avgGTAcc) - if step%100 == 0: - self.updateProgress(step/float(len(quesIds))) - step = step + 1 + + def evaluate(self, quesIds=None): + if quesIds == None: + quesIds = [quesId for quesId in self.params['question_id']] + gts = {} + res = {} + for quesId in quesIds: + gts[quesId] = self.vqa.qa[quesId] + res[quesId] = self.vqaRes.qa[quesId] + + # ================================================= + # Compute accuracy + # ================================================= + accQA = [] + accQuesType = {} + accAnsType = {} + print("computing accuracy") + step = 0 + for quesId in quesIds: + resAns = res[quesId]['answer'] + resAns = resAns.replace('\n', ' ') + resAns = resAns.replace('\t', ' ') + resAns = resAns.strip() + resAns = self.processPunctuation(resAns) + resAns = self.processDigitArticle(resAns) + gtAcc = [] + gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']] + if len(set(gtAnswers)) > 1: + for ansDic in gts[quesId]['answers']: + ansDic['answer'] = self.processPunctuation(ansDic['answer']) + for gtAnsDatum in gts[quesId]['answers']: + otherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum] + matchingAns = [item for item in otherGTAns if item['answer']==resAns] + acc = min(1, float(len(matchingAns))/3) + gtAcc.append(acc) + quesType = gts[quesId]['question_type'] + ansType = gts[quesId]['answer_type'] + avgGTAcc = float(sum(gtAcc))/len(gtAcc) + accQA.append(avgGTAcc) + if quesType not in accQuesType: + accQuesType[quesType] = [] + accQuesType[quesType].append(avgGTAcc) + if ansType not in accAnsType: + accAnsType[ansType] = [] + accAnsType[ansType].append(avgGTAcc) + self.setEvalQA(quesId, avgGTAcc) + self.setEvalQuesType(quesId, quesType, avgGTAcc) + self.setEvalAnsType(quesId, ansType, avgGTAcc) + if step%100 == 0: + self.updateProgress(step/float(len(quesIds))) + step = step + 1 - self.setAccuracy(accQA, accQuesType, accAnsType) - print "Done computing accuracy" - - def processPunctuation(self, inText): - outText = inText - for p in self.punct: - if (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None): - outText = outText.replace(p, '') - else: - outText = outText.replace(p, ' ') - outText = self.periodStrip.sub("", - outText, - re.UNICODE) - return outText - - def processDigitArticle(self, inText): - outText = [] - tempText = inText.lower().split() - for word in tempText: - word = self.manualMap.setdefault(word, word) - if word not in self.articles: - outText.append(word) - else: - pass - for wordId, word in enumerate(outText): - if word in self.contractions: - outText[wordId] = self.contractions[word] - outText = ' '.join(outText) - return outText + self.setAccuracy(accQA, accQuesType, accAnsType) + print("Done computing accuracy") + + def processPunctuation(self, inText): + outText = inText + for p in self.punct: + if (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None): + outText = outText.replace(p, '') + else: + outText = outText.replace(p, ' ') + outText = self.periodStrip.sub("", + outText, + re.UNICODE) + return outText + + def processDigitArticle(self, inText): + outText = [] + tempText = inText.lower().split() + for word in tempText: + word = self.manualMap.setdefault(word, word) + if word not in self.articles: + outText.append(word) + else: + pass + for wordId, word in enumerate(outText): + if word in self.contractions: + outText[wordId] = self.contractions[word] + outText = ' '.join(outText) + return outText - def setAccuracy(self, accQA, accQuesType, accAnsType): - self.accuracy['overall'] = round(100*float(sum(accQA))/len(accQA), self.n) - self.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} - self.accuracy['perAnswerType'] = {ansType: round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} - - def setEvalQA(self, quesId, acc): - self.evalQA[quesId] = round(100*acc, self.n) + def setAccuracy(self, accQA, accQuesType, accAnsType): + self.accuracy['overall'] = round(100*float(sum(accQA))/len(accQA), self.n) + self.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} + self.accuracy['perAnswerType'] = {ansType: round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} + + def setEvalQA(self, quesId, acc): + self.evalQA[quesId] = round(100*acc, self.n) - def setEvalQuesType(self, quesId, quesType, acc): - if quesType not in self.evalQuesType: - self.evalQuesType[quesType] = {} - self.evalQuesType[quesType][quesId] = round(100*acc, self.n) - - def setEvalAnsType(self, quesId, ansType, acc): - if ansType not in self.evalAnsType: - self.evalAnsType[ansType] = {} - self.evalAnsType[ansType][quesId] = round(100*acc, self.n) + def setEvalQuesType(self, quesId, quesType, acc): + if quesType not in self.evalQuesType: + self.evalQuesType[quesType] = {} + self.evalQuesType[quesType][quesId] = round(100*acc, self.n) + + def setEvalAnsType(self, quesId, ansType, acc): + if ansType not in self.evalAnsType: + self.evalAnsType[ansType] = {} + self.evalAnsType[ansType][quesId] = round(100*acc, self.n) - def updateProgress(self, progress): - barLength = 20 - status = "" - if isinstance(progress, int): - progress = float(progress) - if not isinstance(progress, float): - progress = 0 - status = "error: progress var must be float\r\n" - if progress < 0: - progress = 0 - status = "Halt...\r\n" - if progress >= 1: - progress = 1 - status = "Done...\r\n" - block = int(round(barLength*progress)) - text = "\rFinshed Percent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), int(progress*100), status) - sys.stdout.write(text) - sys.stdout.flush() + def updateProgress(self, progress): + barLength = 20 + status = "" + if isinstance(progress, int): + progress = float(progress) + if not isinstance(progress, float): + progress = 0 + status = "error: progress var must be float\r\n" + if progress < 0: + progress = 0 + status = "Halt...\r\n" + if progress >= 1: + progress = 1 + status = "Done...\r\n" + block = int(round(barLength*progress)) + text = "\rFinshed Percent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), int(progress*100), status) + sys.stdout.write(text) + sys.stdout.flush() diff --git a/PythonHelperTools/vqaTools/vqa.py b/PythonHelperTools/vqaTools/vqa.py index fa0abeb..a63ba41 100644 --- a/PythonHelperTools/vqaTools/vqa.py +++ b/PythonHelperTools/vqaTools/vqa.py @@ -21,158 +21,159 @@ import copy class VQA: - def __init__(self, annotation_file=None, question_file=None): - """ - Constructor of VQA helper class for reading and visualizing questions and answers. + def __init__(self, annotation_file=None, question_file=None): + """ + Constructor of VQA helper class for reading and visualizing questions and answers. :param annotation_file (str): location of VQA annotation file :return: - """ + """ # load dataset - self.dataset = {} - self.questions = {} - self.qa = {} - self.qqa = {} - self.imgToQA = {} - if not annotation_file == None and not question_file == None: - print 'loading VQA annotations and questions into memory...' - time_t = datetime.datetime.utcnow() - dataset = json.load(open(annotation_file, 'r')) - questions = json.load(open(question_file, 'r')) - print datetime.datetime.utcnow() - time_t - self.dataset = dataset - self.questions = questions - self.createIndex() - - def createIndex(self): + self.dataset = {} + self.questions = {} + self.qa = {} + self.qqa = {} + self.imgToQA = {} + if not annotation_file == None and not question_file == None: + print('loading VQA annotations and questions into memory...') + time_t = datetime.datetime.utcnow() + dataset = json.load(open(annotation_file, 'r')) + questions = json.load(open(question_file, 'r')) + print(datetime.datetime.utcnow() - time_t) + self.dataset = dataset + self.questions = questions + self.createIndex() + + def createIndex(self): # create index - print 'creating index...' - imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} - qa = {ann['question_id']: [] for ann in self.dataset['annotations']} - qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} - for ann in self.dataset['annotations']: - imgToQA[ann['image_id']] += [ann] - qa[ann['question_id']] = ann - for ques in self.questions['questions']: - qqa[ques['question_id']] = ques - print 'index created!' - - # create class members - self.qa = qa - self.qqa = qqa - self.imgToQA = imgToQA - - def info(self): - """ - Print information about the VQA annotation file. - :return: - """ - for key, value in self.datset['info'].items(): - print '%s: %s'%(key, value) - - def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): - """ - Get question ids that satisfy given filter conditions. default skips that filter - :param imgIds (int array) : get question ids for given imgs - quesTypes (str array) : get question ids for given question types - ansTypes (str array) : get question ids for given answer types - :return: ids (int array) : integer array of question ids - """ - imgIds = imgIds if type(imgIds) == list else [imgIds] - quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] - ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] - - if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: - anns = self.dataset['annotations'] - else: - if not len(imgIds) == 0: - anns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],[]) - else: - anns = self.dataset['annotations'] - anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] - anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] - ids = [ann['question_id'] for ann in anns] - return ids - - def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): - """ - Get image ids that satisfy given filter conditions. default skips that filter - :param quesIds (int array) : get image ids for given question ids + print('creating index...') + imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} + qa = {ann['question_id']: [] for ann in self.dataset['annotations']} + qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} + for ann in self.dataset['annotations']: + imgToQA[ann['image_id']] += [ann] + qa[ann['question_id']] = ann + for ques in self.questions['questions']: + qqa[ques['question_id']] = ques + print('index created!') + + # create class members + self.qa = qa + self.qqa = qqa + self.imgToQA = imgToQA + + def info(self): + """ + Print information about the VQA annotation file. + :return: + """ + for key, value in self.datset['info'].items(): + print('%s: %s'%(key, value)) + + def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): + """ + Get question ids that satisfy given filter conditions. default skips that filter + :param + imgIds (int array) : get question ids for given imgs + quesTypes (str array) : get question ids for given question types + ansTypes (str array) : get question ids for given answer types + :return: ids (int array) : integer array of question ids + """ + imgIds = imgIds if type(imgIds) == list else [imgIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + anns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],[]) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['question_id'] for ann in anns] + return ids + + def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): + """ + Get image ids that satisfy given filter conditions. default skips that filter + :param quesIds (int array) : get image ids for given question ids quesTypes (str array) : get image ids for given question types ansTypes (str array) : get image ids for given answer types - :return: ids (int array) : integer array of image ids - """ - quesIds = quesIds if type(quesIds) == list else [quesIds] - quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] - ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] - - if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: - anns = self.dataset['annotations'] - else: - if not len(quesIds) == 0: - anns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa],[]) - else: - anns = self.dataset['annotations'] - anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] - anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] - ids = [ann['image_id'] for ann in anns] - return ids - - def loadQA(self, ids=[]): - """ - Load questions and answers with the specified question ids. - :param ids (int array) : integer ids specifying question ids - :return: qa (object array) : loaded qa objects - """ - if type(ids) == list: - return [self.qa[id] for id in ids] - elif type(ids) == int: - return [self.qa[ids]] - - def showQA(self, anns): - """ - Display the specified annotations. - :param anns (array of object): annotations to display - :return: None - """ - if len(anns) == 0: - return 0 - for ann in anns: - quesId = ann['question_id'] - print "Question: %s" %(self.qqa[quesId]['question']) - for ans in ann['answers']: - print "Answer %d: %s" %(ans['answer_id'], ans['answer']) - - def loadRes(self, resFile, quesFile): - """ - Load result file and return a result object. - :param resFile (str) : file name of result file - :return: res (obj) : result api object - """ - res = VQA() - res.questions = json.load(open(quesFile)) - res.dataset['info'] = copy.deepcopy(self.questions['info']) - res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) - res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) - res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype']) - res.dataset['license'] = copy.deepcopy(self.questions['license']) - - print 'Loading and preparing results... ' - time_t = datetime.datetime.utcnow() - anns = json.load(open(resFile)) - assert type(anns) == list, 'results is not an array of objects' - annsQuesIds = [ann['question_id'] for ann in anns] - assert set(annsQuesIds) == set(self.getQuesIds()), \ - 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' - for ann in anns: - quesId = ann['question_id'] - if res.dataset['task_type'] == 'Multiple Choice': - assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' - qaAnn = self.qa[quesId] - ann['image_id'] = qaAnn['image_id'] - ann['question_type'] = qaAnn['question_type'] - ann['answer_type'] = qaAnn['answer_type'] - print 'DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds()) - - res.dataset['annotations'] = anns - res.createIndex() - return res + :return: ids (int array) : integer array of image ids + """ + quesIds = quesIds if type(quesIds) == list else [quesIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(quesIds) == 0: + anns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa],[]) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['image_id'] for ann in anns] + return ids + + def loadQA(self, ids=[]): + """ + Load questions and answers with the specified question ids. + :param ids (int array) : integer ids specifying question ids + :return: qa (object array) : loaded qa objects + """ + if type(ids) == list: + return [self.qa[id] for id in ids] + elif type(ids) == int: + return [self.qa[ids]] + + def showQA(self, anns): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + for ann in anns: + quesId = ann['question_id'] + print("Question: %s" %(self.qqa[quesId]['question'])) + for ans in ann['answers']: + print("Answer %d: %s" %(ans['answer_id'], ans['answer'])) + + def loadRes(self, resFile, quesFile): + """ + Load result file and return a result object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = VQA() + res.questions = json.load(open(quesFile)) + res.dataset['info'] = copy.deepcopy(self.questions['info']) + res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) + res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) + res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype']) + res.dataset['license'] = copy.deepcopy(self.questions['license']) + + print('Loading and preparing results... ') + time_t = datetime.datetime.utcnow() + anns = json.load(open(resFile)) + assert type(anns) == list, 'results is not an array of objects' + annsQuesIds = [ann['question_id'] for ann in anns] + assert set(annsQuesIds) == set(self.getQuesIds()), \ + 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' + for ann in anns: + quesId = ann['question_id'] + if res.dataset['task_type'] == 'Multiple Choice': + assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' + qaAnn = self.qa[quesId] + ann['image_id'] = qaAnn['image_id'] + ann['question_type'] = qaAnn['question_type'] + ann['answer_type'] = qaAnn['answer_type'] + print('DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds())) + + res.dataset['annotations'] = anns + res.createIndex() + return res From e42f8f578357aa976588b7feef0da029d0ec8dad Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Tue, 29 May 2018 13:08:04 -0400 Subject: [PATCH 2/7] evaluation script for VQA 2 dataset --- PythonEvaluationTools/vqa_eval.py | 105 ++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 PythonEvaluationTools/vqa_eval.py diff --git a/PythonEvaluationTools/vqa_eval.py b/PythonEvaluationTools/vqa_eval.py new file mode 100644 index 0000000..f1b6a86 --- /dev/null +++ b/PythonEvaluationTools/vqa_eval.py @@ -0,0 +1,105 @@ +# coding: utf-8 + +import sys +root_dir = '../../VQA' +sys.path.insert(0, '%s/PythonHelperTools/vqaTools' %(root_dir)) +from vqa import VQA +from vqaEvaluation.vqaEval import VQAEval +import matplotlib.pyplot as plt +import skimage.io as io +import json +import random +import os +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("results") + parser.add_argument("annotations") + parser.add_argument("questions") + parser.add_argument("image_dir") + parser.add_argument("--data_dir") + + return parser.parse_args() + + +args = parse_args() + +# set up file names and paths +data_dir = args.data_dir +version ='v2' # this should be 'v2' when using VQA v2.0 dataset +task ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +data ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. +data_subtype ='val2014' +ann_file = args.annotations or '%s/Annotations/%s%s_%s_annotations.json'%(data_dir, version, data, data_subtype) +ques_file = args.questions or '%s/Questions/%s%s_%s_%s_questions.json'%(data_dir, version, task, data, data_subtype) +img_dir = args.image_dir or '%s/Images/%s/%s/' %(data_dir, data, data_subtype) +file_types = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] + +# An example result json file has been provided in './Results' folder. + +file_template = "{data_dir}/results/{version}_{task}_{data}_{data_subtype}_{file_type}.json" +[res_file, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = [file_template.format(data_dir=data_dir, version=version, task=task, data=data, data_subtype=data_subtype, file_type=file_type) for file_type in file_types] + +res_file = args.results or res_file + +# create vqa object and vqaRes object +vqa = VQA(ann_file, ques_file) +vqaRes = vqa.loadRes(res_file, ques_file) + +# create vqaEval object by taking vqa and vqaRes +vqaEval = VQAEval(vqa, vqaRes, n=3) # n is precision of accuracy (number of places after decimal), default is 2 + +# evaluate results +""" +If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function +By default it uses all the question ids in annotation file +""" +vqaEval.evaluate() + +# print accuracies +print("\n") +print("Overall Accuracy is: %.02f\n" %(vqaEval.accuracy['overall'])) +print("Per Question Type Accuracy is the following:") +for quesType in vqaEval.accuracy['perQuestionType']: + print("%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType])) +print("\n") +print("Per Answer Type Accuracy is the following:") +for ansType in vqaEval.accuracy['perAnswerType']: + print("%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType])) +print("\n") +# demo how to use evalQA to retrieve low score result +evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35] #35 is per question percentage accuracy +if len(evals) > 0: + print('ground truth answers') + randomEval = random.choice(evals) + randomAnn = vqa.loadQA(randomEval) + vqa.showQA(randomAnn) + + print('\n') + print('generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval])) + ann = vqaRes.loadQA(randomEval)[0] + print("Answer: %s\n" %(ann['answer'])) + + imgId = randomAnn[0]['image_id'] + imgFilename = 'COCO_' + data_subtype + '_'+ str(imgId).zfill(12) + '.jpg' + if os.path.isfile(img_dir + imgFilename): + I = io.imread(img_dir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() + +# plot accuracy for various question types +plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center') +plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10) +plt.title('Per Question Type Accuracy', fontsize=10) +plt.xlabel('Question Types', fontsize=10) +plt.ylabel('Accuracy', fontsize=10) +plt.show() + +# save evaluation results to ./Results folder +json.dump(vqaEval.accuracy, open(accuracyFile, 'w')) +json.dump(vqaEval.evalQA, open(evalQAFile, 'w')) +json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w')) +json.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w')) + From d97cd208179ca6aa7affed454ac5ada654528ceb Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Thu, 26 Jul 2018 16:08:54 -0400 Subject: [PATCH 3/7] generic path for root_dir and update for v2 --- PythonEvaluationTools/vqa_eval.py | 88 +++++++++++++++++-------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/PythonEvaluationTools/vqa_eval.py b/PythonEvaluationTools/vqa_eval.py index f1b6a86..e67be16 100644 --- a/PythonEvaluationTools/vqa_eval.py +++ b/PythonEvaluationTools/vqa_eval.py @@ -1,8 +1,10 @@ # coding: utf-8 import sys -root_dir = '../../VQA' -sys.path.insert(0, '%s/PythonHelperTools/vqaTools' %(root_dir)) +import os.path as osp +root_dir = osp.join("../../", + osp.basename(osp.dirname(osp.abspath('.')))) +sys.path.insert(0, '%s/PythonHelperTools/vqaTools' % (root_dir)) from vqa import VQA from vqaEvaluation.vqaEval import VQAEval import matplotlib.pyplot as plt @@ -13,7 +15,7 @@ import argparse def parse_args(): - parser = argparse.ArgumentParser() + parser=argparse.ArgumentParser() parser.add_argument("results") parser.add_argument("annotations") parser.add_argument("questions") @@ -23,75 +25,84 @@ def parse_args(): return parser.parse_args() -args = parse_args() +args=parse_args() # set up file names and paths -data_dir = args.data_dir -version ='v2' # this should be 'v2' when using VQA v2.0 dataset -task ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 -data ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. -data_subtype ='val2014' -ann_file = args.annotations or '%s/Annotations/%s%s_%s_annotations.json'%(data_dir, version, data, data_subtype) -ques_file = args.questions or '%s/Questions/%s%s_%s_%s_questions.json'%(data_dir, version, task, data, data_subtype) -img_dir = args.image_dir or '%s/Images/%s/%s/' %(data_dir, data, data_subtype) -file_types = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] - -# An example result json file has been provided in './Results' folder. - -file_template = "{data_dir}/results/{version}_{task}_{data}_{data_subtype}_{file_type}.json" -[res_file, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = [file_template.format(data_dir=data_dir, version=version, task=task, data=data, data_subtype=data_subtype, file_type=file_type) for file_type in file_types] - -res_file = args.results or res_file +data_dir=args.data_dir +version='v2' # this should be 'v2' when using VQA v2.0 dataset +task='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +# 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. +data='mscoco' +data_subtype='val2014' +ann_file=args.annotations or '%s/Annotations/%s%s_%s_annotations.json' % ( + data_dir, version, data, data_subtype) +ques_file=args.questions or '%s/Questions/%s%s_%s_%s_questions.json' % ( + data_dir, version, task, data, data_subtype) +img_dir=args.image_dir or '%s/Images/%s/%s/' % (data_dir, data, data_subtype) +file_types=['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] + +# An example result json file has been provided in './Results' folder. + +file_template="{data_dir}/results/{version}_{task}_{data}_{data_subtype}_{file_type}.json" +[res_file, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile]=[file_template.format( + data_dir=data_dir, version=version, task=task, data=data, data_subtype=data_subtype, file_type=file_type) for file_type in file_types] + +res_file=args.results or res_file # create vqa object and vqaRes object -vqa = VQA(ann_file, ques_file) -vqaRes = vqa.loadRes(res_file, ques_file) +vqa=VQA(ann_file, ques_file) +vqaRes=vqa.loadRes(res_file, ques_file) # create vqaEval object by taking vqa and vqaRes -vqaEval = VQAEval(vqa, vqaRes, n=3) # n is precision of accuracy (number of places after decimal), default is 2 +# n is precision of accuracy (number of places after decimal), default is 2 +vqaEval=VQAEval(vqa, vqaRes, n=3) # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ -vqaEval.evaluate() +vqaEval.evaluate() # print accuracies print("\n") -print("Overall Accuracy is: %.02f\n" %(vqaEval.accuracy['overall'])) +print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) print("Per Question Type Accuracy is the following:") for quesType in vqaEval.accuracy['perQuestionType']: - print("%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType])) + print("%s : %.02f" % + (quesType, vqaEval.accuracy['perQuestionType'][quesType])) print("\n") print("Per Answer Type Accuracy is the following:") for ansType in vqaEval.accuracy['perAnswerType']: - print("%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType])) + print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) print("\n") # demo how to use evalQA to retrieve low score result -evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35] #35 is per question percentage accuracy +# 35 is per question percentage accuracy +evals=[quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId] < 35] if len(evals) > 0: print('ground truth answers') - randomEval = random.choice(evals) - randomAnn = vqa.loadQA(randomEval) + randomEval=random.choice(evals) + randomAnn=vqa.loadQA(randomEval) vqa.showQA(randomAnn) print('\n') - print('generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval])) - ann = vqaRes.loadQA(randomEval)[0] - print("Answer: %s\n" %(ann['answer'])) + print('generated answer (accuracy %.02f)' % (vqaEval.evalQA[randomEval])) + ann=vqaRes.loadQA(randomEval)[0] + print("Answer: %s\n" % (ann['answer'])) - imgId = randomAnn[0]['image_id'] - imgFilename = 'COCO_' + data_subtype + '_'+ str(imgId).zfill(12) + '.jpg' + imgId=randomAnn[0]['image_id'] + imgFilename='COCO_' + data_subtype + '_' + str(imgId).zfill(12) + '.jpg' if os.path.isfile(img_dir + imgFilename): - I = io.imread(img_dir + imgFilename) + I=io.imread(img_dir + imgFilename) plt.imshow(I) plt.axis('off') plt.show() # plot accuracy for various question types -plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center') -plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10) +plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), + vqaEval.accuracy['perQuestionType'].values(), align='center') +plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), + vqaEval.accuracy['perQuestionType'].keys(), rotation='0', fontsize=10) plt.title('Per Question Type Accuracy', fontsize=10) plt.xlabel('Question Types', fontsize=10) plt.ylabel('Accuracy', fontsize=10) @@ -102,4 +113,3 @@ def parse_args(): json.dump(vqaEval.evalQA, open(evalQAFile, 'w')) json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w')) json.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w')) - From fd9af529cbb68bc58bc83560a68b4429d8935889 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Wed, 9 Jan 2019 14:21:44 -0500 Subject: [PATCH 4/7] download scripts for Questions, Annotations and Images --- Annotations/download.sh | 13 +++++++++++++ Images/download.sh | 18 ++++++++++++++++++ Questions/download.sh | 18 ++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 Annotations/download.sh create mode 100644 Images/download.sh create mode 100644 Questions/download.sh diff --git a/Annotations/download.sh b/Annotations/download.sh new file mode 100644 index 0000000..c86c8e0 --- /dev/null +++ b/Annotations/download.sh @@ -0,0 +1,13 @@ +URL=https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa +TRAIN=v2_Annotations_Train_mscoco.zip +VAL=v2_Annotations_Val_mscoco.zip + +if [ ! -f $TRAIN ]; then + wget $URL/$TRAIN +fi +if [ ! -f $VAL ]; then + wget $URL/$VAL +fi + +unzip $TRAIN +unzip $VAL diff --git a/Images/download.sh b/Images/download.sh new file mode 100644 index 0000000..30c8691 --- /dev/null +++ b/Images/download.sh @@ -0,0 +1,18 @@ +URL=http://images.cocodataset.org/zips +TRAIN=train2014.zip +VAL=val2014.zip +TEST=test2015.zip + +if [ ! -f $TRAIN ]; then + wget $URL/$TRAIN +fi +if [ ! -f $VAL ]; then + wget $URL/$VAL +fi +if [ ! -f $TEST ]; then + wget $URL/$TEST +fi + +unzip $TRAIN +unzip $VAL +unzip $TEST diff --git a/Questions/download.sh b/Questions/download.sh new file mode 100644 index 0000000..b00ce0d --- /dev/null +++ b/Questions/download.sh @@ -0,0 +1,18 @@ +URL=https://s3.amazonaws.com/cvmlp/vqa/mscoco/vqa +TRAIN=v2_Questions_Train_mscoco.zip +VAL=v2_Questions_Val_mscoco.zip +TEST=v2_Questions_Test_mscoco.zip + +if [ ! -f $TRAIN ]; then + wget $URL/$TRAIN +fi +if [ ! -f $VAL ]; then + wget $URL/$VAL +fi +if [ ! -f $TEST ]; then + wget $URL/$TEST +fi + +unzip $TRAIN +unzip $VAL +unzip $TEST From e6845dc8e7eb405018577d4e0152a4e5fe8900cb Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Wed, 9 Jan 2019 14:48:54 -0500 Subject: [PATCH 5/7] restructured the vqa evaluation tools so it resembles a python package --- PythonEvaluationTools/vqaEvalDemo.py | 89 -------- .../vqaEvaluation/__init__.py | 1 - PythonHelperTools/vqaDemo.py | 73 ------- vqa/__init__.py | 2 + vqa/vqa.py | 190 ++++++++++++++++++ .../vqaEvaluation => vqa}/vqaEval.py | 153 +++++++------- vqaDemo.py | 76 +++++++ .../vqa_eval.py => vqa_evaluate.py | 72 +++---- 8 files changed, 385 insertions(+), 271 deletions(-) delete mode 100644 PythonEvaluationTools/vqaEvalDemo.py delete mode 100644 PythonEvaluationTools/vqaEvaluation/__init__.py delete mode 100644 PythonHelperTools/vqaDemo.py create mode 100644 vqa/__init__.py create mode 100644 vqa/vqa.py rename {PythonEvaluationTools/vqaEvaluation => vqa}/vqaEval.py (68%) create mode 100644 vqaDemo.py rename PythonEvaluationTools/vqa_eval.py => vqa_evaluate.py (62%) diff --git a/PythonEvaluationTools/vqaEvalDemo.py b/PythonEvaluationTools/vqaEvalDemo.py deleted file mode 100644 index 07ca21d..0000000 --- a/PythonEvaluationTools/vqaEvalDemo.py +++ /dev/null @@ -1,89 +0,0 @@ -# coding: utf-8 - -import sys -dataDir = '../../VQA' -sys.path.insert(0, '%s/PythonHelperTools/vqaTools' %(dataDir)) -from vqa import VQA -from vqaEvaluation.vqaEval import VQAEval -import matplotlib.pyplot as plt -import skimage.io as io -import json -import random -import os - -# set up file names and paths -versionType ='v2_' # this should be '' when using VQA v2.0 dataset -taskType ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 -dataType ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. -dataSubType ='train2014' -annFile ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType) -quesFile ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType) -imgDir ='%s/Images/%s/%s/' %(dataDir, dataType, dataSubType) -resultType ='fake' -fileTypes = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] - -# An example result json file has been provided in './Results' folder. - -[resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['%s/Results/%s%s_%s_%s_%s_%s.json'%(dataDir, versionType, taskType, dataType, dataSubType, \ -resultType, fileType) for fileType in fileTypes] - -# create vqa object and vqaRes object -vqa = VQA(annFile, quesFile) -vqaRes = vqa.loadRes(resFile, quesFile) - -# create vqaEval object by taking vqa and vqaRes -vqaEval = VQAEval(vqa, vqaRes, n=2) #n is precision of accuracy (number of places after decimal), default is 2 - -# evaluate results -""" -If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function -By default it uses all the question ids in annotation file -""" -vqaEval.evaluate() - -# print accuracies -print "\n" -print "Overall Accuracy is: %.02f\n" %(vqaEval.accuracy['overall']) -print "Per Question Type Accuracy is the following:" -for quesType in vqaEval.accuracy['perQuestionType']: - print "%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType]) -print "\n" -print "Per Answer Type Accuracy is the following:" -for ansType in vqaEval.accuracy['perAnswerType']: - print "%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType]) -print "\n" -# demo how to use evalQA to retrieve low score result -evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35] #35 is per question percentage accuracy -if len(evals) > 0: - print 'ground truth answers' - randomEval = random.choice(evals) - randomAnn = vqa.loadQA(randomEval) - vqa.showQA(randomAnn) - - print '\n' - print 'generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval]) - ann = vqaRes.loadQA(randomEval)[0] - print "Answer: %s\n" %(ann['answer']) - - imgId = randomAnn[0]['image_id'] - imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' - if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - -# plot accuracy for various question types -plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center') -plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10) -plt.title('Per Question Type Accuracy', fontsize=10) -plt.xlabel('Question Types', fontsize=10) -plt.ylabel('Accuracy', fontsize=10) -plt.show() - -# save evaluation results to ./Results folder -json.dump(vqaEval.accuracy, open(accuracyFile, 'w')) -json.dump(vqaEval.evalQA, open(evalQAFile, 'w')) -json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w')) -json.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w')) - diff --git a/PythonEvaluationTools/vqaEvaluation/__init__.py b/PythonEvaluationTools/vqaEvaluation/__init__.py deleted file mode 100644 index 148424d..0000000 --- a/PythonEvaluationTools/vqaEvaluation/__init__.py +++ /dev/null @@ -1 +0,0 @@ -author='aagrawal' diff --git a/PythonHelperTools/vqaDemo.py b/PythonHelperTools/vqaDemo.py deleted file mode 100644 index 406b596..0000000 --- a/PythonHelperTools/vqaDemo.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding: utf-8 - -from vqaTools.vqa import VQA -import random -import skimage.io as io -import matplotlib.pyplot as plt -import os - -dataDir ='../../VQA' -versionType ='v2_' # this should be '' when using VQA v2.0 dataset -taskType ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 -dataType ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. -dataSubType ='train2014' -annFile ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType) -quesFile ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType) -imgDir = '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType) - -# initialize VQA api for QA annotations -vqa=VQA(annFile, quesFile) - -# load and display QA annotations for given question types -""" -All possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder. -""" -annIds = vqa.getQuesIds(quesTypes='how many'); -anns = vqa.loadQA(annIds) -randomAnn = random.choice(anns) -vqa.showQA([randomAnn]) -imgId = randomAnn['image_id'] -imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' -if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - -# load and display QA annotations for given answer types -""" -ansTypes can be one of the following -yes/no -number -other -""" -annIds = vqa.getQuesIds(ansTypes='yes/no'); -anns = vqa.loadQA(annIds) -randomAnn = random.choice(anns) -vqa.showQA([randomAnn]) -imgId = randomAnn['image_id'] -imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' -if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - -# load and display QA annotations for given images -""" -Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[]) -Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types. -""" -ids = vqa.getImgIds() -annIds = vqa.getQuesIds(imgIds=random.sample(ids,5)); -anns = vqa.loadQA(annIds) -randomAnn = random.choice(anns) -vqa.showQA([randomAnn]) -imgId = randomAnn['image_id'] -imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' -if os.path.isfile(imgDir + imgFilename): - I = io.imread(imgDir + imgFilename) - plt.imshow(I) - plt.axis('off') - plt.show() - diff --git a/vqa/__init__.py b/vqa/__init__.py new file mode 100644 index 0000000..c728069 --- /dev/null +++ b/vqa/__init__.py @@ -0,0 +1,2 @@ +from vqa.vqa import VQA +from vqa.vqaEval import VQAEval diff --git a/vqa/vqa.py b/vqa/vqa.py new file mode 100644 index 0000000..c519527 --- /dev/null +++ b/vqa/vqa.py @@ -0,0 +1,190 @@ +__author__ = 'aagrawal' +__version__ = '0.9' + +# Interface for accessing the VQA dataset. + +# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: +# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py). + +# The following functions are defined: +# VQA - VQA class that loads VQA annotation file and prepares data structures. +# getQuesIds - Get question ids that satisfy given filter conditions. +# getImgIds - Get image ids that satisfy given filter conditions. +# loadQA - Load questions and answers with the specified question ids. +# showQA - Display the specified questions and answers. +# loadRes - Load result file and create result object. + +# Help on each function can be accessed by: "help(COCO.function)" + +import json +import datetime +import copy + + +class VQA: + def __init__(self, annotation_file=None, question_file=None): + """ + Constructor of VQA helper class for reading and visualizing questions and answers. + :param annotation_file (str): location of VQA annotation file + :return: + """ + # load dataset + self.dataset = {} + self.questions = {} + self.qa = {} + self.qqa = {} + self.imgToQA = {} + if not annotation_file == None and not question_file == None: + print('loading VQA annotations and questions into memory...') + time_t = datetime.datetime.utcnow() + dataset = json.load(open(annotation_file, 'r')) + questions = json.load(open(question_file, 'r')) + print(datetime.datetime.utcnow() - time_t) + self.dataset = dataset + self.questions = questions + self.createIndex() + + def createIndex(self): + # create index + print('creating index...') + imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} + qa = {ann['question_id']: [] + for ann in self.dataset['annotations']} + qqa = {ann['question_id']: [] + for ann in self.dataset['annotations']} + for ann in self.dataset['annotations']: + imgToQA[ann['image_id']] += [ann] + qa[ann['question_id']] = ann + for ques in self.questions['questions']: + qqa[ques['question_id']] = ques + print('index created!') + + # create class members + self.qa = qa + self.qqa = qqa + self.imgToQA = imgToQA + + def info(self): + """ + Print information about the VQA annotation file. + :return: + """ + for key, value in self.datset['info'].items(): + print('%s: %s' % (key, value)) + + def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): + """ + Get question ids that satisfy given filter conditions. default skips that filter + :param + imgIds (int array) : get question ids for given imgs + quesTypes (str array) : get question ids for given question types + ansTypes (str array) : get question ids for given answer types + :return: ids (int array) : integer array of question ids + """ + imgIds = imgIds if type(imgIds) == list else [imgIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + anns = sum([self.imgToQA[imgId] + for imgId in imgIds if imgId in self.imgToQA], []) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ + ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ + ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['question_id'] for ann in anns] + return ids + + def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): + """ + Get image ids that satisfy given filter conditions. default skips that filter + :param quesIds (int array) : get image ids for given question ids + quesTypes (str array) : get image ids for given question types + ansTypes (str array) : get image ids for given answer types + :return: ids (int array) : integer array of image ids + """ + quesIds = quesIds if type(quesIds) == list else [quesIds] + quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] + ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] + + if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: + anns = self.dataset['annotations'] + else: + if not len(quesIds) == 0: + anns = sum([self.qa[quesId] + for quesId in quesIds if quesId in self.qa], []) + else: + anns = self.dataset['annotations'] + anns = anns if len(quesTypes) == 0 else [ + ann for ann in anns if ann['question_type'] in quesTypes] + anns = anns if len(ansTypes) == 0 else [ + ann for ann in anns if ann['answer_type'] in ansTypes] + ids = [ann['image_id'] for ann in anns] + return ids + + def loadQA(self, ids=[]): + """ + Load questions and answers with the specified question ids. + :param ids (int array) : integer ids specifying question ids + :return: qa (object array) : loaded qa objects + """ + if type(ids) == list: + return [self.qa[id] for id in ids] + elif type(ids) == int: + return [self.qa[ids]] + + def showQA(self, anns): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + for ann in anns: + quesId = ann['question_id'] + print("Question: %s" % (self.qqa[quesId]['question'])) + for ans in ann['answers']: + print("Answer %d: %s" % (ans['answer_id'], ans['answer'])) + + def loadRes(self, resFile, quesFile): + """ + Load result file and return a result object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = VQA() + res.questions = json.load(open(quesFile)) + res.dataset['info'] = copy.deepcopy(self.questions['info']) + res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) + res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) + res.dataset['data_subtype'] = copy.deepcopy( + self.questions['data_subtype']) + res.dataset['license'] = copy.deepcopy(self.questions['license']) + + print('Loading and preparing results... ') + time_t = datetime.datetime.utcnow() + anns = json.load(open(resFile)) + assert type(anns) == list, 'results is not an array of objects' + annsQuesIds = [ann['question_id'] for ann in anns] + assert set(annsQuesIds) == set(self.getQuesIds()), \ + 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' + for ann in anns: + quesId = ann['question_id'] + if res.dataset['task_type'] == 'Multiple Choice': + assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' + qaAnn = self.qa[quesId] + ann['image_id'] = qaAnn['image_id'] + ann['question_type'] = qaAnn['question_type'] + ann['answer_type'] = qaAnn['answer_type'] + print('DONE (t=%0.2fs)' % + ((datetime.datetime.utcnow() - time_t).total_seconds())) + + res.dataset['annotations'] = anns + res.createIndex() + return res diff --git a/PythonEvaluationTools/vqaEvaluation/vqaEval.py b/vqa/vqaEval.py similarity index 68% rename from PythonEvaluationTools/vqaEvaluation/vqaEval.py rename to vqa/vqaEval.py index 50c38ce..6d61986 100644 --- a/PythonEvaluationTools/vqaEvaluation/vqaEval.py +++ b/vqa/vqaEval.py @@ -1,67 +1,66 @@ # coding=utf-8 -__author__='aagrawal' +__author__ = 'aagrawal' -# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: +# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: # (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py). import sys import re + class VQAEval: def __init__(self, vqa, vqaRes, n=2): self.n = n self.accuracy = {} self.evalQA = {} self.evalQuesType = {} - self.evalAnsType = {} + self.evalAnsType = {} self.vqa = vqa self.vqaRes = vqaRes self.params = {'question_id': vqa.getQuesIds()} - self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", \ - "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", \ - "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", \ - "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", \ - "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", \ - "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", \ - "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", \ - "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", \ - "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", \ - "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", \ - "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", \ - "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", \ - "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", \ - "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", \ - "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", \ - "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", \ - "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", \ - "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", \ + self.contractions = {"aint": "ain't", "arent": "aren't", "cant": "can't", "couldve": "could've", "couldnt": "couldn't", "couldn'tve": "couldn't've", "couldnt've": "couldn't've", "didnt": "didn't", "doesnt": "doesn't", "dont": "don't", "hadnt": "hadn't", "hadnt've": "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent": "haven't", "hed": "he'd", "hed've": "he'd've", "he'dve": "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll", "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", + "Im": "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've": "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's", + "maam": "ma'am", "mightnt": "mightn't", "mightnt've": "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've", + "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't", "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't", + "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat": "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve": "she'd've", + "she's": "she's", "shouldve": "should've", "shouldnt": "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve": "shouldn't've", + "somebody'd": "somebodyd", "somebodyd've": "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll": "somebody'll", + "somebodys": "somebody's", "someoned": "someone'd", "someoned've": "someone'd've", "someone'dve": "someone'd've", + "someonell": "someone'll", "someones": "someone's", "somethingd": "something'd", "somethingd've": "something'd've", + "something'dve": "something'd've", "somethingll": "something'll", "thats": "that's", "thered": "there'd", "thered've": "there'd've", + "there'dve": "there'd've", "therere": "there're", "theres": "there's", "theyd": "they'd", "theyd've": "they'd've", + "they'dve": "they'd've", "theyll": "they'll", "theyre": "they're", "theyve": "they've", "twas": "'twas", "wasnt": "wasn't", + "wed've": "we'd've", "we'dve": "we'd've", "weve": "we've", "werent": "weren't", "whatll": "what'll", "whatre": "what're", + "whats": "what's", "whatve": "what've", "whens": "when's", "whered": "where'd", "wheres": "where's", "whereve": "where've", + "whod": "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl": "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll", + "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve": "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll": "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd": "you'd", "youd've": "you'd've", "you'dve": "you'd've", "youll": "you'll", "youre": "you're", "youve": "you've"} - self.manualMap = { 'none': '0', - 'zero': '0', - 'one': '1', - 'two': '2', - 'three': '3', - 'four': '4', - 'five': '5', - 'six': '6', - 'seven': '7', - 'eight': '8', - 'nine': '9', - 'ten': '10' - } - self.articles = ['a', - 'an', - 'the' - ] - + self.manualMap = {'none': '0', + 'zero': '0', + 'one': '1', + 'two': '2', + 'three': '3', + 'four': '4', + 'five': '5', + 'six': '6', + 'seven': '7', + 'eight': '8', + 'nine': '9', + 'ten': '10' + } + self.articles = ['a', + 'an', + 'the' + ] - self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") - self.commaStrip = re.compile("(\d)(\,)(\d)") - self.punct = [';', r"/", '[', ']', '"', '{', '}', - '(', ')', '=', '+', '\\', '_', '-', - '>', '<', '@', '`', ',', '?', '!'] + self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)") + self.commaStrip = re.compile("(\d)(\,)(\d)") + self.punct = [';', r"/", '[', ']', '"', '{', '}', + '(', ')', '=', '+', '\\', '_', '-', + '>', '<', '@', '`', ',', '?', '!'] - def evaluate(self, quesIds=None): if quesIds == None: quesIds = [quesId for quesId in self.params['question_id']] @@ -70,34 +69,37 @@ def evaluate(self, quesIds=None): for quesId in quesIds: gts[quesId] = self.vqa.qa[quesId] res[quesId] = self.vqaRes.qa[quesId] - + # ================================================= # Compute accuracy # ================================================= - accQA = [] + accQA = [] accQuesType = {} - accAnsType = {} + accAnsType = {} print("computing accuracy") step = 0 for quesId in quesIds: - resAns = res[quesId]['answer'] - resAns = resAns.replace('\n', ' ') - resAns = resAns.replace('\t', ' ') - resAns = resAns.strip() - resAns = self.processPunctuation(resAns) - resAns = self.processDigitArticle(resAns) - gtAcc = [] + resAns = res[quesId]['answer'] + resAns = resAns.replace('\n', ' ') + resAns = resAns.replace('\t', ' ') + resAns = resAns.strip() + resAns = self.processPunctuation(resAns) + resAns = self.processDigitArticle(resAns) + gtAcc = [] gtAnswers = [ans['answer'] for ans in gts[quesId]['answers']] - if len(set(gtAnswers)) > 1: + if len(set(gtAnswers)) > 1: for ansDic in gts[quesId]['answers']: - ansDic['answer'] = self.processPunctuation(ansDic['answer']) + ansDic['answer'] = self.processPunctuation( + ansDic['answer']) for gtAnsDatum in gts[quesId]['answers']: - otherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum] - matchingAns = [item for item in otherGTAns if item['answer']==resAns] + otherGTAns = [item for item in gts[quesId] + ['answers'] if item != gtAnsDatum] + matchingAns = [ + item for item in otherGTAns if item['answer'] == resAns] acc = min(1, float(len(matchingAns))/3) gtAcc.append(acc) - quesType = gts[quesId]['question_type'] - ansType = gts[quesId]['answer_type'] + quesType = gts[quesId]['question_type'] + ansType = gts[quesId]['answer_type'] avgGTAcc = float(sum(gtAcc))/len(gtAcc) accQA.append(avgGTAcc) if quesType not in accQuesType: @@ -109,13 +111,13 @@ def evaluate(self, quesIds=None): self.setEvalQA(quesId, avgGTAcc) self.setEvalQuesType(quesId, quesType, avgGTAcc) self.setEvalAnsType(quesId, ansType, avgGTAcc) - if step%100 == 0: + if step % 100 == 0: self.updateProgress(step/float(len(quesIds))) step = step + 1 self.setAccuracy(accQA, accQuesType, accAnsType) print("Done computing accuracy") - + def processPunctuation(self, inText): outText = inText for p in self.punct: @@ -124,10 +126,10 @@ def processPunctuation(self, inText): else: outText = outText.replace(p, ' ') outText = self.periodStrip.sub("", - outText, - re.UNICODE) + outText, + re.UNICODE) return outText - + def processDigitArticle(self, inText): outText = [] tempText = inText.lower().split() @@ -138,16 +140,19 @@ def processDigitArticle(self, inText): else: pass for wordId, word in enumerate(outText): - if word in self.contractions: + if word in self.contractions: outText[wordId] = self.contractions[word] outText = ' '.join(outText) return outText def setAccuracy(self, accQA, accQuesType, accAnsType): - self.accuracy['overall'] = round(100*float(sum(accQA))/len(accQA), self.n) - self.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} - self.accuracy['perAnswerType'] = {ansType: round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} - + self.accuracy['overall'] = round( + 100*float(sum(accQA))/len(accQA), self.n) + self.accuracy['perQuestionType'] = {quesType: round( + 100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType} + self.accuracy['perAnswerType'] = {ansType: round( + 100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType} + def setEvalQA(self, quesId, acc): self.evalQA[quesId] = round(100*acc, self.n) @@ -155,7 +160,7 @@ def setEvalQuesType(self, quesId, quesType, acc): if quesType not in self.evalQuesType: self.evalQuesType[quesType] = {} self.evalQuesType[quesType][quesId] = round(100*acc, self.n) - + def setEvalAnsType(self, quesId, ansType, acc): if ansType not in self.evalAnsType: self.evalAnsType[ansType] = {} @@ -176,7 +181,7 @@ def updateProgress(self, progress): progress = 1 status = "Done...\r\n" block = int(round(barLength*progress)) - text = "\rFinshed Percent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), int(progress*100), status) + text = "\rFinshed Percent: [{0}] {1}% {2}".format( + "#"*block + "-"*(barLength-block), int(progress*100), status) sys.stdout.write(text) sys.stdout.flush() - diff --git a/vqaDemo.py b/vqaDemo.py new file mode 100644 index 0000000..337115b --- /dev/null +++ b/vqaDemo.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +from vqa import VQA +import random +import skimage.io as io +import matplotlib.pyplot as plt +import os + +dataDir = '../../VQA' +versionType = 'v2_' # this should be '' when not using VQA v2.0 dataset +# 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +taskType = 'OpenEnded' +# 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. +dataType = 'mscoco' +dataSubType = 'train2014' +annFile = '%s/Annotations/%s%s_%s_annotations.json' % ( + dataDir, versionType, dataType, dataSubType) +quesFile = '%s/Questions/%s%s_%s_%s_questions.json' % ( + dataDir, versionType, taskType, dataType, dataSubType) +imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) + +# initialize VQA api for QA annotations +vqa = VQA(annFile, quesFile) + +# load and display QA annotations for given question types +""" +All possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder. +""" +annIds = vqa.getQuesIds(quesTypes='how many') +anns = vqa.loadQA(annIds) +randomAnn = random.choice(anns) +vqa.showQA([randomAnn]) +imgId = randomAnn['image_id'] +imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' +if os.path.isfile(imgDir + imgFilename): + I = io.imread(imgDir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() + +# load and display QA annotations for given answer types +""" +ansTypes can be one of the following +yes/no +number +other +""" +annIds = vqa.getQuesIds(ansTypes='yes/no') +anns = vqa.loadQA(annIds) +randomAnn = random.choice(anns) +vqa.showQA([randomAnn]) +imgId = randomAnn['image_id'] +imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' +if os.path.isfile(imgDir + imgFilename): + I = io.imread(imgDir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() + +# load and display QA annotations for given images +""" +Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[]) +Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types. +""" +ids = vqa.getImgIds() +annIds = vqa.getQuesIds(imgIds=random.sample(ids, 5)) +anns = vqa.loadQA(annIds) +randomAnn = random.choice(anns) +vqa.showQA([randomAnn]) +imgId = randomAnn['image_id'] +imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' +if os.path.isfile(imgDir + imgFilename): + I = io.imread(imgDir + imgFilename) + plt.imshow(I) + plt.axis('off') + plt.show() diff --git a/PythonEvaluationTools/vqa_eval.py b/vqa_evaluate.py similarity index 62% rename from PythonEvaluationTools/vqa_eval.py rename to vqa_evaluate.py index e67be16..b056409 100644 --- a/PythonEvaluationTools/vqa_eval.py +++ b/vqa_evaluate.py @@ -2,11 +2,7 @@ import sys import os.path as osp -root_dir = osp.join("../../", - osp.basename(osp.dirname(osp.abspath('.')))) -sys.path.insert(0, '%s/PythonHelperTools/vqaTools' % (root_dir)) -from vqa import VQA -from vqaEvaluation.vqaEval import VQAEval +from vqa import VQA, VQAEval import matplotlib.pyplot as plt import skimage.io as io import json @@ -14,48 +10,56 @@ import os import argparse + def parse_args(): - parser=argparse.ArgumentParser() - parser.add_argument("results") - parser.add_argument("annotations") - parser.add_argument("questions") - parser.add_argument("image_dir") - parser.add_argument("--data_dir") + parser = argparse.ArgumentParser() + parser.add_argument("--results") + parser.add_argument("--annotations") + parser.add_argument("--questions") + parser.add_argument("--image_dir") + parser.add_argument("--data_dir", default=".") + parser.add_argument("--version", default="v2_") + parser.add_argument("--task", default="OpenEnded") + parser.add_argument("--data", default="mscoco") + parser.add_argument("--data_subtype", default="val2014") return parser.parse_args() -args=parse_args() +args = parse_args() # set up file names and paths -data_dir=args.data_dir -version='v2' # this should be 'v2' when using VQA v2.0 dataset -task='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +data_dir = args.data_dir +version = args.version # this should be 'v2' when using VQA v2.0 dataset +# 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 +task = args.task # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. -data='mscoco' -data_subtype='val2014' -ann_file=args.annotations or '%s/Annotations/%s%s_%s_annotations.json' % ( +data = args.data +data_subtype = args.data_subtype + +ann_file = args.annotations or '%s/Annotations/%s%s_%s_annotations.json' % ( data_dir, version, data, data_subtype) -ques_file=args.questions or '%s/Questions/%s%s_%s_%s_questions.json' % ( +ques_file = args.questions or '%s/Questions/%s%s_%s_%s_questions.json' % ( data_dir, version, task, data, data_subtype) -img_dir=args.image_dir or '%s/Images/%s/%s/' % (data_dir, data, data_subtype) -file_types=['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] + +img_dir = args.image_dir or '%s/Images/%s/%s/' % (data_dir, data, data_subtype) +file_types = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder. -file_template="{data_dir}/results/{version}_{task}_{data}_{data_subtype}_{file_type}.json" -[res_file, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile]=[file_template.format( +file_template = "{data_dir}/Results/{version}_{task}_{data}_{data_subtype}_{file_type}.json" +[res_file, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = [file_template.format( data_dir=data_dir, version=version, task=task, data=data, data_subtype=data_subtype, file_type=file_type) for file_type in file_types] -res_file=args.results or res_file +res_file = args.results or res_file # create vqa object and vqaRes object -vqa=VQA(ann_file, ques_file) -vqaRes=vqa.loadRes(res_file, ques_file) +vqa = VQA(ann_file, ques_file) +vqaRes = vqa.loadRes(res_file, ques_file) # create vqaEval object by taking vqa and vqaRes # n is precision of accuracy (number of places after decimal), default is 2 -vqaEval=VQAEval(vqa, vqaRes, n=3) +vqaEval = VQAEval(vqa, vqaRes, n=3) # evaluate results """ @@ -78,22 +82,22 @@ def parse_args(): print("\n") # demo how to use evalQA to retrieve low score result # 35 is per question percentage accuracy -evals=[quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId] < 35] +evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId] < 35] if len(evals) > 0: print('ground truth answers') - randomEval=random.choice(evals) - randomAnn=vqa.loadQA(randomEval) + randomEval = random.choice(evals) + randomAnn = vqa.loadQA(randomEval) vqa.showQA(randomAnn) print('\n') print('generated answer (accuracy %.02f)' % (vqaEval.evalQA[randomEval])) - ann=vqaRes.loadQA(randomEval)[0] + ann = vqaRes.loadQA(randomEval)[0] print("Answer: %s\n" % (ann['answer'])) - imgId=randomAnn[0]['image_id'] - imgFilename='COCO_' + data_subtype + '_' + str(imgId).zfill(12) + '.jpg' + imgId = randomAnn[0]['image_id'] + imgFilename = 'COCO_' + data_subtype + '_' + str(imgId).zfill(12) + '.jpg' if os.path.isfile(img_dir + imgFilename): - I=io.imread(img_dir + imgFilename) + I = io.imread(img_dir + imgFilename) plt.imshow(I) plt.axis('off') plt.show() From 36b661381bd2de4a8dc3c8cbe6d6921b16e6496e Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Thu, 17 Jan 2019 22:57:47 -0500 Subject: [PATCH 6/7] restructuring completed --- .gitignore | 3 + PythonHelperTools/vqaTools/__init__.py | 1 - PythonHelperTools/vqaTools/vqa.py | 179 ------------------------- README.md | 2 +- vqa/__init__.py | 4 +- vqa/vqa.py | 8 +- vqa_evaluate.py | 13 +- 7 files changed, 16 insertions(+), 194 deletions(-) delete mode 100644 PythonHelperTools/vqaTools/__init__.py delete mode 100644 PythonHelperTools/vqaTools/vqa.py diff --git a/.gitignore b/.gitignore index b7fa6fb..284a6b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ *.pyc *.swp* + +**/*.json +**/*.zip \ No newline at end of file diff --git a/PythonHelperTools/vqaTools/__init__.py b/PythonHelperTools/vqaTools/__init__.py deleted file mode 100644 index 072d8d9..0000000 --- a/PythonHelperTools/vqaTools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'aagrawal' diff --git a/PythonHelperTools/vqaTools/vqa.py b/PythonHelperTools/vqaTools/vqa.py deleted file mode 100644 index a63ba41..0000000 --- a/PythonHelperTools/vqaTools/vqa.py +++ /dev/null @@ -1,179 +0,0 @@ -__author__ = 'aagrawal' -__version__ = '0.9' - -# Interface for accessing the VQA dataset. - -# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: -# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py). - -# The following functions are defined: -# VQA - VQA class that loads VQA annotation file and prepares data structures. -# getQuesIds - Get question ids that satisfy given filter conditions. -# getImgIds - Get image ids that satisfy given filter conditions. -# loadQA - Load questions and answers with the specified question ids. -# showQA - Display the specified questions and answers. -# loadRes - Load result file and create result object. - -# Help on each function can be accessed by: "help(COCO.function)" - -import json -import datetime -import copy - -class VQA: - def __init__(self, annotation_file=None, question_file=None): - """ - Constructor of VQA helper class for reading and visualizing questions and answers. - :param annotation_file (str): location of VQA annotation file - :return: - """ - # load dataset - self.dataset = {} - self.questions = {} - self.qa = {} - self.qqa = {} - self.imgToQA = {} - if not annotation_file == None and not question_file == None: - print('loading VQA annotations and questions into memory...') - time_t = datetime.datetime.utcnow() - dataset = json.load(open(annotation_file, 'r')) - questions = json.load(open(question_file, 'r')) - print(datetime.datetime.utcnow() - time_t) - self.dataset = dataset - self.questions = questions - self.createIndex() - - def createIndex(self): - # create index - print('creating index...') - imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} - qa = {ann['question_id']: [] for ann in self.dataset['annotations']} - qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} - for ann in self.dataset['annotations']: - imgToQA[ann['image_id']] += [ann] - qa[ann['question_id']] = ann - for ques in self.questions['questions']: - qqa[ques['question_id']] = ques - print('index created!') - - # create class members - self.qa = qa - self.qqa = qqa - self.imgToQA = imgToQA - - def info(self): - """ - Print information about the VQA annotation file. - :return: - """ - for key, value in self.datset['info'].items(): - print('%s: %s'%(key, value)) - - def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): - """ - Get question ids that satisfy given filter conditions. default skips that filter - :param - imgIds (int array) : get question ids for given imgs - quesTypes (str array) : get question ids for given question types - ansTypes (str array) : get question ids for given answer types - :return: ids (int array) : integer array of question ids - """ - imgIds = imgIds if type(imgIds) == list else [imgIds] - quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] - ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] - - if len(imgIds) == len(quesTypes) == len(ansTypes) == 0: - anns = self.dataset['annotations'] - else: - if not len(imgIds) == 0: - anns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],[]) - else: - anns = self.dataset['annotations'] - anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] - anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] - ids = [ann['question_id'] for ann in anns] - return ids - - def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]): - """ - Get image ids that satisfy given filter conditions. default skips that filter - :param quesIds (int array) : get image ids for given question ids - quesTypes (str array) : get image ids for given question types - ansTypes (str array) : get image ids for given answer types - :return: ids (int array) : integer array of image ids - """ - quesIds = quesIds if type(quesIds) == list else [quesIds] - quesTypes = quesTypes if type(quesTypes) == list else [quesTypes] - ansTypes = ansTypes if type(ansTypes) == list else [ansTypes] - - if len(quesIds) == len(quesTypes) == len(ansTypes) == 0: - anns = self.dataset['annotations'] - else: - if not len(quesIds) == 0: - anns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa],[]) - else: - anns = self.dataset['annotations'] - anns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes] - anns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes] - ids = [ann['image_id'] for ann in anns] - return ids - - def loadQA(self, ids=[]): - """ - Load questions and answers with the specified question ids. - :param ids (int array) : integer ids specifying question ids - :return: qa (object array) : loaded qa objects - """ - if type(ids) == list: - return [self.qa[id] for id in ids] - elif type(ids) == int: - return [self.qa[ids]] - - def showQA(self, anns): - """ - Display the specified annotations. - :param anns (array of object): annotations to display - :return: None - """ - if len(anns) == 0: - return 0 - for ann in anns: - quesId = ann['question_id'] - print("Question: %s" %(self.qqa[quesId]['question'])) - for ans in ann['answers']: - print("Answer %d: %s" %(ans['answer_id'], ans['answer'])) - - def loadRes(self, resFile, quesFile): - """ - Load result file and return a result object. - :param resFile (str) : file name of result file - :return: res (obj) : result api object - """ - res = VQA() - res.questions = json.load(open(quesFile)) - res.dataset['info'] = copy.deepcopy(self.questions['info']) - res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) - res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) - res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype']) - res.dataset['license'] = copy.deepcopy(self.questions['license']) - - print('Loading and preparing results... ') - time_t = datetime.datetime.utcnow() - anns = json.load(open(resFile)) - assert type(anns) == list, 'results is not an array of objects' - annsQuesIds = [ann['question_id'] for ann in anns] - assert set(annsQuesIds) == set(self.getQuesIds()), \ - 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' - for ann in anns: - quesId = ann['question_id'] - if res.dataset['task_type'] == 'Multiple Choice': - assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' - qaAnn = self.qa[quesId] - ann['image_id'] = qaAnn['image_id'] - ann['question_type'] = qaAnn['question_type'] - ann['answer_type'] = qaAnn['answer_type'] - print('DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds())) - - res.dataset['annotations'] = anns - res.createIndex() - return res diff --git a/README.md b/README.md index 439d59d..4255971 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ There are two types of tasks - Multiple-choice task (18 choices per question) ## Requirements ## -- python 2.7 +- python 3.5+ - scikit-image (visit [this page](http://scikit-image.org/docs/dev/install.html) for installation) - matplotlib (visit [this page](http://matplotlib.org/users/installing.html) for installation) diff --git a/vqa/__init__.py b/vqa/__init__.py index c728069..2631dbb 100644 --- a/vqa/__init__.py +++ b/vqa/__init__.py @@ -1,2 +1,2 @@ -from vqa.vqa import VQA -from vqa.vqaEval import VQAEval +from .vqa import VQA +from .vqaEval import VQAEval diff --git a/vqa/vqa.py b/vqa/vqa.py index c519527..8b3865d 100644 --- a/vqa/vqa.py +++ b/vqa/vqa.py @@ -48,10 +48,8 @@ def createIndex(self): # create index print('creating index...') imgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']} - qa = {ann['question_id']: [] - for ann in self.dataset['annotations']} - qqa = {ann['question_id']: [] - for ann in self.dataset['annotations']} + qa = {ann['question_id']: [] for ann in self.dataset['annotations']} + qqa = {ann['question_id']: [] for ann in self.dataset['annotations']} for ann in self.dataset['annotations']: imgToQA[ann['image_id']] += [ann] qa[ann['question_id']] = ann @@ -69,7 +67,7 @@ def info(self): Print information about the VQA annotation file. :return: """ - for key, value in self.datset['info'].items(): + for key, value in self.dataset['info'].items(): print('%s: %s' % (key, value)) def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]): diff --git a/vqa_evaluate.py b/vqa_evaluate.py index b056409..af58424 100644 --- a/vqa_evaluate.py +++ b/vqa_evaluate.py @@ -9,6 +9,7 @@ import random import os import argparse +import pprint def parse_args(): @@ -18,7 +19,7 @@ def parse_args(): parser.add_argument("--questions") parser.add_argument("--image_dir") parser.add_argument("--data_dir", default=".") - parser.add_argument("--version", default="v2_") + parser.add_argument("--version", default="v2") parser.add_argument("--task", default="OpenEnded") parser.add_argument("--data", default="mscoco") parser.add_argument("--data_subtype", default="val2014") @@ -28,6 +29,8 @@ def parse_args(): args = parse_args() +pprint.pprint(vars(args)) + # set up file names and paths data_dir = args.data_dir version = args.version # this should be 'v2' when using VQA v2.0 dataset @@ -37,12 +40,10 @@ def parse_args(): data = args.data data_subtype = args.data_subtype -ann_file = args.annotations or '%s/Annotations/%s%s_%s_annotations.json' % ( - data_dir, version, data, data_subtype) -ques_file = args.questions or '%s/Questions/%s%s_%s_%s_questions.json' % ( - data_dir, version, task, data, data_subtype) +ann_file = args.annotations or '{args.data_dir}/Annotations/{args.version}_{args.data}_{args.data_subtype}_annotations.json'.format(args=args) +ques_file = args.questions or '{args.data_dir}/Questions/{args.version}_{args.task}_{args.data}_{args.data_subtype}_questions.json'.format(args=args) -img_dir = args.image_dir or '%s/Images/%s/%s/' % (data_dir, data, data_subtype) +img_dir = args.image_dir or '{args.data_dir}/Images/{args.data}/{args.data_subtype}/'.format(args=args) file_types = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder. From f27b4b92892ee57bbe1c5210a25ff6045e2c3e37 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Thu, 10 Jan 2019 17:09:44 -0500 Subject: [PATCH 7/7] remove custom eval script --- vqa_evaluate.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vqa_evaluate.py b/vqa_evaluate.py index af58424..86a524e 100644 --- a/vqa_evaluate.py +++ b/vqa_evaluate.py @@ -40,10 +40,13 @@ def parse_args(): data = args.data data_subtype = args.data_subtype -ann_file = args.annotations or '{args.data_dir}/Annotations/{args.version}_{args.data}_{args.data_subtype}_annotations.json'.format(args=args) -ques_file = args.questions or '{args.data_dir}/Questions/{args.version}_{args.task}_{args.data}_{args.data_subtype}_questions.json'.format(args=args) +ann_file = args.annotations or '{args.data_dir}/Annotations/{args.version}_{args.data}_{args.data_subtype}_annotations.json'.format( + args=args) +ques_file = args.questions or '{args.data_dir}/Questions/{args.version}_{args.task}_{args.data}_{args.data_subtype}_questions.json'.format( + args=args) -img_dir = args.image_dir or '{args.data_dir}/Images/{args.data}/{args.data_subtype}/'.format(args=args) +img_dir = args.image_dir or '{args.data_dir}/Images/{args.data}/{args.data_subtype}/'.format( + args=args) file_types = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder.