From 189f1d9e8915e6ec05aedca27c0244f8ec5fa3aa Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 28 Mar 2024 03:10:44 +0100 Subject: [PATCH 01/17] download script moved to new repo --- download_collection.py | 334 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 download_collection.py diff --git a/download_collection.py b/download_collection.py new file mode 100644 index 0000000..e0fafeb --- /dev/null +++ b/download_collection.py @@ -0,0 +1,334 @@ +# Script to download Scanned Objects by Google Research dataset and Stanford models +# Distributed by CC-BY 4.0 License + +# Dataset page: +# https://app.gazebosim.org/GoogleResearch/fuel/collections/Scanned%20Objects%20by%20Google%20Research + +import sys, json, requests +from pathlib import Path +import zipfile, tarfile, gzip + +import cv2 as cv + +if sys.version_info[0] < 3 or sys.version_info[1] < 5: + raise Exception("Python 3.5 or greater is required. Try running `python3 download_collection.py`") + +verbose = False + +class ModelData: + def __init__(self, name : str, description : str, filesize : int, thumb_url : str, categories ) -> None: + self.name = name + self.description = description + self.filesize = filesize + self.thumb_url = thumb_url + self.categories = set(categories) + +def print_size(num): + if num < 1024: + return str(num) + " B" + elif num < 1 << 20: + return "%.3f KiB" % (num / 1024) + elif num < 1 << 30: + return "%.3f MiB" % (num / (1 << 20)) + else: + return "%.3f GiB" % (num / (1 << 30)) + +collection_name = "Scanned Objects by Google Research" +owner_name = "GoogleResearch" + +base_url ='https://fuel.gazebosim.org/' +fuel_version = '1.0' + +def download_model(model_name, dir): + if verbose: + print() + print("{}: {}".format(model.name, model.description)) + print("Categories: [", ", ".join(model.categories), "]") + print("Size:", print_size(model.filesize)) + + download_url = base_url + fuel_version + '/{}/models/'.format(owner_name) + model_name + '.zip' + + archive_path = Path(dir) / Path(model_name+'.zip') + tmp_archive_path = Path(dir) / Path(model_name+'.zip.tmp') + mesh_path = Path(dir) / Path(model_name+'.obj') + tmp_mesh_path = Path(dir) / Path(model_name+'.obj.tmp') + mtl_path = Path(dir) / Path('model.mtl') + tmp_mtl_path = Path(dir) / Path('model.mtl.tmp') + texture_path = Path(dir) / Path('texture.png') + tmp_texture_path = Path(dir) / Path('texture.png.tmp') + + for tmp in [tmp_archive_path, tmp_mesh_path, tmp_mtl_path, tmp_texture_path]: + tmp.unlink(missing_ok=True) + + if archive_path.exists(): + if verbose: + print("Archive exists") + else: + print("URL:", download_url) + attempt = 1 + while True: + print("download attempt "+str(attempt)+"...", end="") + try: + download = requests.get(download_url, stream=True, timeout=5.0) + break + except requests.exceptions.Timeout: + print("Timed out") + attempt = attempt + 1 + with open(tmp_archive_path, 'wb') as fd: + for chunk in download.iter_content(chunk_size=1024*1024): + fd.write(chunk) + print(".", end="") + tmp_archive_path.rename(archive_path) + print("..downloaded") + + with zipfile.ZipFile(archive_path) as z: + if mesh_path.exists(): + if verbose: + print("OBJ exists") + else: + with open(tmp_mesh_path, 'wb') as f: + f.write(z.read("meshes/model.obj")) + tmp_mesh_path.rename(mesh_path) + print("OBJ unpacked") + if texture_path.exists(): + if verbose: + print("Texture exists") + else: + with open(tmp_texture_path, 'wb') as f: + f.write(z.read("materials/textures/texture.png")) + tmp_texture_path.rename(texture_path) + print("Texture unpacked") + + if mtl_path.exists(): + if verbose: + print("Material exists") + else: + mtlFile = """ +# Copyright 2020 Google LLC. +# +# This work is licensed under the Creative Commons Attribution 4.0 +# International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by/4.0/ or send a letter +# to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +newmtl material_0 +# shader_type beckmann +map_Kd texture.png + +# Kd: Diffuse reflectivity. +Kd 1.000000 1.000000 1.000000 +""" + with open(tmp_mtl_path, 'xt') as f: + f.writelines(mtlFile) + tmp_mtl_path.rename(mtl_path) + print("Material written") + return mesh_path, texture_path + +def get_thumb(model : ModelData, dir): + if verbose: + print(model.name) + img_url = base_url + fuel_version + model.thumb_url + img_path = Path(dir) / Path(model.name+'.jpg') + tmp_path = Path(dir) / Path(model.name+'.jpg.tmp') + tmp_path.unlink(missing_ok=True) + if img_path.exists(): + if verbose: + print("...exists") + else: + print("URL:", img_url) + attempt = 1 + while True: + print("download attempt "+str(attempt)+"...") + try: + download = requests.get(img_url, stream=True, timeout=5.0) + break + except requests.exceptions.Timeout: + print("Timed out") + attempt = attempt + 1 + with open(tmp_path, 'wb') as fd: + for chunk in download.iter_content(chunk_size=1024*1024): + fd.write(chunk) + print(".", end="") + tmp_path.rename(img_path) + print("..downloaded") + + +def get_content(content_file): + # Getting model names and URLs + models_json = [] + + # Iterate over the pages + page = 0 + while True: + page = page + 1 + next_url = '/models?page={}&per_page=100&q=collections:{}'.format(page,collection_name) + page_url = base_url + fuel_version + next_url + + print("Gettting page %d..." % page) + r = requests.get(page_url) + + if not r or not r.text: + break + + # Convert to JSON + models_page = json.loads(r.text) + + if not models_page: + break + + models_json.extend(models_page) + + print(len(models_json), " models") + + json_object = json.dumps(models_json, indent=4) + with open(content_file, "w") as outfile: + outfile.write(json_object) + + return models_json + + +# let's use different chunk sizes to get rid of timeouts +stanford_models = [ +["http://graphics.stanford.edu/pub/3Dscanrep/bunny.tar.gz", 1, "bunny/reconstruction/bun_zipper.ply"], +["http://graphics.stanford.edu/pub/3Dscanrep/happy/happy_recon.tar.gz", 1024, "happy_recon/happy_vrip.ply"], +["http://graphics.stanford.edu/pub/3Dscanrep/dragon/dragon_recon.tar.gz", 1024, "dragon_recon/dragon_vrip.ply"], +["http://graphics.stanford.edu/pub/3Dscanrep/armadillo/Armadillo.ply.gz", 64, ""], +["http://graphics.stanford.edu/data/3Dscanrep/lucy.tar.gz", 1024, "lucy.ply"], +["http://graphics.stanford.edu/data/3Dscanrep/xyzrgb/xyzrgb_dragon.ply.gz", 1024, ""], +["http://graphics.stanford.edu/data/3Dscanrep/xyzrgb/xyzrgb_manuscript.ply.gz", 1024, ""], +["http://graphics.stanford.edu/data/3Dscanrep/xyzrgb/xyzrgb_statuette.ply.gz", 1024, ""], +] + +def get_stanford_model(url : str, name : str, ext: str, dir : str, chunk_size : int, internal_path : str): + archive_path = Path(dir) / Path(name+'.'+ext) + tmp_archive_path = Path(dir) / Path(name+'.'+ext+'.tmp') + model_path = Path(dir) / Path(name+'.ply') + tmp_model_path = Path(dir) / Path(name+'.ply.tmp') + + for tmp in [tmp_archive_path, tmp_model_path]: + tmp.unlink(missing_ok=True) + + if archive_path.exists(): + if verbose: + print("Archive exists") + else: + print("URL:", url) + attempt = 1 + while True: + print("download attempt "+str(attempt)+"...", end="") + try: + download = requests.get(url, stream=True, timeout=5.0) + break + except requests.exceptions.Timeout: + print("Timed out") + attempt = attempt + 1 + with open(tmp_archive_path, 'wb') as fd: + for chunk in download.iter_content(chunk_size=chunk_size*1024): + fd.write(chunk) + print(".", end="") + tmp_archive_path.rename(archive_path) + print("..downloaded") + + if model_path.exists(): + if verbose: + print("Model exists") + else: + # to reduce memory footprint for big models + max_size = 1024*1024*16 + print("Extracting..", end="") + with open(tmp_model_path, 'xb') as of: + if ext=="tar.gz": + tar_obj = tarfile.open(archive_path, 'r', encoding='utf-8', errors='surrogateescape') + try: + reader = tar_obj.extractfile(internal_path) + while buf := reader.read(max_size): + of.write(buf) + print(".", end="") + except Exception as err: + print(err) + tar_obj.close() + elif ext=="ply.gz": + with gzip.open(archive_path) as gz: + while buf := gz.read(max_size): + of.write(buf) + print(".", end="") + tmp_model_path.rename(model_path) + print("done") + return model_path, "" + + +# ================================================== + +dirname = "dlmodels" + +all_models = [] + +print("Getting Google Research models") + +content_file = Path(dirname) / Path("content.json") +if content_file.exists(): + with open(content_file, "r") as openfile: + models_json = json.load(openfile) +else: + Path(dirname).mkdir(parents=True, exist_ok=True) + models_json = get_content(content_file) + +models = [] +for model in models_json: + model_name = model['name'] + desc = model['description'] + fsize = model['filesize'] + thumb_url = model['thumbnail_url'] + if 'categories' in model: + categories = model['categories'] + else: + categories = [ ] + models.append(ModelData(model_name, desc, fsize, thumb_url, categories)) + +print("Getting thumbnail images") +for model in models: + get_thumb(model, dirname) + +print("Downloading models from the {}/{} collection.".format(owner_name, collection_name)) + +for model in models: + model_dir = Path(dirname) / Path(model.name) + Path(model_dir).mkdir(parents=True, exist_ok=True) + model_path, texture_path = download_model(model.name, model_dir) + all_models.append((model_path, texture_path)) + +print('Done.') + +categories = set() +for model in models: + for cat in model.categories: + categories.add(cat) +print("Categories:", categories) +#{'Consumer Goods', 'Bag', 'Car Seat', +# 'Keyboard', 'Media Cases', 'Toys', +# 'Action Figures', 'Bottles and Cans and Cups', +# 'Shoe', 'Legos', 'Hat', +# 'Mouse', 'Headphones', 'Stuffed Toys', +# 'Board Games', 'Camera'} + +print("\nGetting Stanford models") + +for m in stanford_models: + url, chunk_size, internal_path = m + + s = url.split("/")[-1].split(".") + name = "stanford_"+s[0] + ext = s[1]+"."+s[2] + + if verbose: + print(name + ":") + model_dir = Path(dirname) / Path(name) + Path(model_dir).mkdir(parents=True, exist_ok=True) + model_path, texture_path = get_stanford_model(url, name, ext, model_dir, chunk_size, internal_path) + all_models.append((model_path, texture_path)) + +print("\nSubsampling") + +for mf, tf in all_models: + print(mf, tf) + verts, indices, normals, colors = cv.loadMesh(mf) + From 93782e31fab1fea1b215d7e609f09e6312a4a909 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 4 Apr 2024 17:03:25 +0200 Subject: [PATCH 02/17] formatting, minors --- download_collection.py | 117 +++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 58 deletions(-) diff --git a/download_collection.py b/download_collection.py index e0fafeb..44ab3b4 100644 --- a/download_collection.py +++ b/download_collection.py @@ -13,6 +13,7 @@ if sys.version_info[0] < 3 or sys.version_info[1] < 5: raise Exception("Python 3.5 or greater is required. Try running `python3 download_collection.py`") +# global params verbose = False class ModelData: @@ -123,6 +124,7 @@ def download_model(model_name, dir): print("Material written") return mesh_path, texture_path + def get_thumb(model : ModelData, dir): if verbose: print(model.name) @@ -257,78 +259,77 @@ def get_stanford_model(url : str, name : str, ext: str, dir : str, chunk_size : # ================================================== +if __name__ == "__main__": + verbose = False -dirname = "dlmodels" - -all_models = [] + dirname = "dlmodels" -print("Getting Google Research models") + all_models = [] -content_file = Path(dirname) / Path("content.json") -if content_file.exists(): - with open(content_file, "r") as openfile: - models_json = json.load(openfile) -else: - Path(dirname).mkdir(parents=True, exist_ok=True) - models_json = get_content(content_file) + print("Getting Google Research models") -models = [] -for model in models_json: - model_name = model['name'] - desc = model['description'] - fsize = model['filesize'] - thumb_url = model['thumbnail_url'] - if 'categories' in model: - categories = model['categories'] + content_file = Path(dirname) / Path("content.json") + if content_file.exists(): + with open(content_file, "r") as openfile: + models_json = json.load(openfile) else: - categories = [ ] - models.append(ModelData(model_name, desc, fsize, thumb_url, categories)) + Path(dirname).mkdir(parents=True, exist_ok=True) + models_json = get_content(content_file) + + models = [] + for model in models_json: + model_name = model['name'] + desc = model['description'] + fsize = model['filesize'] + thumb_url = model['thumbnail_url'] + if 'categories' in model: + categories = model['categories'] + else: + categories = [ ] + models.append(ModelData(model_name, desc, fsize, thumb_url, categories)) -print("Getting thumbnail images") -for model in models: - get_thumb(model, dirname) + print("Getting thumbnail images") + for model in models: + get_thumb(model, dirname) -print("Downloading models from the {}/{} collection.".format(owner_name, collection_name)) + print("Downloading models from the {}/{} collection.".format(owner_name, collection_name)) -for model in models: - model_dir = Path(dirname) / Path(model.name) - Path(model_dir).mkdir(parents=True, exist_ok=True) - model_path, texture_path = download_model(model.name, model_dir) - all_models.append((model_path, texture_path)) + for model in models: + model_dir = Path(dirname) / Path(model.name) + Path(model_dir).mkdir(parents=True, exist_ok=True) + model_path, texture_path = download_model(model.name, model_dir) + all_models.append((model_path, texture_path)) -print('Done.') + print('Done.') -categories = set() -for model in models: - for cat in model.categories: - categories.add(cat) -print("Categories:", categories) -#{'Consumer Goods', 'Bag', 'Car Seat', -# 'Keyboard', 'Media Cases', 'Toys', -# 'Action Figures', 'Bottles and Cans and Cups', -# 'Shoe', 'Legos', 'Hat', -# 'Mouse', 'Headphones', 'Stuffed Toys', -# 'Board Games', 'Camera'} + categories = set() + for model in models: + for cat in model.categories: + categories.add(cat) + print("Categories:", categories) + # 'Consumer Goods', 'Bag', 'Car Seat', 'Keyboard', 'Media Cases', 'Toys', + # 'Action Figures', 'Bottles and Cans and Cups', 'Shoe', 'Legos', 'Hat', + # 'Mouse', 'Headphones', 'Stuffed Toys', 'Board Games', 'Camera' -print("\nGetting Stanford models") + print("\nGetting Stanford models") -for m in stanford_models: - url, chunk_size, internal_path = m + for m in stanford_models: + url, chunk_size, internal_path = m - s = url.split("/")[-1].split(".") - name = "stanford_"+s[0] - ext = s[1]+"."+s[2] + s = url.split("/")[-1].split(".") + name = "stanford_"+s[0] + ext = s[1]+"."+s[2] - if verbose: - print(name + ":") - model_dir = Path(dirname) / Path(name) - Path(model_dir).mkdir(parents=True, exist_ok=True) - model_path, texture_path = get_stanford_model(url, name, ext, model_dir, chunk_size, internal_path) - all_models.append((model_path, texture_path)) + if verbose: + print(name + ":") + model_dir = Path(dirname) / Path(name) + Path(model_dir).mkdir(parents=True, exist_ok=True) + model_path, texture_path = get_stanford_model(url, name, ext, model_dir, chunk_size, internal_path) + all_models.append((model_path, texture_path)) -print("\nSubsampling") + print("\nSubsampling") -for mf, tf in all_models: - print(mf, tf) - verts, indices, normals, colors = cv.loadMesh(mf) + for mf, tf in all_models: + print(mf, tf) + verts, indices, normals, colors = cv.loadMesh(mf) From a0db78e2d3b4b533c77179a7e8bb86585f1d2bf0 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 11 Apr 2024 02:08:05 +0200 Subject: [PATCH 03/17] draft version is done --- download_collection.py | 181 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 2 deletions(-) diff --git a/download_collection.py b/download_collection.py index 44ab3b4..b4678bd 100644 --- a/download_collection.py +++ b/download_collection.py @@ -5,8 +5,11 @@ # https://app.gazebosim.org/GoogleResearch/fuel/collections/Scanned%20Objects%20by%20Google%20Research import sys, json, requests +import subprocess from pathlib import Path import zipfile, tarfile, gzip +import numpy as np +import math import cv2 as cv @@ -258,6 +261,30 @@ def get_stanford_model(url : str, name : str, ext: str, dir : str, chunk_size : return model_path, "" +def lookAtMatrixCal(position, lookat, upVector): + tmp = position - lookat + norm = np.linalg.norm(tmp) + w = tmp / norm + tmp = np.cross(upVector, w) + norm = np.linalg.norm(tmp) + u = tmp / norm + v = np.cross(w, u) + res = np.array([ + [u[0], u[1], u[2], 0], + [v[0], v[1], v[2], 0], + [w[0], w[1], w[2], 0], + [0, 0, 0, 1.0] + ], dtype=np.float32) + translate = np.array([ + [1.0, 0, 0, -position[0]], + [0, 1.0, 0, -position[1]], + [0, 0, 1.0, -position[2]], + [0, 0, 0, 1.0] + ], dtype=np.float32) + res = np.matmul(res, translate) + return res + + # ================================================== if __name__ == "__main__": verbose = False @@ -327,9 +354,159 @@ def get_stanford_model(url : str, name : str, ext: str, dir : str, chunk_size : model_path, texture_path = get_stanford_model(url, name, ext, model_dir, chunk_size, internal_path) all_models.append((model_path, texture_path)) + width, height = 640, 480 + fovDegrees = 45.0 + fovY = fovDegrees * math.pi / 180.0 + print("\nSubsampling") for mf, tf in all_models: print(mf, tf) - verts, indices, normals, colors = cv.loadMesh(mf) - + verts, list_indices, normals, colors, texCoords = cv.loadMesh(mf) + texture = cv.imread(tf) / 255.0 + verts = verts[0, :, :] + print("verts: ", verts.shape) + # list_indices is a tuple of 1x3 arrays of dtype=int32 + indices = np.zeros((len(list_indices), 3), dtype=np.int32) + for i, ind in enumerate(list_indices): + if ind.shape == (3, 1): + ind = ind.t() + if ind.shape != (1, 3): + raise ValueError() + indices[i, :] = ind[:] + print("indices: ", indices.shape) + normals = normals[0, :, :] + print("normals: ", normals.shape) + #print("colors: ", colors.shape) # empty now + #texCoords = texCoords[0, :, :] + print("texCoords: ", texCoords.shape) # empty now + + bbox = np.array([np.min(verts[:, 0]), np.min(verts[:, 1]), np.min(verts[:, 2]), + np.max(verts[:, 0]), np.max(verts[:, 1]), np.max(verts[:, 2])]) + + minx, miny, minz = np.min(verts[:, 0]), np.min(verts[:, 1]), np.min(verts[:, 2]) + maxx, maxy, maxz = np.max(verts[:, 0]), np.max(verts[:, 1]), np.max(verts[:, 2]) + + print("bounding box: [%f...%f, %f...%f, %f...%f]" % (minx, maxx, miny, maxy, minz, maxz)) + + # this could be used for slow and dirty texturing + doRemap = False + + nverts = verts.shape[0] + texsz = texture.shape[0:2] + texw, texh = texsz[1], texsz[0] + minv = np.array([minx, miny, minz]) + maxv = np.array([maxx, maxy, maxz]) + diffv = 1.0/(maxv - minv) + colors = np.ones((nverts, 3), dtype=np.float32) + for i in range(nverts): + tc = texCoords[i, :] + u, v = int(tc[0] * texw - 0.5), int((1.0-tc[1]) * texh - 0.5) + if doRemap: + colors[i, :] = [tc[0], 1-tc[1], 0] + else: + colors[i, :] = texture[v, u, :] + + ctgY = 1./math.tan(fovY / 2.0) + ctgX = ctgY / width * height + zat = maxz + max([abs(maxy) * ctgY, + abs(miny) * ctgY, + abs(maxx) * ctgX, + abs(minx) * ctgX]) + + zNear = zat - maxz + zFar = zat - minz + position = np.array([0.0, 0.0, zat], dtype=np.float32) + lookat = np.array([0.0, 0.0, 0.0], dtype=np.float32) + upVector = np.array([0.0, 1.0, 0.0], dtype=np.float32) + cameraPose = lookAtMatrixCal(position, lookat, upVector) + + depth_buf = np.ones((height, width), dtype=np.float32) * zFar + color_buf = np.zeros((height, width, 3), dtype=np.float32) + + settings = cv.TriangleRasterizeSettings().setShadingType(cv.RASTERIZE_SHADING_SHADED) + settings = settings.setCullingMode(cv.RASTERIZE_CULLING_NONE) + + color_buf, depth_buf = cv.triangleRasterize(verts, indices, colors, color_buf, depth_buf, + cameraPose, fovY, zNear, zFar, settings) + + if doRemap: + mapx = color_buf[:, :, 0] * texture.shape[1] - 0.5 + mapy = color_buf[:, :, 1] * texture.shape[0] - 0.5 + remapped = cv.remap(texture, mapx, mapy, cv.INTER_LINEAR) + + colorRasterize = color_buf + depthRasterize = (depth_buf * 1000.0) + cv.imwrite("/home/savuor/logs/loadmesh/color_raster.png", color_buf * 255.0) + if doRemap: + cv.imwrite("/home/savuor/logs/loadmesh/remap.png", remapped * 255.0) + cv.imwrite("/home/savuor/logs/loadmesh/depth_raster.png", depthRasterize.astype(np.ushort)) + + # send mesh to OpenGL rasterizer + + vertsToSave = np.expand_dims(verts, axis=0) + colorsToSave = np.expand_dims(colors, axis=0) + colorsToSave[0, :, ::] = colorsToSave[0, :, ::-1] + indicesToSave = [] + for i in range(indices.shape[0]): + ix = indices[i, :] + indicesToSave.append(ix) + cv.saveMesh("/home/savuor/logs/loadmesh/colvert.ply", vertsToSave, indicesToSave, None, colorsToSave) + + args = ["bin/example_opengl_opengl_testdata_generator"] + [ + "--modelPath=/home/savuor/logs/loadmesh/colvert.ply", + "--custom", + "--fov="+str(fovDegrees), + "--posx="+str(position[0]), + "--posy="+str(position[1]), + "--posz="+str(position[2]), + "--lookatx="+str(lookat[0]), + "--lookaty="+str(lookat[1]), + "--lookatz="+str(lookat[2]), + "--upx="+str(upVector[0]), + "--upy="+str(upVector[1]), + "--upz="+str(upVector[2]), + "--resx="+str(width), + "--resy="+str(height), + "--zNear="+str(zNear), + "--zFar="+str(zFar), + # white/flat/shaded + "--shading=shaded", + # none/cw/ccw + "--culling=cw", + "--colorPath=/home/savuor/logs/loadmesh/color.png", + "--depthPath=/home/savuor/logs/loadmesh/depth.png", + ] + + print(args) + p = subprocess.run(args) + + # compare results + + colorGl = cv.imread("/home/savuor/logs/loadmesh/color.png") + colorGl = colorGl.astype(np.float32) * (1.0/255.0) + colorDiff = np.ravel(colorGl - colorRasterize) + normInfRgb = np.linalg.norm(colorDiff, ord=np.inf) + normL2Rgb = np.linalg.norm(colorDiff, ord=2) / (width * height) + print("rgb L2: %f Inf: %f" % (normL2Rgb, normInfRgb)) + + cv.imwrite("/home/savuor/logs/loadmesh/color_diff.png", (colorDiff.reshape((height, width, 3)) + 1) * 0.5 * 255.0) + + depthGl = cv.imread("/home/savuor/logs/loadmesh/depth.png", cv.IMREAD_GRAYSCALE | cv.IMREAD_ANYDEPTH).astype(np.float32) + depthDiff = depthGl - depthRasterize + threshold = math.floor(zFar * 1000) + maskGl = depthGl < threshold + maskRaster = depthRasterize < threshold + maskDiff = maskGl != maskRaster + nzDepthDiff = np.count_nonzero(maskDiff) + print("depth nzdiff: %d" % (nzDepthDiff)) + + jointMask = maskRaster & maskGl + nzJointMask = np.count_nonzero(jointMask) + # maskedDiff = np.ma.masked_array(depthDiff, jointMask) + maskedDiff = np.ravel(depthDiff[jointMask]) + normInfDepth = np.linalg.norm(maskedDiff, ord=np.inf) + normL2Depth = np.linalg.norm(maskedDiff, ord=2) / nzJointMask + print("depth L2: %f Inf: %f" % (normL2Depth, normInfDepth)) + + cv.imwrite("/home/savuor/logs/loadmesh/depth_diff.png", ((depthDiff) + (1 << 15)).astype(np.ushort)) From 2fb65eec2ad203676756206e135e87255dc84344 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 11 Apr 2024 04:40:14 +0200 Subject: [PATCH 04/17] add argparse --- download_collection.py | 82 ++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/download_collection.py b/download_collection.py index b4678bd..f8ca8f7 100644 --- a/download_collection.py +++ b/download_collection.py @@ -5,6 +5,7 @@ # https://app.gazebosim.org/GoogleResearch/fuel/collections/Scanned%20Objects%20by%20Google%20Research import sys, json, requests +import argparse import subprocess from pathlib import Path import zipfile, tarfile, gzip @@ -287,9 +288,24 @@ def lookAtMatrixCal(position, lookat, upVector): # ================================================== if __name__ == "__main__": - verbose = False + parser = argparse.ArgumentParser(prog="bench3d", description="3D algorithm benchmark for OpenCV") + parser.add_argument("--work_dir") + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--renderer_path") + parser.add_argument("--debug_pics_dir") - dirname = "dlmodels" + args = parser.parse_args() + + if args.verbose: + verbose = True + + dirname = args.work_dir + + debug_pics_dir = "" + if args.debug_pics_dir: + debug_pics_dir = args.debug_pics_dir + + renderer_path = args.renderer_path all_models = [] @@ -360,12 +376,13 @@ def lookAtMatrixCal(position, lookat, upVector): print("\nSubsampling") - for mf, tf in all_models: - print(mf, tf) - verts, list_indices, normals, colors, texCoords = cv.loadMesh(mf) - texture = cv.imread(tf) / 255.0 + for model_fname, texture_fname in all_models: + print(model_fname, texture_fname) + verts, list_indices, normals, colors, texCoords = cv.loadMesh(model_fname) + if texture_fname: + texture = cv.imread(texture_fname) / 255.0 verts = verts[0, :, :] - print("verts: ", verts.shape) + # list_indices is a tuple of 1x3 arrays of dtype=int32 indices = np.zeros((len(list_indices), 3), dtype=np.int32) for i, ind in enumerate(list_indices): @@ -374,20 +391,17 @@ def lookAtMatrixCal(position, lookat, upVector): if ind.shape != (1, 3): raise ValueError() indices[i, :] = ind[:] - print("indices: ", indices.shape) normals = normals[0, :, :] - print("normals: ", normals.shape) - #print("colors: ", colors.shape) # empty now - #texCoords = texCoords[0, :, :] - print("texCoords: ", texCoords.shape) # empty now - - bbox = np.array([np.min(verts[:, 0]), np.min(verts[:, 1]), np.min(verts[:, 2]), - np.max(verts[:, 0]), np.max(verts[:, 1]), np.max(verts[:, 2])]) minx, miny, minz = np.min(verts[:, 0]), np.min(verts[:, 1]), np.min(verts[:, 2]) maxx, maxy, maxz = np.max(verts[:, 0]), np.max(verts[:, 1]), np.max(verts[:, 2]) - print("bounding box: [%f...%f, %f...%f, %f...%f]" % (minx, maxx, miny, maxy, minz, maxz)) + if verbose: + print("verts: ", verts.shape) + print("indices: ", indices.shape) + print("normals: ", normals.shape) + print("texCoords: ", texCoords.shape) # empty now + print("bounding box: [%f...%f, %f...%f, %f...%f]" % (minx, maxx, miny, maxy, minz, maxz)) # this could be used for slow and dirty texturing doRemap = False @@ -437,10 +451,12 @@ def lookAtMatrixCal(position, lookat, upVector): colorRasterize = color_buf depthRasterize = (depth_buf * 1000.0) - cv.imwrite("/home/savuor/logs/loadmesh/color_raster.png", color_buf * 255.0) - if doRemap: - cv.imwrite("/home/savuor/logs/loadmesh/remap.png", remapped * 255.0) - cv.imwrite("/home/savuor/logs/loadmesh/depth_raster.png", depthRasterize.astype(np.ushort)) + + if debug_pics_dir: + cv.imwrite(Path(debug_pics_dir) / Path("color_raster.png"), color_buf * 255.0) + if doRemap: + cv.imwrite(Path(debug_pics_dir) / Path("remap.png"), remapped * 255.0) + cv.imwrite(Path(debug_pics_dir) / Path("depth_raster.png"), depthRasterize.astype(np.ushort)) # send mesh to OpenGL rasterizer @@ -451,10 +467,13 @@ def lookAtMatrixCal(position, lookat, upVector): for i in range(indices.shape[0]): ix = indices[i, :] indicesToSave.append(ix) - cv.saveMesh("/home/savuor/logs/loadmesh/colvert.ply", vertsToSave, indicesToSave, None, colorsToSave) + colvert_path = Path(model_fname).parent / Path("colvert.ply") + cv.saveMesh(colvert_path, vertsToSave, indicesToSave, None, colorsToSave) - args = ["bin/example_opengl_opengl_testdata_generator"] + [ - "--modelPath=/home/savuor/logs/loadmesh/colvert.ply", + rgb_gl_path = Path(model_fname).parent / Path("color.png") + depth_gl_path = Path(model_fname).parent / Path("depth.png") + renderer_args = [renderer_path] + [ + "--modelPath="+str(colvert_path), "--custom", "--fov="+str(fovDegrees), "--posx="+str(position[0]), @@ -474,25 +493,26 @@ def lookAtMatrixCal(position, lookat, upVector): "--shading=shaded", # none/cw/ccw "--culling=cw", - "--colorPath=/home/savuor/logs/loadmesh/color.png", - "--depthPath=/home/savuor/logs/loadmesh/depth.png", + "--colorPath="+str(rgb_gl_path), + "--depthPath="+str(depth_gl_path), ] - print(args) - p = subprocess.run(args) + if verbose: + print(renderer_args) + p = subprocess.run(renderer_args) # compare results - colorGl = cv.imread("/home/savuor/logs/loadmesh/color.png") + colorGl = cv.imread(rgb_gl_path) colorGl = colorGl.astype(np.float32) * (1.0/255.0) colorDiff = np.ravel(colorGl - colorRasterize) normInfRgb = np.linalg.norm(colorDiff, ord=np.inf) normL2Rgb = np.linalg.norm(colorDiff, ord=2) / (width * height) print("rgb L2: %f Inf: %f" % (normL2Rgb, normInfRgb)) - cv.imwrite("/home/savuor/logs/loadmesh/color_diff.png", (colorDiff.reshape((height, width, 3)) + 1) * 0.5 * 255.0) + cv.imwrite(Path(model_fname).parent / Path("color_diff.png"), (colorDiff.reshape((height, width, 3)) + 1) * 0.5 * 255.0) - depthGl = cv.imread("/home/savuor/logs/loadmesh/depth.png", cv.IMREAD_GRAYSCALE | cv.IMREAD_ANYDEPTH).astype(np.float32) + depthGl = cv.imread(depth_gl_path, cv.IMREAD_GRAYSCALE | cv.IMREAD_ANYDEPTH).astype(np.float32) depthDiff = depthGl - depthRasterize threshold = math.floor(zFar * 1000) maskGl = depthGl < threshold @@ -509,4 +529,6 @@ def lookAtMatrixCal(position, lookat, upVector): normL2Depth = np.linalg.norm(maskedDiff, ord=2) / nzJointMask print("depth L2: %f Inf: %f" % (normL2Depth, normInfDepth)) + cv.imwrite(Path(model_fname).parent / Path("depth_diff.png"), ((depthDiff) + (1 << 15)).astype(np.ushort)) + cv.imwrite("/home/savuor/logs/loadmesh/depth_diff.png", ((depthDiff) + (1 << 15)).astype(np.ushort)) From d2b465d41faa10f461827dc390169fc6b683a4e1 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 11 Apr 2024 04:40:37 +0200 Subject: [PATCH 05/17] output stats --- download_collection.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/download_collection.py b/download_collection.py index f8ca8f7..212b9a7 100644 --- a/download_collection.py +++ b/download_collection.py @@ -374,7 +374,9 @@ def lookAtMatrixCal(position, lookat, upVector): fovDegrees = 45.0 fovY = fovDegrees * math.pi / 180.0 - print("\nSubsampling") + print("\ntriangleRasterize() test") + + stat_data = {} for model_fname, texture_fname in all_models: print(model_fname, texture_fname) @@ -531,4 +533,16 @@ def lookAtMatrixCal(position, lookat, upVector): cv.imwrite(Path(model_fname).parent / Path("depth_diff.png"), ((depthDiff) + (1 << 15)).astype(np.ushort)) - cv.imwrite("/home/savuor/logs/loadmesh/depth_diff.png", ((depthDiff) + (1 << 15)).astype(np.ushort)) + stat_data[str(model_fname)] = { + "normL2Rgb" : str(normL2Rgb), + "normInfRgb" : str(normInfRgb), + "nzDepthDiff" : str(nzDepthDiff), + "normL2Depth" : str(normL2Depth), + "normInfDepth" : str(normInfDepth) + } + + stat_json = json.dumps(stat_data, indent=4) + with open(Path(dirname) / Path("stat.json"), "w") as outfile: + outfile.write(stat_json) + + print("...next") From 85683901d6757b48e88b79a3081d60fd086c8bf4 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 11 Apr 2024 19:41:06 +0200 Subject: [PATCH 06/17] imports --- download_collection.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/download_collection.py b/download_collection.py index 212b9a7..55097a1 100644 --- a/download_collection.py +++ b/download_collection.py @@ -4,14 +4,15 @@ # Dataset page: # https://app.gazebosim.org/GoogleResearch/fuel/collections/Scanned%20Objects%20by%20Google%20Research -import sys, json, requests -import argparse -import subprocess +import sys, argparse, subprocess from pathlib import Path +import requests import zipfile, tarfile, gzip -import numpy as np +import csv, json import math +import numpy as np +# cv2 library should be in PYTHONPATH variable import cv2 as cv if sys.version_info[0] < 3 or sys.version_info[1] < 5: From e27c82b60b225027418525b36c6cf4aacffee673 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 11 Apr 2024 19:42:40 +0200 Subject: [PATCH 07/17] out stats --- download_collection.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/download_collection.py b/download_collection.py index 55097a1..6530c70 100644 --- a/download_collection.py +++ b/download_collection.py @@ -342,7 +342,7 @@ def lookAtMatrixCal(position, lookat, upVector): model_dir = Path(dirname) / Path(model.name) Path(model_dir).mkdir(parents=True, exist_ok=True) model_path, texture_path = download_model(model.name, model_dir) - all_models.append((model_path, texture_path)) + all_models.append((model.name, model_path, texture_path)) print('Done.') @@ -369,18 +369,18 @@ def lookAtMatrixCal(position, lookat, upVector): model_dir = Path(dirname) / Path(name) Path(model_dir).mkdir(parents=True, exist_ok=True) model_path, texture_path = get_stanford_model(url, name, ext, model_dir, chunk_size, internal_path) - all_models.append((model_path, texture_path)) + all_models.append((name, model_path, texture_path)) + + print("\ntriangleRasterize() test") width, height = 640, 480 fovDegrees = 45.0 fovY = fovDegrees * math.pi / 180.0 - print("\ntriangleRasterize() test") - stat_data = {} - for model_fname, texture_fname in all_models: - print(model_fname, texture_fname) + for model_name, model_fname, texture_fname in all_models: + print(model_name) verts, list_indices, normals, colors, texCoords = cv.loadMesh(model_fname) if texture_fname: texture = cv.imread(texture_fname) / 255.0 @@ -461,6 +461,9 @@ def lookAtMatrixCal(position, lookat, upVector): cv.imwrite(Path(debug_pics_dir) / Path("remap.png"), remapped * 255.0) cv.imwrite(Path(debug_pics_dir) / Path("depth_raster.png"), depthRasterize.astype(np.ushort)) + cv.imwrite(Path(model_fname).parent / Path("color_raster.png"), color_buf * 255.0) + cv.imwrite(Path(model_fname).parent / Path("depth_raster.png"), depthRasterize.astype(np.ushort)) + # send mesh to OpenGL rasterizer vertsToSave = np.expand_dims(verts, axis=0) @@ -534,7 +537,7 @@ def lookAtMatrixCal(position, lookat, upVector): cv.imwrite(Path(model_fname).parent / Path("depth_diff.png"), ((depthDiff) + (1 << 15)).astype(np.ushort)) - stat_data[str(model_fname)] = { + stat_data[str(model_name)] = { "normL2Rgb" : str(normL2Rgb), "normInfRgb" : str(normInfRgb), "nzDepthDiff" : str(nzDepthDiff), From 4d9c5a9c6d9973f473be86e9493de1bb5d95f985 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 11 Apr 2024 19:45:01 +0200 Subject: [PATCH 08/17] export stat to csv --- download_collection.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/download_collection.py b/download_collection.py index 6530c70..fa162c9 100644 --- a/download_collection.py +++ b/download_collection.py @@ -379,6 +379,22 @@ def lookAtMatrixCal(position, lookat, upVector): stat_data = {} + stat_file = Path(dirname) / Path("stat.json") + if stat_file.exists(): + with open(stat_file, "r") as openfile: + stat_data = json.load(openfile) + + with open(Path(dirname) / Path("stat.csv"), 'w', newline='') as csvfile: + fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", + "normL2Depth", "normInfDepth"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for name, sd in stat_data.items(): + # dict merge operator | is not supported until 3.9 + sdname = sd + sdname["model"] = name + writer.writerow(sdname) + for model_name, model_fname, texture_fname in all_models: print(model_name) verts, list_indices, normals, colors, texCoords = cv.loadMesh(model_fname) @@ -546,7 +562,7 @@ def lookAtMatrixCal(position, lookat, upVector): } stat_json = json.dumps(stat_data, indent=4) - with open(Path(dirname) / Path("stat.json"), "w") as outfile: + with open(stat_file, "w") as outfile: outfile.write(stat_json) print("...next") From fd9017176e458eb380f369b8e03650f7c72d17b1 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 12 Apr 2024 04:37:09 +0200 Subject: [PATCH 09/17] messy but works --- download_collection.py | 173 +++++++++++++++++++++++++++++++++-------- 1 file changed, 141 insertions(+), 32 deletions(-) diff --git a/download_collection.py b/download_collection.py index fa162c9..202a807 100644 --- a/download_collection.py +++ b/download_collection.py @@ -287,11 +287,64 @@ def lookAtMatrixCal(position, lookat, upVector): return res +# walk over all triangles, collect per-vertex normals +def generateNormals(points, indices): + success = False + maxTri = 8 + while not success and maxTri < 64: + preNormals = np.zeros((points.shape[0], maxTri, 3), dtype=np.float32) + nNormals = np.zeros((points.shape[0],), dtype=np.int) + + pArr = [ np.take(points, indices[:, j], axis=0) for j in range(3) ] + + crossArr = np.cross(pArr[1] - pArr[0], pArr[2] - pArr[0]) + # normalizing, fixing div by zero + crossNorm = np.linalg.norm(crossArr, axis=1) + crossNorm[abs(crossNorm) < 1e-6 ] = 1 + crossNorm3 = np.vstack([crossNorm, crossNorm, crossNorm]).T + crossArr = crossArr * (1.0 / crossNorm3) + + needMoreTri = False + for i in range(crossArr.shape[0]): + tri = indices[i, :] + cross = crossArr[i, :] + + for j in range(3): + idx = tri[j] + n = nNormals[idx] + if n == maxTri: + print(maxTri, " triangles per vertex is not enough, adding 8 more") + needMoreTri = True + break + nNormals[idx] = n+1 + preNormals[idx, n, :] = cross + if needMoreTri: + break + + if needMoreTri: + maxTri += 8 + else: + normals = np.sum(preNormals, axis=1) + # normalizing, fixing div by zero + norm = np.linalg.norm(normals, axis=1) + norm[abs(norm) < 1e-6 ] = 1 + norm3 = np.vstack([norm, norm, norm]).T + normals = normals * (1.0 / norm3) + success = True + + if success: + print ("normals generated") + return normals + else: + raise RuntimeError("too much triangle per vertex") + + # ================================================== if __name__ == "__main__": parser = argparse.ArgumentParser(prog="bench3d", description="3D algorithm benchmark for OpenCV") parser.add_argument("--work_dir") parser.add_argument("--verbose", action="store_true") + # see file example_opengl_opengl_testdata_generator parser.add_argument("--renderer_path") parser.add_argument("--debug_pics_dir") @@ -396,21 +449,36 @@ def lookAtMatrixCal(position, lookat, upVector): writer.writerow(sdname) for model_name, model_fname, texture_fname in all_models: + #DEBUG + #if "stanford_lucy" in model_name: + # continue + print(model_name) verts, list_indices, normals, colors, texCoords = cv.loadMesh(model_fname) + if verbose: + print("loading finished") + if texture_fname: texture = cv.imread(texture_fname) / 255.0 - verts = verts[0, :, :] + verts = verts.squeeze() # list_indices is a tuple of 1x3 arrays of dtype=int32 - indices = np.zeros((len(list_indices), 3), dtype=np.int32) - for i, ind in enumerate(list_indices): - if ind.shape == (3, 1): - ind = ind.t() + if not list_indices: + raise ValueError("empty index list") + for ind in list_indices: if ind.shape != (1, 3): - raise ValueError() - indices[i, :] = ind[:] - normals = normals[0, :, :] + raise ValueError("wrong index shape") + + indices = np.array(list_indices, dtype=np.int32).squeeze() + + if len(indices.shape) != 2 or (indices.shape[1] != 3): + raise ValueError("wrong index shape") + + if normals is not None: + normals = normals.squeeze() + + if colors is not None: + colors = colors.squeeze() minx, miny, minz = np.min(verts[:, 0]), np.min(verts[:, 1]), np.min(verts[:, 2]) maxx, maxy, maxz = np.max(verts[:, 0]), np.max(verts[:, 1]), np.max(verts[:, 2]) @@ -418,27 +486,40 @@ def lookAtMatrixCal(position, lookat, upVector): if verbose: print("verts: ", verts.shape) print("indices: ", indices.shape) - print("normals: ", normals.shape) - print("texCoords: ", texCoords.shape) # empty now + print("normals: ", (normals.shape if normals is not None else 0)) + print("colors: ", (colors.shape if colors is not None else 0)) + print("texCoords: ", (texCoords.shape if texCoords is not None else 0)) print("bounding box: [%f...%f, %f...%f, %f...%f]" % (minx, maxx, miny, maxy, minz, maxz)) + nverts = verts.shape[0] + # this could be used for slow and dirty texturing doRemap = False + remapCoords = np.ones((nverts, 3), dtype=np.float32) + + if colors is None: + if texCoords is not None: + # sample vertex color from texture + if doRemap: + colors = np.ones((nverts, 3), dtype=np.float32) + for i in range(nverts): + remapCoords[i, :] = [texCoords[i, 0], 1.0-texCoords[i, 1], 0] + + uArr = ( texCoords[:, 0] * texture.shape[1] - 0.5).astype(np.int) + vArr = ((1.0 - texCoords[:, 1]) * texture.shape[0] - 0.5).astype(np.int) + colors = (texture[vArr, uArr, :]).astype(np.float32) - nverts = verts.shape[0] - texsz = texture.shape[0:2] - texw, texh = texsz[1], texsz[0] - minv = np.array([minx, miny, minz]) - maxv = np.array([maxx, maxy, maxz]) - diffv = 1.0/(maxv - minv) - colors = np.ones((nverts, 3), dtype=np.float32) - for i in range(nverts): - tc = texCoords[i, :] - u, v = int(tc[0] * texw - 0.5), int((1.0-tc[1]) * texh - 0.5) - if doRemap: - colors[i, :] = [tc[0], 1-tc[1], 0] else: - colors[i, :] = texture[v, u, :] + if nverts < 1_000_000: + # generate normals and set colors to normals + normals = generateNormals(verts, indices) + colors = abs(normals) + else: + t = np.arange(0, nverts, dtype=np.float32) + colors = 0.5 + 0.5* np.vstack([np.cos(t / (nverts)), + np.sin(t / (nverts)), + np.sin(t / (nverts * 2))]).astype(np.float32).T + print("colors generated") ctgY = 1./math.tan(fovY / 2.0) ctgX = ctgY / width * height @@ -453,6 +534,7 @@ def lookAtMatrixCal(position, lookat, upVector): lookat = np.array([0.0, 0.0, 0.0], dtype=np.float32) upVector = np.array([0.0, 1.0, 0.0], dtype=np.float32) cameraPose = lookAtMatrixCal(position, lookat, upVector) + scaleCoeff = 65535.0 / zFar depth_buf = np.ones((height, width), dtype=np.float32) * zFar color_buf = np.zeros((height, width, 3), dtype=np.float32) @@ -463,18 +545,22 @@ def lookAtMatrixCal(position, lookat, upVector): color_buf, depth_buf = cv.triangleRasterize(verts, indices, colors, color_buf, depth_buf, cameraPose, fovY, zNear, zFar, settings) + if verbose: + print("rasterized") + if doRemap: - mapx = color_buf[:, :, 0] * texture.shape[1] - 0.5 - mapy = color_buf[:, :, 1] * texture.shape[0] - 0.5 + remap_color_buf, _ = cv.triangleRasterizeColor(verts, indices, remapCoords, color_buf, depth_buf, + cameraPose, fovY, zNear, zFar, settings) + mapx = remap_color_buf[:, :, 0] * texture.shape[1] - 0.5 + mapy = remap_color_buf[:, :, 1] * texture.shape[0] - 0.5 remapped = cv.remap(texture, mapx, mapy, cv.INTER_LINEAR) + cv.imwrite(Path(debug_pics_dir) / Path("remap.png"), remapped * 255.0) colorRasterize = color_buf - depthRasterize = (depth_buf * 1000.0) + depthRasterize = (depth_buf * scaleCoeff) if debug_pics_dir: cv.imwrite(Path(debug_pics_dir) / Path("color_raster.png"), color_buf * 255.0) - if doRemap: - cv.imwrite(Path(debug_pics_dir) / Path("remap.png"), remapped * 255.0) cv.imwrite(Path(debug_pics_dir) / Path("depth_raster.png"), depthRasterize.astype(np.ushort)) cv.imwrite(Path(model_fname).parent / Path("color_raster.png"), color_buf * 255.0) @@ -492,6 +578,9 @@ def lookAtMatrixCal(position, lookat, upVector): colvert_path = Path(model_fname).parent / Path("colvert.ply") cv.saveMesh(colvert_path, vertsToSave, indicesToSave, None, colorsToSave) + if verbose: + print("mesh saved for OpenGL renderer") + rgb_gl_path = Path(model_fname).parent / Path("color.png") depth_gl_path = Path(model_fname).parent / Path("depth.png") renderer_args = [renderer_path] + [ @@ -511,6 +600,7 @@ def lookAtMatrixCal(position, lookat, upVector): "--resy="+str(height), "--zNear="+str(zNear), "--zFar="+str(zFar), + "--scaleCoeff="+str(scaleCoeff), # white/flat/shaded "--shading=shaded", # none/cw/ccw @@ -536,7 +626,7 @@ def lookAtMatrixCal(position, lookat, upVector): depthGl = cv.imread(depth_gl_path, cv.IMREAD_GRAYSCALE | cv.IMREAD_ANYDEPTH).astype(np.float32) depthDiff = depthGl - depthRasterize - threshold = math.floor(zFar * 1000) + threshold = math.floor(zFar * scaleCoeff) maskGl = depthGl < threshold maskRaster = depthRasterize < threshold maskDiff = maskGl != maskRaster @@ -547,8 +637,12 @@ def lookAtMatrixCal(position, lookat, upVector): nzJointMask = np.count_nonzero(jointMask) # maskedDiff = np.ma.masked_array(depthDiff, jointMask) maskedDiff = np.ravel(depthDiff[jointMask]) - normInfDepth = np.linalg.norm(maskedDiff, ord=np.inf) - normL2Depth = np.linalg.norm(maskedDiff, ord=2) / nzJointMask + if nzJointMask: + normInfDepth = np.linalg.norm(maskedDiff, ord=np.inf) + normL2Depth = np.linalg.norm(maskedDiff, ord=2) / nzJointMask + else: + normInfDepth = math.nan + normL2Depth = math.nan print("depth L2: %f Inf: %f" % (normL2Depth, normInfDepth)) cv.imwrite(Path(model_fname).parent / Path("depth_diff.png"), ((depthDiff) + (1 << 15)).astype(np.ushort)) @@ -565,4 +659,19 @@ def lookAtMatrixCal(position, lookat, upVector): with open(stat_file, "w") as outfile: outfile.write(stat_json) - print("...next") +#TODO: no code duplication +stat_file = Path(dirname) / Path("stat.json") +if stat_file.exists(): + with open(stat_file, "r") as openfile: + stat_data = json.load(openfile) + + with open(Path(dirname) / Path("stat.csv"), 'w', newline='') as csvfile: + fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", + "normL2Depth", "normInfDepth"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for name, sd in stat_data.items(): + # dict merge operator | is not supported until 3.9 + sdname = sd + sdname["model"] = name + writer.writerow(sdname) \ No newline at end of file From 8415f02bf30891513d768b8dc582dedbfaa20261 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 12 Apr 2024 05:25:52 +0200 Subject: [PATCH 10/17] better output --- download_collection.py | 44 ++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/download_collection.py b/download_collection.py index 202a807..979b0cd 100644 --- a/download_collection.py +++ b/download_collection.py @@ -581,8 +581,8 @@ def generateNormals(points, indices): if verbose: print("mesh saved for OpenGL renderer") - rgb_gl_path = Path(model_fname).parent / Path("color.png") - depth_gl_path = Path(model_fname).parent / Path("depth.png") + rgb_gl_path = Path(model_fname).parent / Path("color_gl.png") + depth_gl_path = Path(model_fname).parent / Path("depth_gl.png") renderer_args = [renderer_path] + [ "--modelPath="+str(colvert_path), "--custom", @@ -625,19 +625,20 @@ def generateNormals(points, indices): cv.imwrite(Path(model_fname).parent / Path("color_diff.png"), (colorDiff.reshape((height, width, 3)) + 1) * 0.5 * 255.0) depthGl = cv.imread(depth_gl_path, cv.IMREAD_GRAYSCALE | cv.IMREAD_ANYDEPTH).astype(np.float32) + depthGl /= scaleCoeff + depthRasterize /= scaleCoeff depthDiff = depthGl - depthRasterize - threshold = math.floor(zFar * scaleCoeff) - maskGl = depthGl < threshold - maskRaster = depthRasterize < threshold + + maskGl = abs(depthGl - zFar) < 1e-5 + maskRaster = abs(depthRasterize - zFar) < 1e-5 maskDiff = maskGl != maskRaster nzDepthDiff = np.count_nonzero(maskDiff) print("depth nzdiff: %d" % (nzDepthDiff)) jointMask = maskRaster & maskGl nzJointMask = np.count_nonzero(jointMask) - # maskedDiff = np.ma.masked_array(depthDiff, jointMask) - maskedDiff = np.ravel(depthDiff[jointMask]) if nzJointMask: + maskedDiff = np.ravel(depthDiff[jointMask]) normInfDepth = np.linalg.norm(maskedDiff, ord=np.inf) normL2Depth = np.linalg.norm(maskedDiff, ord=2) / nzJointMask else: @@ -646,6 +647,8 @@ def generateNormals(points, indices): print("depth L2: %f Inf: %f" % (normL2Depth, normInfDepth)) cv.imwrite(Path(model_fname).parent / Path("depth_diff.png"), ((depthDiff) + (1 << 15)).astype(np.ushort)) + cv.imwrite(Path(model_fname).parent / Path("mask_gl.png"), maskGl.astype(np.ubyte) * 255) + cv.imwrite(Path(model_fname).parent / Path("mask_raster.png"), maskRaster.astype(np.ubyte) * 255) stat_data[str(model_name)] = { "normL2Rgb" : str(normL2Rgb), @@ -659,19 +662,14 @@ def generateNormals(points, indices): with open(stat_file, "w") as outfile: outfile.write(stat_json) -#TODO: no code duplication -stat_file = Path(dirname) / Path("stat.json") -if stat_file.exists(): - with open(stat_file, "r") as openfile: - stat_data = json.load(openfile) - - with open(Path(dirname) / Path("stat.csv"), 'w', newline='') as csvfile: - fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", - "normL2Depth", "normInfDepth"] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for name, sd in stat_data.items(): - # dict merge operator | is not supported until 3.9 - sdname = sd - sdname["model"] = name - writer.writerow(sdname) \ No newline at end of file + #TODO: no code duplication + with open(Path(dirname) / Path("stat.csv"), 'w', newline='') as csvfile: + fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", + "normL2Depth", "normInfDepth"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for name, sd in stat_data.items(): + # dict merge operator | is not supported until 3.9 + sdname = sd + sdname["model"] = name + writer.writerow(sdname) \ No newline at end of file From 597e15c82de04c5655cd3b42051458ae0387f90f Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 12 Apr 2024 22:42:58 +0200 Subject: [PATCH 11/17] writeCSV --- download_collection.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/download_collection.py b/download_collection.py index 979b0cd..39a1dfe 100644 --- a/download_collection.py +++ b/download_collection.py @@ -339,6 +339,18 @@ def generateNormals(points, indices): raise RuntimeError("too much triangle per vertex") +def writeCsv(path, stat_data): + with open(path, 'w', newline='') as csvfile: + fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", + "normL2Depth", "normInfDepth"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for name, sd in stat_data.items(): + # dict merge operator | is not supported until 3.9 + sdname = sd + sdname["model"] = name + writer.writerow(sdname) + # ================================================== if __name__ == "__main__": parser = argparse.ArgumentParser(prog="bench3d", description="3D algorithm benchmark for OpenCV") @@ -433,20 +445,15 @@ def generateNormals(points, indices): stat_data = {} stat_file = Path(dirname) / Path("stat.json") + csv_file = Path(dirname) / Path("stat.csv") + + for file in [stat_file, csv_file]: + file.unlink(missing_ok=True) + if stat_file.exists(): with open(stat_file, "r") as openfile: stat_data = json.load(openfile) - - with open(Path(dirname) / Path("stat.csv"), 'w', newline='') as csvfile: - fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", - "normL2Depth", "normInfDepth"] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for name, sd in stat_data.items(): - # dict merge operator | is not supported until 3.9 - sdname = sd - sdname["model"] = name - writer.writerow(sdname) + writeCsv(csv_file, stat_data) for model_name, model_fname, texture_fname in all_models: #DEBUG @@ -661,15 +668,4 @@ def generateNormals(points, indices): stat_json = json.dumps(stat_data, indent=4) with open(stat_file, "w") as outfile: outfile.write(stat_json) - - #TODO: no code duplication - with open(Path(dirname) / Path("stat.csv"), 'w', newline='') as csvfile: - fieldnames = [ "model", "normL2Rgb", "normInfRgb", "nzDepthDiff", - "normL2Depth", "normInfDepth"] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for name, sd in stat_data.items(): - # dict merge operator | is not supported until 3.9 - sdname = sd - sdname["model"] = name - writer.writerow(sdname) \ No newline at end of file + writeCsv(csv_file, stat_data) \ No newline at end of file From d298d124841a120259a6f6f14557a9dfbfba131f Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 12 Apr 2024 22:44:19 +0200 Subject: [PATCH 12/17] remove files --- download_collection.py | 79 +++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/download_collection.py b/download_collection.py index 39a1dfe..7737bbf 100644 --- a/download_collection.py +++ b/download_collection.py @@ -461,12 +461,41 @@ def writeCsv(path, stat_data): # continue print(model_name) + + colvert_path = Path(model_fname).parent / Path("colvert.ply") + + remap_path = Path(debug_pics_dir) / Path("remap.png") + debug_color_path = Path(debug_pics_dir) / Path("color_raster.png") + debug_depth_path = Path(debug_pics_dir) / Path("depth_raster.png") + + color_raster_path = Path(model_fname).parent / Path("color_raster.png") + depth_raster_path = Path(model_fname).parent / Path("depth_raster.png") + + rgb_gl_path = Path(model_fname).parent / Path("color_gl.png") + depth_gl_path = Path(model_fname).parent / Path("depth_gl.png") + + color_diff_path = Path(model_fname).parent / Path("color_diff.png") + + depth_diff_path = Path(model_fname).parent / Path("depth_diff.png") + mask_gl_path = Path(model_fname).parent / Path("mask_gl.png") + mask_raster_path = Path(model_fname).parent / Path("mask_raster.png") + + for file in [ colvert_path, remap_path, + color_raster_path, depth_raster_path, + rgb_gl_path, depth_gl_path, + color_diff_path, depth_diff_path, mask_gl_path, mask_raster_path ]: + file.unlink(missing_ok=True) + + if verbose: + print("temp files removed") + verts, list_indices, normals, colors, texCoords = cv.loadMesh(model_fname) if verbose: print("loading finished") if texture_fname: texture = cv.imread(texture_fname) / 255.0 + verts = verts.squeeze() # list_indices is a tuple of 1x3 arrays of dtype=int32 @@ -528,6 +557,23 @@ def writeCsv(path, stat_data): np.sin(t / (nverts * 2))]).astype(np.float32).T print("colors generated") + # save mesh for OpenGL renderer + + vertsToSave = np.expand_dims(verts, axis=0) + colorsToSave = np.expand_dims(colors, axis=0) + colorsToSave = colorsToSave[:, :, ::-1] + indicesToSave = [] + for i in range(indices.shape[0]): + ix = indices[i, :] + indicesToSave.append(ix) + + cv.saveMesh(colvert_path, vertsToSave, indicesToSave, None, colorsToSave) + + if verbose: + print("mesh saved for OpenGL renderer") + + # rasterize + ctgY = 1./math.tan(fovY / 2.0) ctgX = ctgY / width * height zat = maxz + max([abs(maxy) * ctgY, @@ -561,35 +607,20 @@ def writeCsv(path, stat_data): mapx = remap_color_buf[:, :, 0] * texture.shape[1] - 0.5 mapy = remap_color_buf[:, :, 1] * texture.shape[0] - 0.5 remapped = cv.remap(texture, mapx, mapy, cv.INTER_LINEAR) - cv.imwrite(Path(debug_pics_dir) / Path("remap.png"), remapped * 255.0) + cv.imwrite(remap_path, remapped * 255.0) colorRasterize = color_buf depthRasterize = (depth_buf * scaleCoeff) if debug_pics_dir: - cv.imwrite(Path(debug_pics_dir) / Path("color_raster.png"), color_buf * 255.0) - cv.imwrite(Path(debug_pics_dir) / Path("depth_raster.png"), depthRasterize.astype(np.ushort)) + cv.imwrite(debug_color_path, color_buf * 255.0) + cv.imwrite(debug_depth_path, depthRasterize.astype(np.ushort)) - cv.imwrite(Path(model_fname).parent / Path("color_raster.png"), color_buf * 255.0) - cv.imwrite(Path(model_fname).parent / Path("depth_raster.png"), depthRasterize.astype(np.ushort)) + cv.imwrite(color_raster_path, color_buf * 255.0) + cv.imwrite(depth_raster_path, depthRasterize.astype(np.ushort)) # send mesh to OpenGL rasterizer - vertsToSave = np.expand_dims(verts, axis=0) - colorsToSave = np.expand_dims(colors, axis=0) - colorsToSave[0, :, ::] = colorsToSave[0, :, ::-1] - indicesToSave = [] - for i in range(indices.shape[0]): - ix = indices[i, :] - indicesToSave.append(ix) - colvert_path = Path(model_fname).parent / Path("colvert.ply") - cv.saveMesh(colvert_path, vertsToSave, indicesToSave, None, colorsToSave) - - if verbose: - print("mesh saved for OpenGL renderer") - - rgb_gl_path = Path(model_fname).parent / Path("color_gl.png") - depth_gl_path = Path(model_fname).parent / Path("depth_gl.png") renderer_args = [renderer_path] + [ "--modelPath="+str(colvert_path), "--custom", @@ -629,7 +660,7 @@ def writeCsv(path, stat_data): normL2Rgb = np.linalg.norm(colorDiff, ord=2) / (width * height) print("rgb L2: %f Inf: %f" % (normL2Rgb, normInfRgb)) - cv.imwrite(Path(model_fname).parent / Path("color_diff.png"), (colorDiff.reshape((height, width, 3)) + 1) * 0.5 * 255.0) + cv.imwrite(color_diff_path, (colorDiff.reshape((height, width, 3)) + 1) * 0.5 * 255.0) depthGl = cv.imread(depth_gl_path, cv.IMREAD_GRAYSCALE | cv.IMREAD_ANYDEPTH).astype(np.float32) depthGl /= scaleCoeff @@ -653,9 +684,9 @@ def writeCsv(path, stat_data): normL2Depth = math.nan print("depth L2: %f Inf: %f" % (normL2Depth, normInfDepth)) - cv.imwrite(Path(model_fname).parent / Path("depth_diff.png"), ((depthDiff) + (1 << 15)).astype(np.ushort)) - cv.imwrite(Path(model_fname).parent / Path("mask_gl.png"), maskGl.astype(np.ubyte) * 255) - cv.imwrite(Path(model_fname).parent / Path("mask_raster.png"), maskRaster.astype(np.ubyte) * 255) + cv.imwrite(depth_diff_path, ((depthDiff) + (1 << 15)).astype(np.ushort)) + cv.imwrite(mask_gl_path, maskGl.astype(np.ubyte) * 255) + cv.imwrite(mask_raster_path, maskRaster.astype(np.ubyte) * 255) stat_data[str(model_name)] = { "normL2Rgb" : str(normL2Rgb), From 2fc618f9495ad61b640b5fd34a90351fe746658c Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 12 Apr 2024 22:44:41 +0200 Subject: [PATCH 13/17] colors --- download_collection.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/download_collection.py b/download_collection.py index 7737bbf..95a5b15 100644 --- a/download_collection.py +++ b/download_collection.py @@ -552,10 +552,11 @@ def writeCsv(path, stat_data): colors = abs(normals) else: t = np.arange(0, nverts, dtype=np.float32) - colors = 0.5 + 0.5* np.vstack([np.cos(t / (nverts)), - np.sin(t / (nverts)), - np.sin(t / (nverts * 2))]).astype(np.float32).T - print("colors generated") + colors = 0.5 + 0.5 * np.vstack([np.cos(t * (2 * math.pi / nverts)), + np.sin(t * (3 * math.pi / nverts)), + np.sin(t * (2 * math.pi / nverts))]).astype(np.float32).T + if verbose: + print("colors generated") # save mesh for OpenGL renderer From 0a51cc1a0e90994ecc4201203131afe3efc3dce4 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Fri, 12 Apr 2024 22:45:00 +0200 Subject: [PATCH 14/17] masks fixed --- download_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/download_collection.py b/download_collection.py index 95a5b15..fbce9c9 100644 --- a/download_collection.py +++ b/download_collection.py @@ -668,8 +668,8 @@ def writeCsv(path, stat_data): depthRasterize /= scaleCoeff depthDiff = depthGl - depthRasterize - maskGl = abs(depthGl - zFar) < 1e-5 - maskRaster = abs(depthRasterize - zFar) < 1e-5 + maskGl = abs(depthGl - zFar) > 1e-6 + maskRaster = abs(depthRasterize - zFar) > 1e-6 maskDiff = maskGl != maskRaster nzDepthDiff = np.count_nonzero(maskDiff) print("depth nzdiff: %d" % (nzDepthDiff)) From 2345f48b740788f3e6334d1d0ce1e4e62273c615 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Mon, 15 Apr 2024 15:44:43 +0200 Subject: [PATCH 15/17] import error message --- download_collection.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/download_collection.py b/download_collection.py index fbce9c9..c199555 100644 --- a/download_collection.py +++ b/download_collection.py @@ -12,8 +12,11 @@ import math import numpy as np -# cv2 library should be in PYTHONPATH variable -import cv2 as cv +try: + import cv2 as cv +except ImportError: + raise ImportError("Failed to import OpenCV, please check that cv2 library is in PYTHONPATH env variable, " + "for example use the path {opencv_build_dir}/lib/python3/") if sys.version_info[0] < 3 or sys.version_info[1] < 5: raise Exception("Python 3.5 or greater is required. Try running `python3 download_collection.py`") From 9c5ed4e2e6e7efd29be4c5fb73d13823fbe87612 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Mon, 15 Apr 2024 15:45:19 +0200 Subject: [PATCH 16/17] help added --- download_collection.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/download_collection.py b/download_collection.py index c199555..8fcf2fd 100644 --- a/download_collection.py +++ b/download_collection.py @@ -355,13 +355,33 @@ def writeCsv(path, stat_data): writer.writerow(sdname) # ================================================== + if __name__ == "__main__": - parser = argparse.ArgumentParser(prog="bench3d", description="3D algorithm benchmark for OpenCV") - parser.add_argument("--work_dir") - parser.add_argument("--verbose", action="store_true") - # see file example_opengl_opengl_testdata_generator - parser.add_argument("--renderer_path") - parser.add_argument("--debug_pics_dir") + parser = argparse.ArgumentParser(prog="bench3d", description="3D algorithm benchmark for OpenCV", + formatter_class=argparse.RawTextHelpFormatter, + epilog="The tool works with the latest OpenCV 5.x, " + "a proper path to OpenCV library should be provided in PYTHONPATH env. variable, " + "usually it is {opencv_build_dir}/lib/python3/\n" + "The script first downloads dataset contents to {work_dir}/contents.json " + "then downloads and unpacks the models and their thumbnails. " + "If this process is interrupted, you can start the script again and " + "it will continue from the same point.\n" + "Next, the benchmark for triangleRasterize() starts: each model is " + "rendered by both OpenCV and OpenGL reference tool, results are compared.\n" + "The statistics is saved to {work_dir}/stat.json and {work_dir}/stat.csv " + "after each model processed.\n" + "NOTE: statistics is erased at each script start\n" + "NOTE #2: stanford_* models are quite big, may require several minutes " + "and up to 12 Gb RAM per each") + parser.add_argument("--work_dir", required=True, help="a directory to store downloaded dataset and generated files") + parser.add_argument("--verbose", action="store_true", + help="indicates whether to display more information to the console or not") + parser.add_argument("--renderer_path", required=True, + help="path to OpenGL tool to render models, usually it is " + "{opencv_build_dir}/bin/example_opengl_opengl_testdata_generator") + parser.add_argument("--debug_pics_dir", + help="optional, a directory to store temporary images " + "showing in realtime what is being rendered right now") args = parser.parse_args() From 2df70382cbe615ec8ef846e7d85ca60b28a8657f Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Mon, 15 Apr 2024 15:45:35 +0200 Subject: [PATCH 17/17] minor --- download_collection.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/download_collection.py b/download_collection.py index 8fcf2fd..e66e948 100644 --- a/download_collection.py +++ b/download_collection.py @@ -479,10 +479,6 @@ def writeCsv(path, stat_data): writeCsv(csv_file, stat_data) for model_name, model_fname, texture_fname in all_models: - #DEBUG - #if "stanford_lucy" in model_name: - # continue - print(model_name) colvert_path = Path(model_fname).parent / Path("colvert.ply")