Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanse-like refactorings #3

Open
wants to merge 2 commits into
base: ticket-11395
Choose a base branch
from
Open

Cleanse-like refactorings #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e6e4ab1
Refactor fixpyramids to act more like cleanse
joshmoore Oct 4, 2013
34bbbc6
Refactor to use os.walk
joshmoore Oct 4, 2013
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 15 additions & 31 deletions components/tools/OmeroPy/src/omero/plugins/admin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ def __init__(this, name, help, wait=False):

Action("ice", """Drop user into icegridadmin console or execute arguments""")

Action("fixpyramids", """Remove empty pyramid pixels files""")
fixpyramids = Action("fixpyramids", """Remove empty pyramid pixels files""").parser
# See cleanse options below

Action("diagnostics", """Run a set of checks on the current, preferably active server""")

Expand Down Expand Up @@ -194,9 +195,11 @@ def __init__(this, name, help, wait=False):
bin/omero admin cleanse /volumes/data/OMERO # Delete from a standard location.

""").parser
cleanse.add_argument("--dry-run", action = "store_true", help = "Print out which files would be deleted")
cleanse.add_argument("data_dir", type=DirectoryType(), help = "omero.data.dir directory value (e.g. /OMERO")
cleanse.add_login_arguments()

for x in (cleanse, fixpyramids):
x.add_argument("--dry-run", action = "store_true", help = "Print out which files would be deleted")
x.add_argument("data_dir", type=DirectoryType(), help = "omero.data.dir directory value (e.g. /OMERO")
x.add_login_arguments()

Action("checkwindows", """Run simple check of the local installation (Windows-only)""")
Action("checkice", """Run simple check of the Ice installation""")
Expand Down Expand Up @@ -632,32 +635,13 @@ def ice(self, args):

@with_config
def fixpyramids(self, args, config):
self.check_access();
config = config.as_map()
omero_data_dir = '/OMERO'
try:
omero_data_dir = config['omero.data.dir']
except KeyError:
pass

# look for any pyramid files with length 0
# if there is no matching .*.tmp or .*.pyr_lock file, then
# the pyramid file will be removed

pixels_dir = os.path.join(omero_data_dir, "Pixels")
for f in os.listdir(pixels_dir):
pixels_file = os.path.join(pixels_dir, f)
length = os.path.getsize(pixels_file)
if length == 0 and f.endswith("_pyramid"):
delete_pyramid = True
for lockfile in os.listdir(pixels_dir):
if lockfile.startswith("." + f) and (lockfile.endswith(".tmp") or lockfile.endswith(".pyr_lock")):
delete_pyramid = False
break

if delete_pyramid:
self.ctx.out("Removing %s" % f)
os.remove(pixels_file)
self.check_access()
from omero.util.cleanse import fixpyramids
client = self.ctx.conn(args)
key = client.getSessionId()
fixpyramids(data_dir=args.data_dir, dry_run=args.dry_run, \
query_service=client.sf.getQueryService(), \
config_service=client.sf.getConfigService())

@with_config
def diagnostics(self, args, config):
Expand Down Expand Up @@ -714,7 +698,7 @@ def exists(p):
lcl = l.lower()
found_err = lcl.find("error") >= 0
found_warn = lcl.find("warn") >= 0

if found_err:
err += 1
elif found_warn:
Expand Down
42 changes: 38 additions & 4 deletions components/tools/OmeroPy/src/omero/util/cleanse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,15 +139,15 @@ def do_cleanse(self):
object_id = omero.rtypes.rlong(-1)
except ValueError:
object_id = omero.rtypes.rlong(-1)
object_ids.append(object_id)
object_ids.append(object_id)

parameters = omero.sys.Parameters()
parameters.map = {'ids': omero.rtypes.rlist(object_ids)}
rows = self.query_service.projection(
"select o.id from %s as o where o.id in (:ids)" % self.object_type,
parameters, {"omero.group":"-1"})
existing_ids = [cols[0].val for cols in rows]

for i, object_id in enumerate(object_ids):
path = self.deferred_paths[i]
if object_id.val not in existing_ids:
Expand Down Expand Up @@ -182,13 +182,14 @@ def __str__(self):
return "Cleansing context: %d files (%d bytes)" % \
(len(self.cleansed), self.bytes_cleansed)

def cleanse(data_dir, query_service, dry_run = False, config_service = None):

def initial_check(config_service):
#
# Compare server versions. See ticket #3123
#
if config_service is None:
print "No config service provided! Waiting 10 seconds to allow cancellation"
print ("No config service provided! "
"Waiting 10 seconds to allow cancellation")
from threading import Event
Event().wait(10)

Expand All @@ -198,6 +199,9 @@ def cleanse(data_dir, query_service, dry_run = False, config_service = None):
print "Server version is too old! (%s) Aborting..." % server_version
sys.exit(3)


def cleanse(data_dir, query_service, dry_run=False, config_service=None):
initial_check(config_service)
try:
cleanser = ""
for directory in SEARCH_DIRECTORIES:
Expand All @@ -213,6 +217,36 @@ def cleanse(data_dir, query_service, dry_run = False, config_service = None):
if dry_run:
print cleanser


def fixpyramids(data_dir, query_service, dry_run=False, config_service=None):
initial_check(config_service)

# look for any pyramid files with length 0
# if there is no matching .*.tmp or .*.pyr_lock file, then
# the pyramid file will be removed

pixels_dir = os.path.join(data_dir, "Pixels")
for root, dirs, files in os.walk(pixels_dir):
for f in files:
pixels_file = os.path.join(root, f)
length = os.path.getsize(pixels_file)
if length == 0 and f.endswith("_pyramid"):
delete_pyramid = True
for lockfile in os.listdir(pixels_dir):
if lockfile.startswith("." + f) and \
(lockfile.endswith(".tmp") or
lockfile.endswith(".pyr_lock")):
delete_pyramid = False
break

if delete_pyramid:
if dry_run:
print "Would remove %s" % f
else:
print "Removing %s" % f
os.remove(pixels_file)


def main():
"""
Default main() that performs OMERO data directory cleansing.
Expand Down