diff --git a/.gitattributes b/.gitattributes index fe17464c52..d7bf76eb2b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ *.eps binary *.fits binary +# Pickle files are binary (protocol-0 pickles are corrupted by CRLF conversion) +*.p binary +*.pkl binary diff --git a/galsim/_astropy_shim.py b/galsim/_astropy_shim.py new file mode 100644 index 0000000000..80dc1a26f7 --- /dev/null +++ b/galsim/_astropy_shim.py @@ -0,0 +1,108 @@ +# Copyright (c) 2012-2023 by the GalSim developers team on GitHub +# https://github.com/GalSim-developers +# +# This file is part of GalSim: The modular galaxy image simulation toolkit. +# https://github.com/GalSim-developers/GalSim +# +# GalSim is free software: redistribution and use in source and binary forms, +# with or without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions, and the disclaimer given in the accompanying LICENSE +# file. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the disclaimer given in the documentation +# and/or other materials provided with the distribution. +# +"""Single-probe shim around the astropy modules GalSim imports at top level. + +astropy.units cannot initialise inside frozen binaries (Nuitka onefile, +PyInstaller-style bundles): its PLY parser builds grammar tables through +``sys._getframe()`` locals introspection that compiled frames do not provide +(see astropy/astropy#15069). GalSim only needs astropy.units for +spectral/chromatic features; the geometry, zernike and photon-shooting paths +never touch it. + +This module probes astropy ONCE per process. In any normal source install the +real modules are re-exported unchanged, so behaviour is identical to importing +astropy directly. When astropy.units cannot initialise, inert fallbacks are +exported instead: imports succeed, config parameter schemas keep working +(upper-case attribute names resolve to a placeholder type that is safe in +``isinstance`` checks), and spectral features fail only if actually exercised. +""" + +__all__ = ["units", "constants", "wcs", "Quantity", "Unit", + "HAVE_ASTROPY_UNITS"] + + +class _FallbackType: + """Placeholder for unit/quantity classes in isinstance() checks.""" + + def __init__(self, *args, **kwargs): + pass + + +class _FallbackValue: + """Inert placeholder surviving attribute access, calls and arithmetic.""" + __slots__ = () + + def __call__(self, *args, **kwargs): + return self + + def __getattr__(self, name): + # Keep introspection honest: probing __file__/__wrapped__/etc. must + # raise, or tools that scan objects (inspect, copy, pickle) misbehave. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _FallbackType if name[:1].isupper() else self + + def __mul__(self, other): + return self + __rmul__ = __truediv__ = __rtruediv__ = __pow__ = __mul__ + __add__ = __radd__ = __sub__ = __rsub__ = __mul__ + + def __float__(self): + return 1.0 + + def __repr__(self): + return "" + + +_FALLBACK_VALUE = _FallbackValue() + + +class _FallbackModule: + """Module stand-in: upper-case attrs -> placeholder type, else values.""" + + def __init__(self, name): + self._name = name + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _FallbackType if name[:1].isupper() else _FALLBACK_VALUE + + def __repr__(self): + return "" % self._name + + +try: + from astropy import units, constants + HAVE_ASTROPY_UNITS = True +except Exception: # pragma: no cover - frozen builds without working units + units = _FallbackModule("astropy.units") + constants = _FallbackModule("astropy.constants") + HAVE_ASTROPY_UNITS = False + +if HAVE_ASTROPY_UNITS: + try: + import astropy.wcs as wcs + except Exception: # pragma: no cover - degrade wcs independently + wcs = _FallbackModule("astropy.wcs") +else: + wcs = _FallbackModule("astropy.wcs") + +# Names galsim.config imports directly. +Quantity = units.Quantity +Unit = units.Unit diff --git a/galsim/airy.py b/galsim/airy.py index b131248425..3673307af1 100644 --- a/galsim/airy.py +++ b/galsim/airy.py @@ -19,7 +19,7 @@ __all__ = [ 'Airy' ] import math -import astropy.units as u +from ._astropy_shim import units as u from . import _galsim from .gsobject import GSObject diff --git a/galsim/bandpass.py b/galsim/bandpass.py index c5c0fbad89..a7be7182b2 100644 --- a/galsim/bandpass.py +++ b/galsim/bandpass.py @@ -20,9 +20,9 @@ import numpy as np import os -from astropy import units +from ._astropy_shim import units from numbers import Real -from pathlib import PosixPath +from pathlib import PurePath from .errors import GalSimRangeError, GalSimValueError, GalSimIncompatibleValuesError from .table import LookupTable, _LookupTable @@ -228,7 +228,7 @@ def _initialize_tp(self): if self._tp is not None: pass - elif isinstance(self._orig_tp, (basestring, PosixPath)): + elif isinstance(self._orig_tp, (basestring, PurePath)): isfile, filename = utilities.check_share_file(self._orig_tp, 'bandpasses') if isfile: self._tp = LookupTable.from_file(filename, interpolant=self.interpolant) diff --git a/galsim/chromatic.py b/galsim/chromatic.py index 581aa23bc8..889f36fc04 100644 --- a/galsim/chromatic.py +++ b/galsim/chromatic.py @@ -25,7 +25,7 @@ import math import numpy as np -from astropy import units +from ._astropy_shim import units import copy from .gsobject import GSObject diff --git a/galsim/config/bandpass.py b/galsim/config/bandpass.py index fd768b4168..fc88666a92 100644 --- a/galsim/config/bandpass.py +++ b/galsim/config/bandpass.py @@ -17,7 +17,7 @@ # import logging -from astropy.units import Quantity, Unit +from .._astropy_shim import Quantity, Unit from .util import LoggerWrapper from .value import ParseValue, GetAllParams, GetIndex diff --git a/galsim/config/extra.py b/galsim/config/extra.py index 634dfa6751..89fa6fb44d 100644 --- a/galsim/config/extra.py +++ b/galsim/config/extra.py @@ -36,6 +36,18 @@ # builder classes that will perform the different processing functions. valid_extra_outputs = {} + +class _OutputManager(SafeManager): + """Manager subclass used by `SetupExtraOutput` to proxy work-space containers across + processes. + + Defined at module scope (rather than nested inside SetupExtraOutput) so the manager type is + picklable under the 'spawn' start method on Windows, where the manager's server process + receives the manager type by reference. + """ + pass + + def SetupExtraOutput(config, logger=None): """ Set up the extra output items as necessary, including building Managers for the work @@ -61,13 +73,11 @@ def SetupExtraOutput(config, logger=None): ParseValue(config['image'], 'nproc', config, int)[0] != 1 ) if use_manager and 'output_manager' not in config: - class OutputManager(SafeManager): pass - # We'll use a list and a dict as work space to do the extra output processing. - OutputManager.register('dict', dict, DictProxy) - OutputManager.register('list', list, ListProxy) + _OutputManager.register('dict', dict, DictProxy) + _OutputManager.register('list', list, ListProxy) # Start up the output_manager - config['output_manager'] = OutputManager() + config['output_manager'] = _OutputManager() with single_threaded(): config['output_manager'].start() diff --git a/galsim/config/input.py b/galsim/config/input.py index b6afb0cd0c..8546a3963e 100644 --- a/galsim/config/input.py +++ b/galsim/config/input.py @@ -59,6 +59,15 @@ def InputProxy(target): """ Create a derived NamespaceProxy class for `target`. """ + # If we already made a proxy type for this target, reuse it. This also makes the + # dynamically generated class picklable by reference (needed for the 'spawn' start + # method, e.g. on Windows), since we publish it as a module attribute below. + # Note: use globals() rather than getattr to avoid recursing into the module-level + # __getattr__ defined below. + proxy_name = target.__name__ + "_Proxy" + if proxy_name in globals(): + return globals()[proxy_name] + # This bit follows what multiprocessing.managers.MakeProxy normally does. dic = {} public_methods = [m for m in dir(target) if m[0] != '_'] @@ -75,8 +84,37 @@ def InputProxy(target): # Expose all the public methods and also __getattribute__ and __setattr__. ProxyType._exposed_ = tuple(public_methods + ['__getattribute__', '__setattr__']) + # Publish the class as a module attribute so pickle can find it by reference. + ProxyType.__module__ = __name__ + globals()[proxy_name] = ProxyType + return ProxyType + +def __getattr__(name): + # PEP 562 module-level __getattr__. + # When pickle looks up a dynamically generated proxy class by reference (e.g. + # galsim.config.input.Catalog_Proxy) in a freshly spawned process, InputProxy has not + # been called there yet, so the attribute doesn't exist. Regenerate it on demand from + # the registered input types. + if name.endswith('_Proxy'): + target_name = name[:-len('_Proxy')] + for loader in valid_input_types.values(): + if loader.init_func.__name__ == target_name: + return InputProxy(loader.init_func) + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + + +class _InputManager(SafeManager): + """Manager subclass used by `ProcessInput` to proxy input objects across processes. + + Defined at module scope (rather than nested inside ProcessInput) so the manager type is + picklable under the 'spawn' start method on Windows, where the manager's server process + receives the manager type by reference. + """ + pass + + def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False): """ Process the input field, reading in any specified input files or setting up @@ -136,9 +174,7 @@ def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False): ParseValue(config['output'], 'nproc', config, int)[0] != 1) ) ) if use_manager and '_input_manager' not in config: - class InputManager(SafeManager): pass - - # Register each input field with the InputManager class + # Register each input field with the _InputManager class for key in all_keys: fields = config['input'][key] nfields = len(fields) if isinstance(fields, list) else 1 @@ -146,9 +182,9 @@ class InputManager(SafeManager): pass tag = key + str(num) init_func = valid_input_types[key].init_func proxy = InputProxy(init_func) - InputManager.register(tag, init_func, proxy) + _InputManager.register(tag, init_func, proxy) # Start up the input_manager - config['_input_manager'] = InputManager() + config['_input_manager'] = _InputManager() with single_threaded(): # Starting in python 3.12, there is a deprecation warning about using fork when # a process is multithreaded. This can get triggered here by the start() diff --git a/galsim/config/sed.py b/galsim/config/sed.py index 1f87d5a827..de6c033bf2 100644 --- a/galsim/config/sed.py +++ b/galsim/config/sed.py @@ -16,7 +16,7 @@ # and/or other materials provided with the distribution. # -from astropy.units import Quantity, Unit +from .._astropy_shim import Quantity, Unit from .util import LoggerWrapper from .value import ParseValue, GetAllParams, GetIndex diff --git a/galsim/config/util.py b/galsim/config/util.py index 0c28dfafeb..1edf4abbfd 100644 --- a/galsim/config/util.py +++ b/galsim/config/util.py @@ -37,6 +37,15 @@ # We make it a settable parameter here really for unit testing. # I don't think there is any reason for end users to want to set this. +def _get_mp_context(): + """Return the 'fork' multiprocessing context, falling back to 'spawn' on + platforms (Windows) where 'fork' is unavailable. + """ + try: + return get_context('fork') + except ValueError: + return get_context('spawn') + def MergeConfig(config1, config2, logger=None): """ Merge config2 into config1 such that it has all the information from either config1 or @@ -240,8 +249,8 @@ def recursive_copy(d): return config1 class SafeManager(BaseManager): - """There are a few places we need a Manager object. This one uses the 'fork' context, - rather than whatever the default is on your system (which may be fork or may be spawn). + """There are a few places we need a Manager object. This one prefers the 'fork' context, + falling back to 'spawn' where 'fork' is unavailable (e.g. Windows); see _get_mp_context. Starting in python 3.8, the spawn context started becoming more popular. It's supposed to be safer, but it wants to pickle a lot of things that aren't picklable, so it doesn't work. @@ -249,7 +258,17 @@ class SafeManager(BaseManager): only have one place to change this is there is a different strategy that works better. """ def __init__(self): - super(SafeManager, self).__init__(ctx=get_context('fork')) + super(SafeManager, self).__init__(ctx=_get_mp_context()) + + +class _LoggerManager(SafeManager): + """Manager subclass used by `GetLoggerProxy` to proxy a logger across processes. + + Defined at module scope (rather than nested inside GetLoggerProxy) so the manager type is + picklable under the 'spawn' start method on Windows, where the manager's server process + receives the manager type by reference. + """ + pass def GetLoggerProxy(logger): @@ -263,10 +282,9 @@ def GetLoggerProxy(logger): a proxy for the given logger """ if logger: - class LoggerManager(SafeManager): pass logger_generator = SimpleGenerator(logger) - LoggerManager.register('logger', callable = logger_generator) - logger_manager = LoggerManager() + _LoggerManager.register('logger', callable = logger_generator) + logger_manager = _LoggerManager() with single_threaded(): logger_manager.start() logger_proxy = logger_manager.logger() @@ -669,6 +687,70 @@ def UpdateConfig(config, new_params, logger=None): SetInConfig(config, key, value, logger) +def _mp_worker(task_queue, results_queue, config, logger, initializers, initargs, item, job_func): + """Run one worker process for `MultiProcess`. + + Defined at module scope (rather than nested inside MultiProcess) so it is picklable and + therefore usable under the 'spawn' start method on Windows, where 'fork' is unavailable. + ``item`` and ``job_func``, previously captured from the enclosing scope, are now passed in + explicitly. + """ + proc = current_process().name + + # Custom modules listed in config['modules'] register their types via import side effects. + # Under the 'spawn' start method (e.g. on Windows), this fresh process hasn't imported them, + # so re-import them here to rebuild the registries. Under 'fork' they are already in + # sys.modules, so this is essentially free. + from .process import ImportModules # Local import; module-level would be circular. + ImportModules(config) + + for init, args in zip(initializers, initargs): + init(*args) + + # The logger object passed in here is a proxy object. This means that all the arguments + # to any logging commands are passed through the pipe to the real Logger object on the + # other end of the pipe. This tends to produce a lot of unnecessary communication, since + # most of those commands don't actually produce any output (e.g. logger.debug(..) commands + # when the logging level is not DEBUG). So it is helpful to wrap this object in a + # LoggerWrapper that checks whether it is worth sending the arguments back to the original + # Logger before calling the functions. + logger = LoggerWrapper(logger) + + if 'profile' in config and config['profile']: + pr = cProfile.Profile() + pr.enable() + else: + pr = None + + for task in iter(task_queue.get, 'STOP'): + try : + logger.debug('%s: Received job to do %d %ss, starting with %s', + proc,len(task),item,task[0][1]) + for kwargs, k in task: + t1 = time.time() + kwargs['config'] = config + kwargs['logger'] = logger + result = job_func(**kwargs) + t2 = time.time() + results_queue.put( (result, k, t2-t1, proc) ) + except Exception as e: + tr = traceback.format_exc() + logger.debug('%s: Caught exception: %s\n%s',proc,str(e),tr) + results_queue.put( (e, k, tr, proc) ) + logger.debug('%s: Received STOP', proc) + if pr is not None: + pr.disable() + pr.dump_stats(config.get('root', 'galsim') + '-' + str(proc) + '.pstats') + s = StringIO() + sortby = 'time' # Note: This is now called tottime, but time seems to be a valid + # alias for this that is backwards compatible to older versions + # of pstats. + ps = pstats.Stats(pr, stream=s).sort_stats(sortby).reverse_order() + ps.print_stats() + logger.error("*** Start profile for %s ***\n%s\n*** End profile for %s ***", + proc,s.getvalue(),proc) + + def MultiProcess(nproc, config, job_func, tasks, item, logger=None, timeout=900, done_func=None, except_func=None, except_abort=True): """A helper function for performing a task using multiprocessing. @@ -718,67 +800,15 @@ def MultiProcess(nproc, config, job_func, tasks, item, logger=None, timeout=900, """ from .input import worker_init_fns, worker_initargs_fns - # The worker function will be run once in each process. - # It pulls tasks off the task_queue, runs them, and puts the results onto the results_queue - # to send them back to the main process. - # The *tasks* can be made up of more than one *job*. Each job involves calling job_func - # with the kwargs from the list of jobs. - # Each job also carries with it its index in the original list of all jobs. - def worker(task_queue, results_queue, config, logger, initializers, initargs): - proc = current_process().name - - for init, args in zip(initializers, initargs): - init(*args) - - # The logger object passed in here is a proxy object. This means that all the arguments - # to any logging commands are passed through the pipe to the real Logger object on the - # other end of the pipe. This tends to produce a lot of unnecessary communication, since - # most of those commands don't actually produce any output (e.g. logger.debug(..) commands - # when the logging level is not DEBUG). So it is helpful to wrap this object in a - # LoggerWrapper that checks whether it is worth sending the arguments back to the original - # Logger before calling the functions. - logger = LoggerWrapper(logger) - - if 'profile' in config and config['profile']: - pr = cProfile.Profile() - pr.enable() - else: - pr = None - - for task in iter(task_queue.get, 'STOP'): - try : - logger.debug('%s: Received job to do %d %ss, starting with %s', - proc,len(task),item,task[0][1]) - for kwargs, k in task: - t1 = time.time() - kwargs['config'] = config - kwargs['logger'] = logger - result = job_func(**kwargs) - t2 = time.time() - results_queue.put( (result, k, t2-t1, proc) ) - except Exception as e: - tr = traceback.format_exc() - logger.debug('%s: Caught exception: %s\n%s',proc,str(e),tr) - results_queue.put( (e, k, tr, proc) ) - logger.debug('%s: Received STOP', proc) - if pr is not None: - pr.disable() - pr.dump_stats(config.get('root', 'galsim') + '-' + str(proc) + '.pstats') - s = StringIO() - sortby = 'time' # Note: This is now called tottime, but time seems to be a valid - # alias for this that is backwards compatible to older versions - # of pstats. - ps = pstats.Stats(pr, stream=s).sort_stats(sortby).reverse_order() - ps.print_stats() - logger.error("*** Start profile for %s ***\n%s\n*** End profile for %s ***", - proc,s.getvalue(),proc) + # The per-process work is done by the module-level _mp_worker (defined above), which is + # picklable and therefore usable under the 'spawn' start method on Windows. njobs = sum([len(task) for task in tasks]) if nproc > 1: logger.warning("Using %d processes for %s processing",nproc,item) - ctx = get_context('fork') + ctx = _get_mp_context() Process = ctx.Process Queue = ctx.Queue @@ -819,6 +849,24 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): # for a new task. If there is one there, it grabs it and does it. If not, it waits # until there is one to grab. When it finds a 'STOP', it shuts down. results_queue = Queue(ntasks) + + # Under the 'spawn' start method (used where 'fork' is unavailable, e.g. Windows), + # the Process args are pickled, but config may hold unpicklable cached values + # (e.g. compiled Eval lambdas in '_fn' items or generator functions in '_gen_fn' + # items). So pass the workers a CopyConfig copy, which strips those caches but + # keeps the already-built '_input_objs'; the workers rebuild the caches lazily as + # needed. Also drop '_eval_gdict', which holds module objects that can't be + # pickled (it too is rebuilt lazily), and 'output_manager', a started BaseManager + # instance (holds weakrefs, unpicklable); the workers only need the dict/list + # proxies stored in the extra builders, which pickle fine. Under 'fork', pass + # config as is to preserve the usual shared-memory semantics. + if ctx.get_start_method() == 'fork': + worker_config = config + else: + worker_config = CopyConfig(config) + worker_config.pop('_eval_gdict', None) + worker_config.pop('output_manager', None) + p_list = [] for j in range(nproc): # The process name is actually the default name that Process would generate on its @@ -828,8 +876,9 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): # processes, so for the sake of the logging output, we name the processes explicitly. initializers = worker_init_fns initargs = [initargs_fn() for initargs_fn in worker_initargs_fns] - p = Process(target=worker, args=(task_queue, results_queue, config, logger_proxy, - initializers, initargs), + p = Process(target=_mp_worker, + args=(task_queue, results_queue, worker_config, logger_proxy, + initializers, initargs, item, job_func), name='Process-%d'%(j+1)) p.start() p_list.append(p) diff --git a/galsim/config/value.py b/galsim/config/value.py index 13b58dd9ec..ea66112cf6 100644 --- a/galsim/config/value.py +++ b/galsim/config/value.py @@ -16,7 +16,7 @@ # and/or other materials provided with the distribution. # -from astropy.units import Quantity, Unit +from .._astropy_shim import Quantity, Unit from .util import PropagateIndexKeyRNGNum, GetIndex, ParseExtendedKey from ..errors import GalSimConfigError, GalSimConfigValueError diff --git a/galsim/config/value_eval.py b/galsim/config/value_eval.py index a324849abf..c29a4fb9f4 100644 --- a/galsim/config/value_eval.py +++ b/galsim/config/value_eval.py @@ -17,7 +17,7 @@ # import numpy as np import re -from astropy.units import Quantity, Unit +from .._astropy_shim import Quantity, Unit from .util import PropagateIndexKeyRNGNum from .value import GetCurrentValue, GetAllParams, RegisterValueType diff --git a/galsim/download_cosmos.py b/galsim/download_cosmos.py index cc9cff7876..2c2810157e 100644 --- a/galsim/download_cosmos.py +++ b/galsim/download_cosmos.py @@ -427,7 +427,17 @@ def make_link(do_link, unpack_dir, link_dir, args, logger): if yn == 'no': return os.remove(link_dir) - os.symlink(os.path.abspath(unpack_dir), link_dir) + try: + os.symlink(os.path.abspath(unpack_dir), link_dir) + except OSError: + # Windows without symlink privilege (WinError 1314): use a directory + # junction (no privilege needed), falling back to a full copy. + target = os.path.abspath(unpack_dir) + try: + import _winapi + _winapi.CreateJunction(target, link_dir) # dir junction, Windows only + except (ImportError, OSError, AttributeError): + shutil.copytree(target, link_dir) logger.info("Made link to %s from %s", unpack_dir, link_dir) diff --git a/galsim/fitswcs.py b/galsim/fitswcs.py index ed3b88b99a..aea22e3b78 100644 --- a/galsim/fitswcs.py +++ b/galsim/fitswcs.py @@ -22,7 +22,7 @@ import warnings import os import numpy as np -import astropy.wcs +from ._astropy_shim import wcs as _astropy_wcs import subprocess import copy @@ -199,7 +199,7 @@ def _load_from_header(self, header): # warnings, since we don't much care if the input file is non-standard # so long as we can make it work. warnings.simplefilter("ignore") - wcs = astropy.wcs.WCS(header.header) + wcs = _astropy_wcs.WCS(header.header) return wcs @property diff --git a/galsim/kolmogorov.py b/galsim/kolmogorov.py index e03993ee38..c9b39a2c11 100644 --- a/galsim/kolmogorov.py +++ b/galsim/kolmogorov.py @@ -19,7 +19,7 @@ __all__ = [ 'Kolmogorov' ] import numpy as np -import astropy.units as u +from ._astropy_shim import units as u import math from . import _galsim diff --git a/galsim/phase_psf.py b/galsim/phase_psf.py index 0178040579..a902a39fe7 100644 --- a/galsim/phase_psf.py +++ b/galsim/phase_psf.py @@ -20,7 +20,7 @@ from heapq import heappush, heappop import numpy as np -import astropy.units as u +from ._astropy_shim import units as u import copy from . import fits diff --git a/galsim/photon_array.py b/galsim/photon_array.py index 502dc2c87e..cd19c920e8 100644 --- a/galsim/photon_array.py +++ b/galsim/photon_array.py @@ -22,7 +22,7 @@ 'ScaleFlux', 'ScaleWavelength' ] import numpy as np -import astropy.units as u +from ._astropy_shim import units as u from . import _galsim from .random import BaseDeviate diff --git a/galsim/second_kick.py b/galsim/second_kick.py index be0ce85aeb..ca0b4c5899 100644 --- a/galsim/second_kick.py +++ b/galsim/second_kick.py @@ -18,7 +18,7 @@ __all__ = [ 'SecondKick' ] -import astropy.units as u +from ._astropy_shim import units as u from . import _galsim from .gsobject import GSObject diff --git a/galsim/sed.py b/galsim/sed.py index 704957d446..731c22d1a8 100644 --- a/galsim/sed.py +++ b/galsim/sed.py @@ -19,10 +19,10 @@ __all__ = [ 'SED', 'EmissionLine' ] import numpy as np -from astropy import units -from astropy import constants +from ._astropy_shim import units +from ._astropy_shim import constants from numbers import Real -from pathlib import PosixPath +from pathlib import PurePath from .table import LookupTable, _LookupTable from ._utilities import WeakMethod, lazy_property, basestring @@ -322,7 +322,7 @@ def _initialize_spec(self): raise GalSimSEDError("Attempt to set spectral SED using float or integer.", self) self._const = True self._spec = lambda w: float(self._orig_spec) - elif isinstance(self._orig_spec, (basestring, PosixPath)): + elif isinstance(self._orig_spec, (basestring, PurePath)): isfile, filename = utilities.check_share_file(self._orig_spec, 'SEDs') if isfile: self._spec = LookupTable.from_file(filename, interpolant=self.interpolant) diff --git a/galsim/vonkarman.py b/galsim/vonkarman.py index c0f42743d9..13b960ec65 100644 --- a/galsim/vonkarman.py +++ b/galsim/vonkarman.py @@ -19,7 +19,7 @@ __all__ = [ 'VonKarman' ] import numpy as np -import astropy.units as u +from ._astropy_shim import units as u from . import _galsim from .gsobject import GSObject diff --git a/include/galsim/Random.h b/include/galsim/Random.h index f40cc8205d..27bccdf7de 100644 --- a/include/galsim/Random.h +++ b/include/galsim/Random.h @@ -32,6 +32,7 @@ */ #include +#include #include "Image.h" @@ -91,7 +92,7 @@ namespace galsim { * * @param[in] lseed A long-integer seed for the RNG. */ - explicit BaseDeviate(long lseed); + explicit BaseDeviate(int64_t lseed); /** * @brief Construct a new BaseDeviate, sharing the random number generator with rhs. @@ -157,7 +158,7 @@ namespace galsim { * * Note that this will reseed all Deviates currently sharing the RNG with this one. */ - virtual void seed(long lseed); + virtual void seed(int64_t lseed); /** * @brief Like seed(lseed), but severs the relationship between other Deviates. @@ -165,7 +166,7 @@ namespace galsim { * Other Deviates that had been using the same RNG will be unaffected, while this * Deviate will obtain a fresh RNG seed according to lseed. */ - void reset(long lseed); + void reset(int64_t lseed); /** * @brief Make this object share its random number generator with another Deviate. @@ -196,7 +197,7 @@ namespace galsim { /** * @brief Get a random value in its raw form as a long integer. */ - long raw(); + int64_t raw(); /** * @brief Draw a new random number from the distribution @@ -281,7 +282,7 @@ namespace galsim { * * @param[in] lseed A long-integer seed for the RNG. */ - UniformDeviate(long lseed); + UniformDeviate(int64_t lseed); /// @brief Construct a new UniformDeviate, sharing the random number generator with rhs. UniformDeviate(const BaseDeviate& rhs); @@ -345,7 +346,7 @@ namespace galsim { * @param[in] mean Mean of the output distribution * @param[in] sigma Standard deviation of the distribution */ - GaussianDeviate(long lseed, double mean, double sigma); + GaussianDeviate(int64_t lseed, double mean, double sigma); /** * @brief Construct a new Gaussian-distributed RNG, sharing the random number @@ -463,7 +464,7 @@ namespace galsim { * @param[in] N Number of "coin flips" per trial * @param[in] p Probability of success per coin flip. */ - BinomialDeviate(long lseed, int N, double p); + BinomialDeviate(int64_t lseed, int N, double p); /** * @brief Construct a new binomial-distributed RNG, sharing the random number @@ -565,7 +566,7 @@ namespace galsim { * @param[in] lseed Seed to use * @param[in] mean Mean of the output distribution */ - PoissonDeviate(long lseed, double mean); + PoissonDeviate(int64_t lseed, double mean); /** * @brief Construct a new Poisson-distributed RNG, sharing the random number @@ -671,7 +672,7 @@ namespace galsim { * @param[in] a Shape parameter of the output distribution, must be > 0. * @param[in] b Scale parameter of the distribution, must be > 0. */ - WeibullDeviate(long lseed, double a, double b); + WeibullDeviate(int64_t lseed, double a, double b); /** * @brief Construct a new Weibull-distributed RNG, sharing the random number @@ -780,7 +781,7 @@ namespace galsim { * @param[in] k Shape parameter of the output distribution, must be > 0. * @param[in] theta Scale parameter of the distribution, must be > 0. */ - GammaDeviate(long lseed, double k, double theta); + GammaDeviate(int64_t lseed, double k, double theta); /** * @brief Construct a new Gamma-distributed RNG, sharing the random number @@ -892,7 +893,7 @@ namespace galsim { * @param[in] lseed Seed to use * @param[in] n Number of degrees of freedom for the output distribution, must be > 0. */ - Chi2Deviate(long lseed, double n); + Chi2Deviate(int64_t lseed, double n); /** * @brief Construct a new Chi^2-distributed RNG, sharing the random number diff --git a/include/galsim/Std.h b/include/galsim/Std.h index a149800e92..6d38a74482 100644 --- a/include/galsim/Std.h +++ b/include/galsim/Std.h @@ -47,6 +47,12 @@ #include #ifdef _WIN32 +// Suppress Windows.h's ``min``/``max`` macros which clash with std::min/std::max +// (and Eigen's templated members). Set this before the include even though the +// MSVC build also passes ``/DNOMINMAX`` -- belt and suspenders for direct includes. +#ifndef NOMINMAX +#define NOMINMAX +#endif #include #else #include diff --git a/include/galsim/Stopwatch.h b/include/galsim/Stopwatch.h index efd062d20b..19b5c9c189 100644 --- a/include/galsim/Stopwatch.h +++ b/include/galsim/Stopwatch.h @@ -20,31 +20,31 @@ #ifndef GalSim_Stopwatch_H #define GalSim_Stopwatch_H -#include +#include namespace galsim { class Stopwatch { private: + typedef std::chrono::steady_clock clock_type; double seconds; - struct timeval tpStart; + clock_type::time_point tpStart; bool running; public: Stopwatch() : seconds(0.), running(false) {} - void start() { gettimeofday(&tpStart, NULL); running=true; } + void start() { tpStart = clock_type::now(); running = true; } void stop() { if (!running) return; - struct timeval tp; - gettimeofday(&tp, NULL); - seconds += (tp.tv_sec - tpStart.tv_sec) - + 1e-6*(tp.tv_usec - tpStart.tv_usec); + auto tp = clock_type::now(); + std::chrono::duration dt = tp - tpStart; + seconds += dt.count(); running = false; } - void reset() { seconds=0.; running=false; } + void reset() { seconds = 0.; running = false; } operator double() const { return seconds; } }; diff --git a/pysrc/Random.cpp b/pysrc/Random.cpp index 5d5f942175..bcf7d49878 100644 --- a/pysrc/Random.cpp +++ b/pysrc/Random.cpp @@ -89,11 +89,11 @@ namespace galsim { void pyExportRandom(py::module& _galsim) { py::class_ (_galsim, "BaseDeviateImpl") - .def(py::init()) + .def(py::init()) .def(py::init()) .def(py::init()) .def("duplicate", &BaseDeviate::duplicate) - .def("seed", (void (BaseDeviate::*) (long) )&BaseDeviate::seed) + .def("seed", (void (BaseDeviate::*) (int64_t) )&BaseDeviate::seed) .def("reset", (void (BaseDeviate::*) (const BaseDeviate&) )&BaseDeviate::reset) .def("clearCache", &BaseDeviate::clearCache) .def("serialize", &BaseDeviate::serialize) diff --git a/setup.py b/setup.py index 05d7a16a98..f9493d1887 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,10 @@ from setuptools import setup, Extension, find_packages from setuptools.command.build_ext import build_ext from setuptools.command.build_clib import build_clib + from setuptools.command.build_py import build_py from setuptools.command.install import install from setuptools.command.install_scripts import install_scripts from setuptools.command.easy_install import easy_install - from setuptools.command.test import test import setuptools print("Using setuptools version",setuptools.__version__) except ImportError: @@ -45,6 +45,14 @@ print() raise +# setuptools.command.test was removed in setuptools 72.0.0; tolerate its absence. +try: + from setuptools.command.test import test # noqa: F401 +except ImportError: + pass + +IS_WINDOWS = sys.platform == 'win32' + # Turn this on for more verbose debugging output about compile attempts. debug = False @@ -85,6 +93,8 @@ def all_files_from(dir, ext=''): '-Wno-openmp-mapping','-Wno-unknown-cuda-version', '-Wno-shorten-64-to-32','-fvisibility=hidden', '-DGALSIM_USE_GPU'], 'nvc++' : ['-O2','-std=c++14','-mp=gpu','-DGALSIM_USE_GPU'], + 'msvc' : ['/O2', '/std:c++14', '/EHsc', '/openmp', + '/Zc:__cplusplus', '/utf-8', '/DNOMINMAX'], 'unknown' : [], } lopt = { @@ -98,6 +108,7 @@ def all_files_from(dir, ext=''): 'clang w/ GPU' : ['-fopenmp','-fopenmp-targets=nvptx64-nvidia-cuda', '-Wno-openmp-mapping','-Wno-unknown-cuda-version'], 'nvc++' : ['-mp=gpu'], + 'msvc' : [], 'unknown' : [], } @@ -116,7 +127,8 @@ def all_files_from(dir, ext=''): else: # Including mmgr.cpp in the library leads to problems if the other files don't # include mmgr.h. So remove it. - cpp_sources.remove('src/mmgr.cpp') + mmgr_path = os.path.join('src', 'mmgr.cpp') + cpp_sources = [s for s in cpp_sources if os.path.normpath(s) != mmgr_path] # Verbose is the default for setuptools logging, but if it's on the command line, we take it # to mean that we should also be verbose. @@ -131,6 +143,12 @@ def get_compiler_type(compiler, check_unknown=True, output=False): be called cc or gcc. """ if debug: output=True + # MSVC's CCompiler subclass does not expose ``compiler_so`` (a Unix-only + # attribute). Detect it directly via ``compiler_type``. + if getattr(compiler, 'compiler_type', None) == 'msvc': + if output: + print('Compiler is MSVC.') + return 'msvc' cc = compiler.compiler_so[0] if cc == 'ccache': cc = compiler.compiler_so[1] @@ -254,10 +272,13 @@ def find_fftw_lib(output=False): if debug: output = True try_libdirs = [] - # Start with the explicit FFTW_DIR, if present. + # Start with the explicit FFTW_DIR, if present. Support both Unix + # ``/lib`` and conda-style ``/Library/lib`` layouts. if 'FFTW_DIR' in os.environ: - try_libdirs.append(os.environ['FFTW_DIR']) - try_libdirs.append(os.path.join(os.environ['FFTW_DIR'],'lib')) + fftw_root = os.environ['FFTW_DIR'] + try_libdirs.append(fftw_root) + try_libdirs.append(os.path.join(fftw_root, 'lib')) + try_libdirs.append(os.path.join(fftw_root, 'Library', 'lib')) # Add the python system library directory. try_libdirs.append(distutils.sysconfig.get_config_var('LIBDIR')) @@ -265,18 +286,27 @@ def find_fftw_lib(output=False): # If using Anaconda, add their lib dir in case fftw is installed there. # (With envs, this might be different than the sysconfig LIBDIR.) if 'CONDA_PREFIX' in os.environ: - try_libdirs.append(os.path.join(os.environ['CONDA_PREFIX'],'lib')) - - # Try some standard locations where things get installed - try_libdirs.extend(['/usr/local/lib', '/usr/lib']) - if sys.platform == "darwin": - try_libdirs.extend(['/sw/lib', '/opt/local/lib']) + conda_root = os.environ['CONDA_PREFIX'] + try_libdirs.append(os.path.join(conda_root, 'lib')) + # On Windows conda installs C libs under ``\Library\lib``. + try_libdirs.append(os.path.join(conda_root, 'Library', 'lib')) + + if IS_WINDOWS: + # vcpkg layout + if 'VCPKG_ROOT' in os.environ: + try_libdirs.append(os.path.join( + os.environ['VCPKG_ROOT'], 'installed', 'x64-windows', 'lib')) + else: + # Try some standard locations where things get installed + try_libdirs.extend(['/usr/local/lib', '/usr/lib']) + if sys.platform == "darwin": + try_libdirs.extend(['/sw/lib', '/opt/local/lib']) # Check the directories in LD_LIBRARY_PATH. This doesn't work on OSX >= 10.11 for path in ['LIBRARY_PATH', 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']: if path in os.environ: - for dir in os.environ[path].split(':'): - try_libdirs.append(dir) + for d in os.environ[path].split(os.pathsep): + try_libdirs.append(d) # The user's home directory is often a good place to check. try_libdirs.append(os.path.join(os.path.expanduser("~"),"lib")) @@ -288,21 +318,34 @@ def find_fftw_lib(output=False): except ImportError: pass - if sys.platform == "darwin": - lib_ext = '.dylib' + if IS_WINDOWS: + # MSVC import library produced by conda-forge / vcpkg / fftw.org. + lib_names = ['fftw3.lib', 'libfftw3.lib', 'libfftw3-3.lib'] + elif sys.platform == "darwin": + lib_names = ['libfftw3.dylib'] else: - lib_ext = '.so' - name = 'libfftw3' + lib_ext - if output: print("Looking for ",name) + lib_names = ['libfftw3.so'] + if output: print("Looking for ", ' or '.join(lib_names)) tried_dirs = set() # Keep track, so we don't try the same thing twice. for dir in try_libdirs: - if dir == '': continue # This messes things up if it's in there. + if not dir: continue # Filters out '' and ``None`` (sysconfig may yield None on win32). if dir in tried_dirs: continue else: tried_dirs.add(dir) if not os.path.isdir(dir): continue - libpath = os.path.join(dir, name) - if not os.path.isfile(libpath): continue + for name in lib_names: + libpath = os.path.join(dir, name) + if os.path.isfile(libpath): + break + else: + continue if output: print(" ", dir, end='') + if IS_WINDOWS: + # On Windows the file we located is the import library, not the + # DLL. ``ctypes.cdll.LoadLibrary`` only loads runtime DLLs and + # would fail here, so just trust the path and let the linker + # use it. + if output: print(" (yes)") + return libpath try: lib = ctypes.cdll.LoadLibrary(libpath) if output: print(" (yes)") @@ -323,13 +366,19 @@ def find_fftw_lib(output=False): # If we didn't find it anywhere, but the user has set FFTW_DIR, trust it. if 'FFTW_DIR' in os.environ: - libpath = os.path.join(os.environ['FFTW_DIR'], name) + libpath = os.path.join(os.environ['FFTW_DIR'], lib_names[0]) print("WARNING:") - print("Could not find an installed fftw3 library named %s"%(name)) + print("Could not find an installed fftw3 library named %s"%(lib_names[0])) print("Trusting the provided FFTW_DIR=%s for the library location."%(libpath)) print("If this is incorrect, you may have errors later when linking.") return libpath + if IS_WINDOWS: + print("Could not find fftw3 library. On Windows, install fftw via conda-forge") + print("(``conda install -c conda-forge fftw``) or vcpkg, or set FFTW_DIR to a") + print("directory containing fftw3.lib (and fftw3.dll on PATH at runtime).") + raise OSError("fftw3 import library not found") + # Last ditch attempt. Use ctypes.util.find_library, which sometimes manages to find it # when the above attempts fail. try: @@ -371,21 +420,29 @@ def find_eigen_dir(output=False): # Add the python system include directory. try_dirs.append(distutils.sysconfig.get_config_var('INCLUDEDIR')) - # If using Anaconda, add their lib dir in case fftw is installed there. - # (With envs, this might be different than the sysconfig LIBDIR.) + # If using Anaconda, add their include dir in case eigen is installed there. + # (With envs, this might be different than the sysconfig INCLUDEDIR.) if 'CONDA_PREFIX' in os.environ: - try_dirs.append(os.path.join(os.environ['CONDA_PREFIX'],'lib')) - - # Some standard install locations: - try_dirs.extend(['/usr/local/include', '/usr/include']) - if sys.platform == "darwin": - try_dirs.extend(['/sw/include', '/opt/local/include']) + conda_root = os.environ['CONDA_PREFIX'] + try_dirs.append(os.path.join(conda_root, 'include')) + # Windows conda packages live under ``\Library\include``. + try_dirs.append(os.path.join(conda_root, 'Library', 'include')) + + if IS_WINDOWS: + if 'VCPKG_ROOT' in os.environ: + try_dirs.append(os.path.join( + os.environ['VCPKG_ROOT'], 'installed', 'x64-windows', 'include')) + else: + # Some standard install locations: + try_dirs.extend(['/usr/local/include', '/usr/include']) + if sys.platform == "darwin": + try_dirs.extend(['/sw/include', '/opt/local/include']) # Also if there is a C_INCLUDE_PATH, check those dirs. for path in ['C_INCLUDE_PATH']: if path in os.environ: - for dir in os.environ[path].split(':'): - try_dirs.append(dir) + for d in os.environ[path].split(os.pathsep): + try_dirs.append(d) # Finally, (last resort) check our own download of eigen. if os.path.isdir('downloaded_eigen'): @@ -496,6 +553,27 @@ def try_compile(cpp_code, compiler, cflags=[], lflags=[], prepend=None, check_wa with tempfile.NamedTemporaryFile(delete=False, suffix='.exe', dir=local_tmp) as exe_file: exe_name = exe_file.name + # MSVC's distutils CCompiler does not expose Unix-style ``compiler_so`` + # / ``linker_so`` lists; compose probe builds via the high-level API. + if getattr(compiler, 'compiler_type', None) == 'msvc': + try: + objects = compiler.compile([cpp_name], output_dir=local_tmp, + extra_postargs=list(cflags)) + exe_root = os.path.splitext(exe_name)[0] + compiler.link_executable(objects, exe_root, + extra_postargs=list(lflags), + target_lang='c++') + except Exception as e: + if debug: + print('MSVC compile/link probe failed: ', repr(e)) + return False + # Probe succeeded. Best-effort cleanup. + for f in [cpp_name, o_name, exe_name]: + if os.path.exists(f): + try: os.remove(f) + except OSError: pass + return True + # Try compiling with the given flags cc = [compiler.compiler_so[0]] if prepend: @@ -776,6 +854,8 @@ def _single_compile(obj): def fix_compiler(compiler, njobs): + is_msvc = getattr(compiler, 'compiler_type', None) == 'msvc' + # Remove any -Wstrict-prototypes in the compiler flags (since invalid for C++) try: compiler.compiler_so.remove("-Wstrict-prototypes") @@ -790,22 +870,27 @@ def fix_compiler(compiler, njobs): # Figure out what compiler it will use comp_type = get_compiler_type(compiler, output=True) - cc = compiler.compiler_so[0] - already_have_ccache = False - if cc == 'ccache': - already_have_ccache = True - cc = compiler.compiler_so[1] - if cc == comp_type: - print('Using compiler %s'%(cc)) + if is_msvc: + cc = 'msvc' + already_have_ccache = False + print('Using compiler MSVC') else: - print('Using compiler %s, which is %s'%(cc,comp_type)) + cc = compiler.compiler_so[0] + already_have_ccache = False + if cc == 'ccache': + already_have_ccache = True + cc = compiler.compiler_so[1] + if cc == comp_type: + print('Using compiler %s'%(cc)) + else: + print('Using compiler %s, which is %s'%(cc,comp_type)) # Make sure the compiler works with a simple c++ code if not try_cpp(compiler): # One failure mode is that sometimes there is a -B /path/to/compiler_compat # which can cause problems. If we get here, try removing that. success = False - if '-B' in compiler.linker_so: + if not is_msvc and '-B' in compiler.linker_so: for i in range(len(compiler.linker_so)): if (compiler.linker_so[i] == '-B' and 'compiler_compat' in compiler.linker_so[i+1]): @@ -815,66 +900,84 @@ def fix_compiler(compiler, njobs): break if not success: print("There seems to be something wrong with the compiler or cflags") - print(str(compiler.compiler_so)) + if not is_msvc: + print(str(compiler.compiler_so)) raise OSError("Compiler does not work for compiling C++ code") # Check if we can use ccache to speed up repeated compilation. - if not already_have_ccache and try_cpp(compiler, prepend='ccache'): + if (not is_msvc and not already_have_ccache and + try_cpp(compiler, prepend='ccache')): print('Using ccache') compiler.set_executable('compiler_so', ['ccache'] + compiler.compiler_so) - if njobs > 1: + if njobs > 1 and not IS_WINDOWS: # Global variable for tracking the number of jobs to use. # We can't pass this to parallel compile, since the signature is fixed. # So if using parallel compile, set this value to use within parallel compile. global glob_use_njobs glob_use_njobs = njobs compiler.compile = types.MethodType(parallel_compile, compiler) - - extra_cflags = copt[comp_type] - extra_lflags = lopt[comp_type] - - success = try_cpp14(compiler, extra_cflags, extra_lflags) - if not success: - # In case libc++ doesn't work, try letting the system use the default stdlib - try: - extra_cflags.remove('-stdlib=libc++') - extra_lflags.remove('-stdlib=libc++') - except (AttributeError, ValueError): - pass - else: - success = try_cpp14(compiler, extra_cflags, extra_lflags) + elif IS_WINDOWS and njobs > 1: + # MSVC drives parallel compilation via ``/MP`` rather than a Python + # multiprocessing pool; the monkey-patched ``parallel_compile`` above + # is built around the Unix one-source-at-a-time invocation pattern + # and currently misbehaves under MSVC. Stick to single-process + # compile here -- per-extension speedup can be regained later by + # adding ``/MP`` to the MSVC ``copt`` entry. + print('Note: forcing single-process compile on Windows.') + + extra_cflags = list(copt[comp_type]) + extra_lflags = list(lopt[comp_type]) + + if is_msvc: + # The Unix-style probe in try_cpp14 has been adapted via the MSVC + # branch in try_compile; trust MSVC for C++14 support. + success = True + else: + success = try_cpp14(compiler, extra_cflags, extra_lflags) + if not success: + # In case libc++ doesn't work, try letting the system use the default stdlib + try: + extra_cflags.remove('-stdlib=libc++') + extra_lflags.remove('-stdlib=libc++') + except (AttributeError, ValueError): + pass + else: + success = try_cpp14(compiler, extra_cflags, extra_lflags) if not success: print('The compiler %s with flags %s did not successfully compile C++14 code'% (cc, ' '.join(extra_cflags))) raise OSError("Compiler is not C++-14 compatible") - # Also see if adding -msse2 works (and doesn't give a warning) - if '-msse2' not in extra_cflags: - extra_cflags.append('-msse2') - if try_cpp14(compiler, extra_cflags, extra_lflags, check_warning=True): - print('Using cflag -msse2') - else: - print('warning with -msse2.') - extra_cflags.remove('-msse2') + if not is_msvc: + # Also see if adding -msse2 works (and doesn't give a warning). This flag + # is GCC/Clang-only; MSVC enables SSE2 by default on x64 builds. + if '-msse2' not in extra_cflags: + extra_cflags.append('-msse2') + if try_cpp14(compiler, extra_cflags, extra_lflags, check_warning=True): + print('Using cflag -msse2') + else: + print('warning with -msse2.') + extra_cflags.remove('-msse2') # If doing develop installation, it's important for the build directory to be before any # other directories. Particularly ones that might have another version of GalSim installed. # Otherwise the wrong library can be linked, which leads to errors. # So, make sure that the -Lbuild/... directive happens first among any -L directives in - # the link flags. - linker_so = compiler.linker_so - # Find the first -L flag among the current flags (if any) - for i, flag in enumerate(linker_so): - if flag.startswith('-L'): - print('Found link: ',i,flag) - break - else: - i = len(linker_so) - # Insert -Llib for any libs that are in build directory, to make sure they are first. - linker_so[i:i] = ['-L' + l for l in compiler.library_dirs if l.startswith('build')] - # Copy this list back to the compiler object - compiler.set_executable('linker_so', linker_so) + # the link flags. ``linker_so`` is a Unix-only attribute, so guard the rewrite. + if hasattr(compiler, 'linker_so'): + linker_so = list(compiler.linker_so) + # Find the first -L flag among the current flags (if any) + for i, flag in enumerate(linker_so): + if flag.startswith('-L'): + print('Found link: ',i,flag) + break + else: + i = len(linker_so) + # Insert -Llib for any libs that are in build directory, to make sure they are first. + linker_so[i:i] = ['-L' + l for l in compiler.library_dirs if l.startswith('build')] + # Copy this list back to the compiler object + compiler.set_executable('linker_so', linker_so) # Return the extra cflags, since those will be added to the build step in a different place. print('Using extra flags ',extra_cflags) @@ -968,6 +1071,71 @@ def parse_njobs(njobs, task=None, command=None, maxn=4): # but we only want to output on the first pass through add_dirs. # (Unless debug = True, then also output in the second pass.) + +class my_build_py(build_py): + """build_py wrapper that ships ``share/`` and ``include/`` data on + Windows checkouts where the ``galsim/share`` and ``galsim/include`` + git symlinks were materialised as text files rather than real links. + + GalSim packages its data tree (Roman SCA, SEDs, filters, sensor + files) by relying on ``galsim/share`` being a symlink to the + repo-level ``share/`` directory; ``package_data={'galsim': + shared_data + headers}`` then resolves to real files via the + symlink. On a Windows checkout with the default + ``core.symlinks=false``, git writes the symlink target text into + a regular file, so setuptools' package_data scan sees an 8-byte + file rather than a directory and the wheel ships no share data. + + After the standard build_py copies whatever package_data it can + locate, this subclass detects the stub-file situation and copies + the repo-level ``share/`` (and likewise ``include/``) tree directly + into ``/galsim/share/`` (``.../include/``) so the wheel + is complete. The source tree itself is not touched. Linux/macOS + checkouts have working symlinks, so the stub detection is False and + this branch is a no-op. + """ + + _STUB_BODIES = { + 'share': ('../share', '..\\share'), + 'include': ('../include', '../include/', '..\\include', '..\\include\\'), + } + + def run(self): + build_py.run(self) + for name, stub_bodies in self._STUB_BODIES.items(): + self._copy_stub_tree(name, stub_bodies) + + def _copy_stub_tree(self, name, stub_bodies): + repo_dir = name + pkg_path = os.path.join('galsim', name) + if not os.path.isdir(repo_dir): + return + # Detect the stub: a regular file whose body is the symlink target. + if os.path.isdir(pkg_path): + return # symlink resolved correctly; nothing to do + if not os.path.isfile(pkg_path): + return + try: + with open(pkg_path, 'r', encoding='utf-8', errors='replace') as f: + body = f.read().strip() + except OSError: + return + if body not in stub_bodies: + return + dest = os.path.join(self.build_lib, 'galsim', name) + # Replace any stale stub copy that the standard build_py may + # have written into build_lib. + if os.path.isdir(dest): + shutil.rmtree(dest) + elif os.path.isfile(dest): + os.remove(dest) + os.makedirs(os.path.dirname(dest), exist_ok=True) + shutil.copytree(repo_dir, dest) + print("galsim: copied %s -> %s " + "(Windows symlink-stub fallback for galsim/%s)" + % (repo_dir, dest, name)) + + # Make a subclass of build_ext so we can add to the -I list. class my_build_clib(build_clib): user_options = build_ext.user_options + [('njobs=', 'j', "Number of jobs to use for compiling")] @@ -1208,7 +1376,10 @@ def run(self): # If requested, also build the shared library. if int(os.environ.get('GALSIM_BUILD_SHARED', 0)): - self.run_command("build_shared_clib") + if IS_WINDOWS: + print('GALSIM_BUILD_SHARED is not supported on Windows yet; skipping.') + else: + self.run_command("build_shared_clib") if int(os.environ.get('GALSIM_RUN_TEST', 0)): self.run_command("run_cpp_test") @@ -1313,11 +1484,20 @@ def run(self): 'depends' : headers + inst, 'include_dirs' : ['include', 'include/galsim'], 'undef_macros' : undef_macros }) -ext=Extension("galsim._galsim", - py_sources, - depends = cpp_sources + headers + inst, - undef_macros = undef_macros, - extra_link_args = ["-lfftw3"]) +if IS_WINDOWS: + # MSVC link line uses ``libraries`` + ``library_dirs`` (populated by + # ``add_dirs`` via ``find_fftw_lib``). ``-lfftw3`` is GCC-only. + ext=Extension("galsim._galsim", + py_sources, + depends = cpp_sources + headers + inst, + undef_macros = undef_macros, + libraries = ['fftw3']) +else: + ext=Extension("galsim._galsim", + py_sources, + depends = cpp_sources + headers + inst, + undef_macros = undef_macros, + extra_link_args = ["-lfftw3"]) build_dep = ['setuptools>=38', 'pybind11>=2.2', 'numpy>=1.17'] run_dep = ['astropy', 'LSSTDESC.Coord'] @@ -1402,7 +1582,8 @@ def run(self): setup_requires=build_dep, install_requires=build_dep + run_dep, tests_require=test_dep, - cmdclass = {'build_ext': my_build_ext, + cmdclass = {'build_py': my_build_py, + 'build_ext': my_build_ext, 'build_clib': my_build_clib, 'build_shared_clib': my_build_shared_clib, 'install': my_install, @@ -1418,10 +1599,10 @@ def run(self): ) # Check that the path includes the directory where the scripts are installed. -real_env_path = [os.path.realpath(d) for d in os.environ['PATH'].split(':')] +real_env_path = [os.path.realpath(d) for d in os.environ['PATH'].split(os.pathsep)] if hasattr(dist,'script_install_dir'): print('scripts installed into ',dist.script_install_dir) - if (dist.script_install_dir not in os.environ['PATH'].split(':') and + if (dist.script_install_dir not in os.environ['PATH'].split(os.pathsep) and os.path.realpath(dist.script_install_dir) not in real_env_path): print('\nWARNING: The GalSim executables were installed in a directory not in your PATH') diff --git a/src/Image.cpp b/src/Image.cpp index 9d5464f108..f92fbd9900 100644 --- a/src/Image.cpp +++ b/src/Image.cpp @@ -733,7 +733,7 @@ void rfft(const BaseImage& in, ImageView > out, dbg<<"Start rfft\n"; dbg<<"self bounds = "<& in, ImageView out, bool shift_in, bool sh dbg<<"Start irfft\n"; dbg<<"self bounds = "<& in, ImageView > out, dbg<<"Start cfft\n"; dbg<<"self bounds = "< #include +#include +#else +#include +#include +#endif #include #include #include -#include #include // For memcpy #ifdef _OPENMP @@ -82,7 +87,7 @@ namespace galsim { _impl(new BaseDeviateImpl()) {} - BaseDeviate::BaseDeviate(long lseed) : + BaseDeviate::BaseDeviate(int64_t lseed) : _impl(new BaseDeviateImpl()) { seed(lseed); } @@ -129,6 +134,14 @@ namespace galsim { void BaseDeviate::seedurandom() { +#ifdef _WIN32 + // Windows has no /dev/urandom; use std::random_device which delegates + // to the platform CSPRNG (CryptGenRandom / BCryptGenRandom on Windows + // CRTs). Same observable contract as the POSIX path: produce a + // single int worth of entropy and feed the Mersenne twister. + std::random_device rd; + _impl->_rng->seed(rd()); +#else // This implementation shamelessly taken from: // http://stackoverflow.com/questions/2572366/how-to-use-dev-random-or-urandom-in-c int randomData = open("/dev/urandom", O_RDONLY); @@ -144,16 +157,25 @@ namespace galsim { } close(randomData); _impl->_rng->seed(myRandomInteger); +#endif } void BaseDeviate::seedtime() { +#ifdef _WIN32 + // Match the POSIX path's observable behaviour: seed with the + // microsecond portion of the wall clock. + auto now = std::chrono::system_clock::now().time_since_epoch(); + auto us = std::chrono::duration_cast(now).count(); + _impl->_rng->seed(static_cast(us % 1000000)); +#else struct timeval tp; gettimeofday(&tp,NULL); _impl->_rng->seed(tp.tv_usec); +#endif } - void BaseDeviate::seed(long lseed) + void BaseDeviate::seed(int64_t lseed) { if (lseed == 0) { try { @@ -189,7 +211,7 @@ namespace galsim { clearCache(); } - void BaseDeviate::reset(long lseed) + void BaseDeviate::reset(int64_t lseed) { _impl.reset(new BaseDeviateImpl()); seed(lseed); } void BaseDeviate::reset(const BaseDeviate& dev) @@ -198,7 +220,7 @@ namespace galsim { void BaseDeviate::discard(int n) { _impl->_rng->discard(n); } - long BaseDeviate::raw() + int64_t BaseDeviate::raw() { return (*_impl->_rng)(); } void BaseDeviate::generate(long long N, double* data) @@ -324,7 +346,7 @@ namespace galsim { boost::random::uniform_real_distribution<> _urd; }; - UniformDeviate::UniformDeviate(long lseed) : + UniformDeviate::UniformDeviate(int64_t lseed) : BaseDeviate(lseed), _devimpl(new UniformDeviateImpl()) {} UniformDeviate::UniformDeviate(const BaseDeviate& rhs) : @@ -356,7 +378,7 @@ namespace galsim { boost::random::normal_distribution<> _normal; }; - GaussianDeviate::GaussianDeviate(long lseed, double mean, double sigma) : + GaussianDeviate::GaussianDeviate(int64_t lseed, double mean, double sigma) : BaseDeviate(lseed), _devimpl(new GaussianDeviateImpl(mean, sigma)) {} GaussianDeviate::GaussianDeviate(const BaseDeviate& rhs, double mean, double sigma) : @@ -455,7 +477,7 @@ namespace galsim { boost::random::binomial_distribution<> _bd; }; - BinomialDeviate::BinomialDeviate(long lseed, int N, double p) : + BinomialDeviate::BinomialDeviate(int64_t lseed, int N, double p) : BaseDeviate(lseed), _devimpl(new BinomialDeviateImpl(N,p)) {} BinomialDeviate::BinomialDeviate(const BaseDeviate& rhs, int N, double p) : @@ -562,7 +584,7 @@ namespace galsim { shared_ptr > _gd; }; - PoissonDeviate::PoissonDeviate(long lseed, double mean) : + PoissonDeviate::PoissonDeviate(int64_t lseed, double mean) : BaseDeviate(lseed), _devimpl(new PoissonDeviateImpl(mean)) {} PoissonDeviate::PoissonDeviate(const BaseDeviate& rhs, double mean) : @@ -611,7 +633,7 @@ namespace galsim { boost::random::weibull_distribution<> _weibull; }; - WeibullDeviate::WeibullDeviate(long lseed, double a, double b) : + WeibullDeviate::WeibullDeviate(int64_t lseed, double a, double b) : BaseDeviate(lseed), _devimpl(new WeibullDeviateImpl(a,b)) {} WeibullDeviate::WeibullDeviate(const BaseDeviate& rhs, double a, double b) : @@ -658,7 +680,7 @@ namespace galsim { boost::random::gamma_distribution<> _gamma; }; - GammaDeviate::GammaDeviate(long lseed, double k, double theta) : + GammaDeviate::GammaDeviate(int64_t lseed, double k, double theta) : BaseDeviate(lseed), _devimpl(new GammaDeviateImpl(k,theta)) {} GammaDeviate::GammaDeviate(const BaseDeviate& rhs, double k, double theta) : @@ -705,7 +727,7 @@ namespace galsim { boost::random::chi_squared_distribution<> _chi_squared; }; - Chi2Deviate::Chi2Deviate(long lseed, double n) : + Chi2Deviate::Chi2Deviate(int64_t lseed, double n) : BaseDeviate(lseed), _devimpl(new Chi2DeviateImpl(n)) {} Chi2Deviate::Chi2Deviate(const BaseDeviate& rhs, double n) : diff --git a/src/SBInterpolatedImage.cpp b/src/SBInterpolatedImage.cpp index c92672fe06..fd64efa69a 100644 --- a/src/SBInterpolatedImage.cpp +++ b/src/SBInterpolatedImage.cpp @@ -200,7 +200,7 @@ namespace galsim { if (q2 > _nonzero_bounds.getYMax()) q2 = _nonzero_bounds.getYMax(); // We'll need these for each row. Save them. - double xwt[p2-p1+1]; + std::vector xwt(p2-p1+1); for (int p=p1, pp=0; p<=p2; ++p, ++pp) xwt[pp] = _xInterp.xval(p-x); double sum = 0.; @@ -427,7 +427,7 @@ namespace galsim { dbg<<"q range = "< xwt(p2-p1+1); for (int p=p1, pp=0; p<=p2; ++p, ++pp) xwt[pp] = _kInterp.xval(p-kx); std::complex sum = 0.; @@ -437,7 +437,7 @@ namespace galsim { dbg<<"kimage bounds = "<<_kimage->getBounds()< xsum = KValueInnerLoop(p2-p1+1,pwrap1,qwrap,No2,N,xwt,*_kimage); + std::complex xsum = KValueInnerLoop(p2-p1+1,pwrap1,qwrap,No2,N,xwt.data(),*_kimage); sum += xsum * _kInterp.xval(q-ky); } @@ -549,9 +549,9 @@ namespace galsim { // a given q is independent of y, so we save that as well. double x = x0; - double xwt[_xInterp.ixrange() * mm]; - double p1ar[mm]; - double p2ar[mm]; + std::vector xwt(_xInterp.ixrange() * mm); + std::vector p1ar(mm); + std::vector p2ar(mm); int k=0; for (int i=i1; i temp(mm); for (int j=j1; j _nonzero_bounds.getYMax()) q2 = _nonzero_bounds.getYMax(); - double xwt[p2-p1+1]; + std::vector xwt(p2-p1+1); for (int p=p1, pp=0; p<=p2; ++p, ++pp) { xwt[pp] = _xInterp.xval(p-x); } @@ -825,9 +825,9 @@ namespace galsim { // is that we need to wrap around the p,q values and handle the conjugation possibility // correctly. (cf. comments in kValue method.) kx = kx0; - double xwt[_kInterp.ixrange() * mm]; - double p1ar[mm]; - double p2ar[mm]; + std::vector xwt(_kInterp.ixrange() * mm); + std::vector p1ar(mm); + std::vector p2ar(mm); int k=0; for (int i=i1; i array on stack, so reinterpret_cast below. + std::vector temp(2*mm); // Backing storage for the complex view below; reinterpret_cast below. for (int j=j1; j xwt(p2-p1+1); for (int p=p1, pp=0; p<=p2; ++p, ++pp) xwt[pp] = _kInterp.xval(p-kx); std::complex sum = 0.; @@ -1448,7 +1448,7 @@ namespace galsim { dbg<<"kimage bounds = "<<_kimage.getBounds()< xsum = KValueInnerLoop(p2-p1+1,pwrap1,qwrap,No2,N,xwt,_kimage); + std::complex xsum = KValueInnerLoop(p2-p1+1,pwrap1,qwrap,No2,N,xwt.data(),_kimage); sum += xsum * _kInterp.xval(q-ky); } diff --git a/src/SBTransform.cpp b/src/SBTransform.cpp index ed84410151..964baf7c10 100644 --- a/src/SBTransform.cpp +++ b/src/SBTransform.cpp @@ -779,12 +779,12 @@ namespace galsim { ky0 *= ceny; dky *= ceny; - // Use the stack rather than the heap for these, since a bit faster and small - // enough that they should fit without any problem. - T xphase_kx[2*m]; - T xphase_ky[2*n]; - std::complex* phase_kx = reinterpret_cast*>(xphase_kx); - std::complex* phase_ky = reinterpret_cast*>(xphase_ky); + // VLAs are not portable (MSVC rejects them); use std::vector and + // .data() for the reinterpret view onto std::complex. + std::vector xphase_kx(2*m); + std::vector xphase_ky(2*n); + std::complex* phase_kx = reinterpret_cast*>(xphase_kx.data()); + std::complex* phase_ky = reinterpret_cast*>(xphase_ky.data()); fillphase_1d(phase_kx, m, kx0, dkx); fillphase_1d(phase_ky, n, ky0, dky); diff --git a/src/WCS.cpp b/src/WCS.cpp index e69a4261e0..ee3c5e693d 100644 --- a/src/WCS.cpp +++ b/src/WCS.cpp @@ -138,7 +138,7 @@ namespace galsim { int nblock = std::min(n, 256); xdbg<<"nblock = "< temp(nblock); if (abp) { dbg<<"Using abp\n"; const double* Ap = abp; @@ -150,8 +150,8 @@ namespace galsim { xdbg<<"v = "< "< "< A_dudx((nab-1)*(nab-1)); + std::vector A_dudy((nab-1)*(nab-1)); + std::vector B_dvdx((nab-1)*(nab-1)); + std::vector B_dvdy((nab-1)*(nab-1)); for (int i=1; i du(nblock); + std::vector dv(nblock); + std::vector dudx(nblock); + std::vector dudy(nblock); + std::vector dvdx(nblock); + std::vector dvdy(nblock); const int MAX_ITER = 10; bool not_converged = false; @@ -210,16 +210,16 @@ namespace galsim { dbg<<"n = "< dudx - Horner2D(x, y, n1, A_dudy, nab-1, nab-1, dudy, temp); // -> dudy - Horner2D(x, y, n1, B_dvdx, nab-1, nab-1, dvdx, temp); // -> dvdx - Horner2D(x, y, n1, B_dvdy, nab-1, nab-1, dvdy, temp); // -> dvdy + Horner2D(x, y, n1, A_dudx.data(), nab-1, nab-1, dudx.data(), temp.data()); // -> dudx + Horner2D(x, y, n1, A_dudy.data(), nab-1, nab-1, dudy.data(), temp.data()); // -> dudy + Horner2D(x, y, n1, B_dvdx.data(), nab-1, nab-1, dvdx.data(), temp.data()); // -> dvdx + Horner2D(x, y, n1, B_dvdy.data(), nab-1, nab-1, dvdy.data(), temp.data()); // -> dvdy xdbg<<"dudx = "< 1 mode, the error message is slightly different. - config['image']['nproc'] = 2 - try: - with CaptureLog() as cl: - galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger) - except (ValueError,IndexError,galsim.GalSimError): - pass - #print(cl.output) - if galsim.config.UpdateNProc(2, nimages, config) > 1: - assert re.search("Process-.: Exception caught when building stamp",cl.output) - - try: - with CaptureLog() as cl: - galsim.config.BuildImages(nimages, config, logger=cl.logger) - except (ValueError,IndexError,galsim.GalSimError): - pass - #print(cl.output) - if galsim.config.UpdateNProc(2, nimages, config) > 1: - assert re.search("Process-.: Exception caught when building image",cl.output) + # Under the 'spawn' start method (e.g. on Windows), the HighN type registered above is + # defined inside this test function, so it cannot be pickled to the spawned worker + # processes. So only check this where 'fork' is available. + from multiprocessing import get_all_start_methods + if 'fork' in get_all_start_methods(): + config['image']['nproc'] = 2 + try: + with CaptureLog() as cl: + galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger) + except (ValueError,IndexError,galsim.GalSimError): + pass + #print(cl.output) + if galsim.config.UpdateNProc(2, nimages, config) > 1: + assert re.search("Process-.: Exception caught when building stamp",cl.output) + + try: + with CaptureLog() as cl: + galsim.config.BuildImages(nimages, config, logger=cl.logger) + except (ValueError,IndexError,galsim.GalSimError): + pass + #print(cl.output) + if galsim.config.UpdateNProc(2, nimages, config) > 1: + assert re.search("Process-.: Exception caught when building image",cl.output) # Finally, if all images give errors, BuildFiles will not raise an exception, but will just # report that no files were written. diff --git a/tests/test_config_output.py b/tests/test_config_output.py index a462030c21..e444b57faf 100644 --- a/tests/test_config_output.py +++ b/tests/test_config_output.py @@ -1192,25 +1192,32 @@ def writeFile(self, file_name, config, base, logger): assert "File 5 = output/test_flaky_fits_5.fits" in cl.output # Also works in nproc > 1 mode - config['output']['nproc'] = 2 - galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - #print(cl.output) - if galsim.config.UpdateNProc(2, nfiles, config) > 1: - assert re.search("Process-.: Exception caught for file 0 = output/test_flaky_fits_0.fits", - cl.output) - assert "File output/test_flaky_fits_0.fits not written! Continuing on..." in cl.output - assert re.search("Process-.: File 1 = output/test_flaky_fits_1.fits", cl.output) - assert re.search("Process-.: File 2 = output/test_flaky_fits_2.fits", cl.output) - assert re.search("Process-.: File 3 = output/test_flaky_fits_3.fits", cl.output) - assert re.search("Process-.: Exception caught for file 4 = output/test_flaky_fits_4.fits", - cl.output) - assert "File output/test_flaky_fits_4.fits not written! Continuing on..." in cl.output - assert re.search("Process-.: File 5 = output/test_flaky_fits_5.fits", cl.output) + # Under the 'spawn' start method (e.g. on Windows), the FlakyFits and flaky_weight types + # registered above are defined inside this test function, so they cannot be pickled to the + # spawned worker processes. So only check this where 'fork' is available. + from multiprocessing import get_all_start_methods + if 'fork' in get_all_start_methods(): + config['output']['nproc'] = 2 + galsim.config.RemoveCurrent(config) + with CaptureLog() as cl: + galsim.config.Process(config, logger=cl.logger) + #print(cl.output) + if galsim.config.UpdateNProc(2, nfiles, config) > 1: + assert re.search( + "Process-.: Exception caught for file 0 = output/test_flaky_fits_0.fits", + cl.output) + assert "File output/test_flaky_fits_0.fits not written! Continuing on..." in cl.output + assert re.search("Process-.: File 1 = output/test_flaky_fits_1.fits", cl.output) + assert re.search("Process-.: File 2 = output/test_flaky_fits_2.fits", cl.output) + assert re.search("Process-.: File 3 = output/test_flaky_fits_3.fits", cl.output) + assert re.search( + "Process-.: Exception caught for file 4 = output/test_flaky_fits_4.fits", + cl.output) + assert "File output/test_flaky_fits_4.fits not written! Continuing on..." in cl.output + assert re.search("Process-.: File 5 = output/test_flaky_fits_5.fits", cl.output) + del config['output']['nproc'] # Otherwise which file fails in non-deterministic. # But with except_abort = True, it will stop after the first failure - del config['output']['nproc'] # Otherwise which file fails in non-deterministic. with CaptureLog() as cl: try: galsim.config.Process(config, logger=cl.logger, except_abort=True) @@ -1695,7 +1702,12 @@ def test_timeout(): im1 = galsim.fits.read('output/test_timeout_%d.fits'%n) assert im1 == im - if platform.python_implementation() != 'PyPy': + # Under the 'spawn' start method (e.g. on Windows), Process.start() blocks while each + # worker boots and reads its (pickled) args, so these fast stamp jobs all finish before + # the main process starts waiting on the results queue, and the tiny timeout below never + # triggers. So only check this where 'fork' is available. + from multiprocessing import get_all_start_methods + if platform.python_implementation() != 'PyPy' and 'fork' in get_all_start_methods(): # Check that it behaves sensibly if it hits timeout limit. # This time, it will continue on after each error, but report the error in the log. config2 = galsim.config.CleanConfig(config2) diff --git a/tests/test_image.py b/tests/test_image.py index 09aeee79a2..610d2aa1df 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -763,7 +763,9 @@ def test_Image_MultiFITS_IO(run_slow): galsim.fits.writeMulti(image_list,test_multi_file) # Check pyfits read for sanity - with pyfits.open(test_multi_file) as fits: + # memmap=False so the lazily-loaded data doesn't keep the file handle open, + # which would break the later writeFile (clobber) on Windows. + with pyfits.open(test_multi_file, memmap=False) as fits: test_array = fits[0].data np.testing.assert_array_equal(ref_array.astype(types[i]), test_array, err_msg="PyFITS failing to read multi file.") @@ -1097,7 +1099,9 @@ def test_Image_CubeFITS_IO(run_slow): galsim.fits.writeCube(image_list,test_cube_file) # Check pyfits read for sanity - with pyfits.open(test_cube_file) as fits: + # memmap=False so the lazily-loaded data doesn't keep the file handle open, + # which would break the later writeCube (clobber) on Windows. + with pyfits.open(test_cube_file, memmap=False) as fits: test_array = fits[0].data wrong_type_error_msg = "%s != %s" % (test_array.dtype.type, types[i]) diff --git a/tests/test_real.py b/tests/test_real.py index 67b92db407..f7e775b961 100644 --- a/tests/test_real.py +++ b/tests/test_real.py @@ -77,8 +77,8 @@ def test_real_galaxy_catalog(): # Test some values that are lazy evaluated: assert rgc.ident[0] == '100533' - assert rgc.gal_file_name[0] == './real_comparison_images/test_images.fits' - assert rgc.psf_file_name[0] == './real_comparison_images/test_images.fits' + assert rgc.gal_file_name[0] == os.path.join('./real_comparison_images', 'test_images.fits') + assert rgc.psf_file_name[0] == os.path.join('./real_comparison_images', 'test_images.fits') assert rgc.noise_file_name is None np.testing.assert_array_equal(rgc.gal_hdu, [0,1]) np.testing.assert_array_equal(rgc.psf_hdu, [2,3]) diff --git a/tests/test_sensor.py b/tests/test_sensor.py index e37662dc8d..8e58c25f14 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -447,7 +447,7 @@ def test_silicon_area(): silicon = galsim.SiliconSensor(name='lsst_itl_8', rng=rng) area_image = silicon.calculate_pixel_areas(im) # Get the area data from the Poisson simulation - area_filename = silicon.vertex_file.split('/')[-1].strip('.dat')+'_areas.dat' + area_filename = os.path.basename(silicon.vertex_file).strip('.dat')+'_areas.dat' area_dir = os.path.join(os.getcwd(),'sensor_validation/') area_data = np.loadtxt(area_dir+area_filename, skiprows = 1) # Now test that they are almost equal