Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
*.eps binary
*.fits binary
# Pickle files are binary (protocol-0 pickles are corrupted by CRLF conversion)
*.p binary
*.pkl binary
108 changes: 108 additions & 0 deletions galsim/_astropy_shim.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
# Copyright (c) 2012-2023 by the GalSim developers team on GitHub
# https://github.com/GalSim-developers
#
# This file is part of GalSim: The modular galaxy image simulation toolkit.
# https://github.com/GalSim-developers/GalSim
#
# GalSim is free software: redistribution and use in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions, and the disclaimer given in the accompanying LICENSE
# file.
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions, and the disclaimer given in the documentation
# and/or other materials provided with the distribution.
#
"""Single-probe shim around the astropy modules GalSim imports at top level.

astropy.units cannot initialise inside frozen binaries (Nuitka onefile,
PyInstaller-style bundles): its PLY parser builds grammar tables through
``sys._getframe()`` locals introspection that compiled frames do not provide
(see astropy/astropy#15069). GalSim only needs astropy.units for
spectral/chromatic features; the geometry, zernike and photon-shooting paths
never touch it.

This module probes astropy ONCE per process. In any normal source install the
real modules are re-exported unchanged, so behaviour is identical to importing
astropy directly. When astropy.units cannot initialise, inert fallbacks are
exported instead: imports succeed, config parameter schemas keep working
(upper-case attribute names resolve to a placeholder type that is safe in
``isinstance`` checks), and spectral features fail only if actually exercised.
"""

__all__ = ["units", "constants", "wcs", "Quantity", "Unit",
"HAVE_ASTROPY_UNITS"]


class _FallbackType:
"""Placeholder for unit/quantity classes in isinstance() checks."""

def __init__(self, *args, **kwargs):
pass


class _FallbackValue:
"""Inert placeholder surviving attribute access, calls and arithmetic."""
__slots__ = ()

def __call__(self, *args, **kwargs):
return self

def __getattr__(self, name):
# Keep introspection honest: probing __file__/__wrapped__/etc. must
# raise, or tools that scan objects (inspect, copy, pickle) misbehave.
if name.startswith("__") and name.endswith("__"):
raise AttributeError(name)
return _FallbackType if name[:1].isupper() else self

def __mul__(self, other):
return self
__rmul__ = __truediv__ = __rtruediv__ = __pow__ = __mul__
__add__ = __radd__ = __sub__ = __rsub__ = __mul__

def __float__(self):
return 1.0

def __repr__(self):
return "<galsim astropy fallback>"


_FALLBACK_VALUE = _FallbackValue()


class _FallbackModule:
"""Module stand-in: upper-case attrs -> placeholder type, else values."""

def __init__(self, name):
self._name = name

def __getattr__(self, name):
if name.startswith("__") and name.endswith("__"):
raise AttributeError(name)
return _FallbackType if name[:1].isupper() else _FALLBACK_VALUE

def __repr__(self):
return "<galsim fallback for %s>" % self._name


try:
from astropy import units, constants
HAVE_ASTROPY_UNITS = True
except Exception: # pragma: no cover - frozen builds without working units
units = _FallbackModule("astropy.units")
constants = _FallbackModule("astropy.constants")
HAVE_ASTROPY_UNITS = False

if HAVE_ASTROPY_UNITS:
try:
import astropy.wcs as wcs
except Exception: # pragma: no cover - degrade wcs independently
wcs = _FallbackModule("astropy.wcs")
else:
wcs = _FallbackModule("astropy.wcs")

# Names galsim.config imports directly.
Quantity = units.Quantity
Unit = units.Unit
2 changes: 1 addition & 1 deletion galsim/airy.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
__all__ = [ 'Airy' ]

import math
import astropy.units as u
from ._astropy_shim import units as u

from . import _galsim
from .gsobject import GSObject
Expand Down
6 changes: 3 additions & 3 deletions galsim/bandpass.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@

import numpy as np
import os
from astropy import units
from ._astropy_shim import units
from numbers import Real
from pathlib import PosixPath
from pathlib import PurePath

from .errors import GalSimRangeError, GalSimValueError, GalSimIncompatibleValuesError
from .table import LookupTable, _LookupTable
Expand Down Expand Up @@ -228,7 +228,7 @@ def _initialize_tp(self):

if self._tp is not None:
pass
elif isinstance(self._orig_tp, (basestring, PosixPath)):
elif isinstance(self._orig_tp, (basestring, PurePath)):
isfile, filename = utilities.check_share_file(self._orig_tp, 'bandpasses')
if isfile:
self._tp = LookupTable.from_file(filename, interpolant=self.interpolant)
Expand Down
2 changes: 1 addition & 1 deletion galsim/chromatic.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import math
import numpy as np
from astropy import units
from ._astropy_shim import units
import copy

from .gsobject import GSObject
Expand Down
2 changes: 1 addition & 1 deletion galsim/config/bandpass.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
#

import logging
from astropy.units import Quantity, Unit
from .._astropy_shim import Quantity, Unit

from .util import LoggerWrapper
from .value import ParseValue, GetAllParams, GetIndex
Expand Down
20 changes: 15 additions & 5 deletions galsim/config/extra.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,18 @@
# builder classes that will perform the different processing functions.
valid_extra_outputs = {}


class _OutputManager(SafeManager):
"""Manager subclass used by `SetupExtraOutput` to proxy work-space containers across
processes.

Defined at module scope (rather than nested inside SetupExtraOutput) so the manager type is
picklable under the 'spawn' start method on Windows, where the manager's server process
receives the manager type by reference.
"""
pass


def SetupExtraOutput(config, logger=None):
"""
Set up the extra output items as necessary, including building Managers for the work
Expand All @@ -61,13 +73,11 @@ def SetupExtraOutput(config, logger=None):
ParseValue(config['image'], 'nproc', config, int)[0] != 1 )

if use_manager and 'output_manager' not in config:
class OutputManager(SafeManager): pass

# We'll use a list and a dict as work space to do the extra output processing.
OutputManager.register('dict', dict, DictProxy)
OutputManager.register('list', list, ListProxy)
_OutputManager.register('dict', dict, DictProxy)
_OutputManager.register('list', list, ListProxy)
# Start up the output_manager
config['output_manager'] = OutputManager()
config['output_manager'] = _OutputManager()
with single_threaded():
config['output_manager'].start()

Expand Down
46 changes: 41 additions & 5 deletions galsim/config/input.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@
def InputProxy(target):
""" Create a derived NamespaceProxy class for `target`. """

# If we already made a proxy type for this target, reuse it. This also makes the
# dynamically generated class picklable by reference (needed for the 'spawn' start
# method, e.g. on Windows), since we publish it as a module attribute below.
# Note: use globals() rather than getattr to avoid recursing into the module-level
# __getattr__ defined below.
proxy_name = target.__name__ + "_Proxy"
if proxy_name in globals():
return globals()[proxy_name]

# This bit follows what multiprocessing.managers.MakeProxy normally does.
dic = {}
public_methods = [m for m in dir(target) if m[0] != '_']
Expand All @@ -75,8 +84,37 @@ def InputProxy(target):
# Expose all the public methods and also __getattribute__ and __setattr__.
ProxyType._exposed_ = tuple(public_methods + ['__getattribute__', '__setattr__'])

# Publish the class as a module attribute so pickle can find it by reference.
ProxyType.__module__ = __name__
globals()[proxy_name] = ProxyType

return ProxyType


def __getattr__(name):
# PEP 562 module-level __getattr__.
# When pickle looks up a dynamically generated proxy class by reference (e.g.
# galsim.config.input.Catalog_Proxy) in a freshly spawned process, InputProxy has not
# been called there yet, so the attribute doesn't exist. Regenerate it on demand from
# the registered input types.
if name.endswith('_Proxy'):
target_name = name[:-len('_Proxy')]
for loader in valid_input_types.values():
if loader.init_func.__name__ == target_name:
return InputProxy(loader.init_func)
raise AttributeError("module %r has no attribute %r" % (__name__, name))


class _InputManager(SafeManager):
"""Manager subclass used by `ProcessInput` to proxy input objects across processes.

Defined at module scope (rather than nested inside ProcessInput) so the manager type is
picklable under the 'spawn' start method on Windows, where the manager's server process
receives the manager type by reference.
"""
pass


def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False):
"""
Process the input field, reading in any specified input files or setting up
Expand Down Expand Up @@ -136,19 +174,17 @@ def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False):
ParseValue(config['output'], 'nproc', config, int)[0] != 1) ) )

if use_manager and '_input_manager' not in config:
class InputManager(SafeManager): pass

# Register each input field with the InputManager class
# Register each input field with the _InputManager class
for key in all_keys:
fields = config['input'][key]
nfields = len(fields) if isinstance(fields, list) else 1
for num in range(nfields):
tag = key + str(num)
init_func = valid_input_types[key].init_func
proxy = InputProxy(init_func)
InputManager.register(tag, init_func, proxy)
_InputManager.register(tag, init_func, proxy)
# Start up the input_manager
config['_input_manager'] = InputManager()
config['_input_manager'] = _InputManager()
with single_threaded():
# Starting in python 3.12, there is a deprecation warning about using fork when
# a process is multithreaded. This can get triggered here by the start()
Expand Down
2 changes: 1 addition & 1 deletion galsim/config/sed.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# and/or other materials provided with the distribution.
#

from astropy.units import Quantity, Unit
from .._astropy_shim import Quantity, Unit

from .util import LoggerWrapper
from .value import ParseValue, GetAllParams, GetIndex
Expand Down
Loading