Support internal imports

Author: asottile

Diff for: README.md

@@ -18,12 +18,17 @@ It is incompatible with pypy3 (for now) due to a lack of c-api.
 ## Usage
 
 Add `setuptools-golang` to the `setup_requires` in your setup.py and
-`build_golang=True`.
+`build_golang={'root': ...}`.  `root` refers to the root go import path of
+your project.
+
+An extension must only contain a single file in the `main` go package.
+You may have multiple extensions in your `setup.py`.
 
 ```python
 setup(
     ...
-    build_golang=True,
+    build_golang={'root': 'github.com/user/project'},
+    ext_modules=[Extension('example', ['example.go'])],
     setup_requires=['setuptools-golang'],
     ...
 )

Diff for: setuptools_golang.py

@@ -76,40 +76,35 @@ def _tmpdir():
         shutil.rmtree(tempdir)
 
 
-class build_ext(_build_ext):
+def _get_build_extension_method(base, root):
     def build_extension(self, ext):
+        def _raise_error(msg):
+            raise IOError(
+                'Error building extension `{}`: '.format(ext.name) + msg,
+            )
+
         # If there are no .go files then the parent should handle this
         if not any(source.endswith('.go') for source in ext.sources):
-            return _build_ext.build_extension(self, ext)
-
-        for source in ext.sources:
-            if not os.path.exists(source):
-                raise IOError(
-                    'Error building extension `{}`: {} does not exist'.format(
-                        ext.name, source,
-                    ),
-                )
-
-        # Passing non-.go files to `go build` results in a failure
-        # Passing only .go files to `go build` causes it to ignore C files
-        # So we'll set our cwd to the root of the files and go from there!
-        source_dirs = {os.path.dirname(src) for src in ext.sources}
-        if len(source_dirs) != 1:
-            raise IOError(
-                'Error building extension `{}`: '
-                'Cannot compile across directories: {}'.format(
-                    ext.name, ' '.join(sorted(source_dirs)),
-                )
+            return base.build_extension(self, ext)
+
+        if len(ext.sources) != 1:
+            _raise_error(
+                'sources must be a single file in the `main` package.\n'
+                'Recieved: {!r}'.format(ext.sources)
             )
-        source_dir, = source_dirs
-        source_dir = os.path.abspath(source_dir)
+
+        main_file, = ext.sources
+        if not os.path.exists(main_file):
+            _raise_error('{} does not exist'.format(main_file))
+        main_dir = os.path.dirname(main_file)
 
         # Copy the package into a temporary GOPATH environment
         with _tmpdir() as tempdir:
-            srcdir = os.path.join(tempdir, 'src')
-            os.mkdir(srcdir)
-            pkg_path = os.path.join(srcdir, '_mypkg')
-            shutil.copytree(source_dir, pkg_path)
+            root_path = os.path.join(tempdir, 'src', root)
+            # Make everything but the last directory (copytree interface)
+            os.makedirs(os.path.dirname(root_path))
+            shutil.copytree('.', root_path)
+            pkg_path = os.path.join(root_path, main_dir)
 
             env = {
                 'GOPATH': tempdir,
@@ -131,8 +126,17 @@ def build_extension(self, ext):
                 cmd_build, cwd=pkg_path, env=dict(os.environ, **env),
             )
 
+    return build_extension
+
+
+def _get_build_ext_cls(base, root):
+    class build_ext(base):
+        build_extension = _get_build_extension_method(base, root)
+
+    return build_ext
+
 
 def set_build_ext(dist, attr, value):
-    if not value:
-        return
-    dist.cmdclass['build_ext'] = build_ext
+    root = value['root']
+    base = dist.cmdclass.get('build_ext', _build_ext)
+    dist.cmdclass['build_ext'] = _get_build_ext_cls(base, root)

Diff for: setuptools_golang_test.py

@@ -36,7 +36,7 @@ def run(*cmd, **kwargs):
     if returncode is not None:
         if proc.returncode != returncode:
             raise AssertionError(
-                '{!r} returned {} (expected {})\nout:\n{}err:\n{}'.format(
+                '{!r} returned {} (expected {})\nout:\n{}\nerr:\n{}\n'.format(
                     cmd, proc.returncode, returncode, out, err,
                 )
             )
@@ -49,14 +49,11 @@ def run_output(*cmd, **kwargs):
 
 def test_sets_cmdclass():
     dist = Distribution()
-    setuptools_golang.set_build_ext(dist, 'build_golang', True)
-    assert dist.cmdclass['build_ext'] == setuptools_golang.build_ext
-
-
-def test_sets_cmdclass_value_falsey():
-    dist = Distribution()
-    setuptools_golang.set_build_ext(dist, 'build_golang', False)
-    assert dist.cmdclass.get('build_ext') != setuptools_golang.build_ext
+    assert not dist.cmdclass.get('build_ext')
+    setuptools_golang.set_build_ext(
+        dist, 'build_golang', {'root': 'github.com/asottile/fake'},
+    )
+    assert dist.cmdclass['build_ext']
 
 
 GET_LDFLAGS = (
@@ -157,5 +154,14 @@ def test_integration_multidir(venv):
     assert ret.returncode != 0
     assert (
         'Error building extension `multidir`: '
-        'Cannot compile across directories: dir1 dir2' in ret.out
+        'sources must be a single file in the `main` package.' in ret.out
     )
+
+
+OHAI = 'import hello_lib; print(hello_lib.ohai(u"Anthony"))'
+
+
+def test_integration_internal_imports(venv):
+    run(venv.pip, 'install', 'testing/internal_imports')
+    out = run_output(venv.python, '-c', OHAI)
+    assert out == 'ohai, Anthony\n'

Diff for: testing/imports_gh/setup.py

@@ -5,7 +5,7 @@
 setup(
     name='imports-gh',
     ext_modules=[Extension('red', ['red.go'])],
-    build_golang=True,
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.

Diff for: testing/internal_imports/README.md

@@ -0,0 +1 @@
+Assume this directory is actually rooted at github.com/asottile/fake

Diff for: testing/internal_imports/hello/ohai.go

@@ -0,0 +1,5 @@
+package hello
+
+func OHai(s string) string {
+	return "ohai, " + s
+}

Diff for: testing/internal_imports/hello_lib/hello_lib.c

@@ -0,0 +1,47 @@
+#include <Python.h>
+
+#if PY_MAJOR_VERSION >= 3
+#define PyHelloLib_Bytes_AS_STRING PyBytes_AS_STRING
+#else
+#define PyHelloLib_Bytes_AS_STRING PyString_AS_STRING
+#endif
+
+/* Will come from go */
+PyObject* ohai(PyObject*);
+
+/* To shim go's missing variadic function support */
+int PyArg_ParseTuple_U(PyObject* args, PyObject** obj) {
+    return PyArg_ParseTuple(args, "U", obj);
+}
+
+/* To shim go's lack of C macro support */
+void PyHelloLib_DECREF(PyObject* obj) {
+    Py_DECREF(obj);
+}
+
+const char* PyHelloLib_Bytes_AsString(PyObject* s) {
+    return PyHelloLib_Bytes_AS_STRING(s);
+}
+
+static struct PyMethodDef methods[] = {
+    {"ohai", (PyCFunction)ohai, METH_VARARGS},
+    {NULL, NULL}
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT,
+    "hello_lib",
+    NULL,
+    -1,
+    methods
+};
+
+PyMODINIT_FUNC PyInit_hello_lib(void) {
+    return PyModule_Create(&module);
+}
+#else
+PyMODINIT_FUNC inithello_lib(void) {
+    Py_InitModule3("hello_lib", methods, NULL);
+}
+#endif

Diff for: testing/internal_imports/hello_lib/main.go

@@ -0,0 +1,30 @@
+package main
+
+// #include <stdlib.h>
+// #include <Python.h>
+// int PyArg_ParseTuple_U(PyObject*, PyObject**);
+// const char* PyHelloLib_Bytes_AsString(PyObject*);
+// void PyHelloLib_DECREF(PyObject*);
+import "C"
+import "unsafe"
+import "github.com/asottile/fake/hello"
+
+//export ohai
+func ohai(self *C.PyObject, args *C.PyObject) *C.PyObject {
+	var obj *C.PyObject
+	if C.PyArg_ParseTuple_U(args, &obj) == 0 {
+		return nil
+	}
+	bytes := C.PyUnicode_AsUTF8String(obj)
+	cstr := C.PyHelloLib_Bytes_AsString(bytes)
+	ohai := hello.OHai(C.GoString(cstr))
+	cstr = C.CString(ohai)
+	ret := C.PyUnicode_FromString(cstr)
+
+	C.free(unsafe.Pointer(cstr))
+	C.PyHelloLib_DECREF(bytes)
+
+	return ret
+}
+
+func main() {}

Diff for: testing/internal_imports/setup.py

@@ -0,0 +1,13 @@
+from setuptools import Extension
+from setuptools import setup
+
+
+setup(
+    name='internal_imports',
+    ext_modules=[Extension('hello_lib', ['hello_lib/main.go'])],
+    build_golang={'root': 'github.com/asottile/fake'},
+    # Would do this, but we're testing *our* implementation and this would
+    # install from pypi.  We can rely on setuptools-golang being already
+    # installed under test.
+    # setup_requires=['setuptools-golang'],
+)

Diff for: testing/multidir/setup.py

@@ -7,7 +7,7 @@
     ext_modules=[Extension(
         'multidir', ['dir1/sum.go', 'dir2/sum_support.go'],
     )],
-    build_golang=True,
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.

Diff for: testing/notfound/setup.py

@@ -5,7 +5,7 @@
 setup(
     name='notfound',
     ext_modules=[Extension('notfound', ['notfound.go'])],
-    build_golang=True,
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.

Diff for: testing/project_with_c/setup.py

@@ -5,13 +5,12 @@
 
 setup(
     name='project_with_c',
-    version='0.1.0',
     packages=find_packages(),
     ext_modules=[
         Extension('project_with_c', ['project_with_c.c']),
         Extension('project_with_c_sum.sum', ['project_with_c_sum/sum.go']),
     ],
-    build_golang=True,
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.

Diff for: testing/sum/setup.py

@@ -5,7 +5,7 @@
 setup(
     name='sum',
     ext_modules=[Extension('sum', ['sum.go'])],
-    build_golang=True,
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.

Diff for: testing/sum_pure_go/setup.py

@@ -4,8 +4,8 @@
 
 setup(
     name='sum_pure_go',
-    ext_modules=[Extension('sum_pure_go', ['sum.go', 'sum_support.go'])],
-    build_golang=True,
+    ext_modules=[Extension('sum_pure_go', ['sum.go'])],
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.

Diff for: testing/sum_sub_package/setup.py

@@ -7,7 +7,7 @@
     name='sum_sub_package',
     ext_modules=[Extension('sum_sub_package.sum', ['sum_sub_package/sum.go'])],
     packages=find_packages(),
-    build_golang=True,
+    build_golang={'root': 'github.com/asottile/fake'},
     # Would do this, but we're testing *our* implementation and this would
     # install from pypi.  We can rely on setuptools-golang being already
     # installed under test.