Update secp, build fixes, add M1 and universal2 support

.github/workflows/wheels.yml

@@ -51,6 +51,7 @@ jobs:
         uses: pypa/cibuildwheel@v2.1.2
         env:
           CIBW_SKIP: cp36-*
+          CIBW_ARCHS_MACOS: "x86_64 arm64 universal2"
 
       - name: Install qemu aarch64
         if: runner.os == 'Linux'

README.md

@@ -64,6 +64,8 @@ $ brew install swig
    optimisations (default: no).
 - `--enable-minimal`. Minimises library size and memory requirements to target
    embedded or resource-constrained environments (default: no).
+- `--enable-asm`. Enables fast assembly language implementations where available.
+   (default: enabled for non-debug builds).
 - `--enable-export-all`. Export all functions from the wally shared library.
    Ordinarily only API functions are exported. (default: no). Enable this
    if you want to test the internal functions of the library or are planning

configure.ac

@@ -27,67 +27,16 @@ case $host in
 esac
 AM_CONDITIONAL([IS_MINGW], [test "x$is_mingw" == "xyes"])
 
-AM_INIT_AUTOMAKE([foreign subdir-objects])
-saved_cflags="$CFLAGS"
-LT_INIT([disable-static])
-CFLAGS="$saved_cflags"
+# Require Automake 1.11.2 for AM_PROG_AR
+AM_INIT_AUTOMAKE([1.11.2 foreign subdir-objects])
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 AC_PROG_CC
-
-# Prefer the compilers native ar/ranlib if available
-# We have to manually loop to test these as autoconf inexplicably does
-# not provide an AC_PATH_TOOLS macro for this purpose.
-candidate_ars="ar"
-candidate_ranlibs="ranlib"
-case $CC in
-*gcc*)
-    candidate_ars="gcc-ar ar"
-    candidate_ranlibs="gcc-ranlib ranlib"
-    ;;
-*clang*)
-    ver=$($CC --version | head -n 1 | cut -d' '  -f3 | cut -d'.' -f1)
-    candidate_ars="llvm-ar-$ver llvm-ar ar"
-    candidate_ranlibs="llvm-ranlib-$ver llvm-ranlib ranlib"
-    ;;
-esac
-
-if test "x$is_osx" == "xyes"; then
-    candidate_ars="libtool $candidate_ars"
-    CCDIR=`dirname $CC`
-    if test x"$CCDIR" != x"."; then
-        if test -x $CCDIR/libtool; then
-            # Use libtool from the same directory as our clang
-            AR=$CCDIR/libtool
-            candidate_ars=""
-        fi
-    fi
-fi
-
-if test -n "$candidate_ars"; then
-    for candidate in $candidate_ars; do
-        AC_PATH_TOOL(AR, $candidate)
-        if test "x$HAVE_AR" == "xyes"; then
-            break
-        fi
-    done
-fi
-case $AR in
-*libtool)
-    ARFLAGS="-static -o"
-    AR_FLAGS="-static -o"
-    AC_SUBST([ARFLAGS])
-    AC_SUBST([AR_FLAGS])
-    ;;
-esac
-
-for candidate in $candidate_ranlibs; do
-    AC_PATH_TOOL(RANLIB, $candidate)
-    if test "x$HAVE_RANLIB" == "xyes"; then
-        break
-    fi
-done
+AM_PROG_AR
+#saved_cflags="$CFLAGS"
+LT_INIT([disable-static])
+#CFLAGS="$saved_cflags"
 
 AC_SUBST([AR])
 AC_SUBST([RANLIB])
@@ -130,6 +79,9 @@ AC_ARG_ENABLE(minimal,
 AC_ARG_ENABLE(secp256k1-tests,
     AS_HELP_STRING([--enable-secp256k1-tests],[enable secp256k1 tests (default: no)]),
     [secp256k1_tests=$enableval], [secp256k1_tests=no])
+AC_ARG_ENABLE(asm,
+    AS_HELP_STRING([--enable-asm],[enable assembly language implementations (default: yes)]),
+    [asm=$enableval], [asm=yes])
 AM_CONDITIONAL([RUN_TESTS], [test "x$tests" == "xyes"])
 AM_CONDITIONAL([BUILD_ELEMENTS], [test "x$elements" == "xyes"])
 AM_CONDITIONAL([BUILD_STANDARD_SECP], [test "x$standard_secp" == "xyes"])
@@ -222,11 +174,6 @@ AC_SUBST([NOOPT_CFLAGS])
 AC_SUBST([NOALIAS_CFLAGS])
 AC_SUBST([NOBUILTIN_CFLAGS])
 
-# Under OSX the compiler accepts this flag but the linker then fails.
-if test "x$is_osx" != "xyes"; then
-    AX_CHECK_COMPILE_FLAG([-Wl,--whole-archive], [whole_archive=yes])
-fi
-
 # SWIG versions vary in generated code quality; skip warnings
 SWIG_WARN_CFLAGS="-fno-strict-aliasing"
 AX_CHECK_COMPILE_FLAG([-Wno-shadow], [SWIG_WARN_CFLAGS="$SWIG_WARN_CFLAGS -Wno-shadow"])
@@ -262,17 +209,19 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <stdlib.h>]],[[return posix_memalign(
 
 AC_CHECK_FUNCS([memset_s explicit_bzero explicit_memset])
 
-AC_MSG_CHECKING(whether we can use inline asm code)
-AC_LINK_IFELSE([AC_LANG_PROGRAM([[
-]], [[
-int a = 42;
-int *pnt = &a;
-__asm__ __volatile__ ("" : : "r"(pnt) : "memory");
-]])],
-  [AC_MSG_RESULT(yes)
-   AC_DEFINE([HAVE_INLINE_ASM], [1], [inline asm code can be used])],
-  [AC_MSG_RESULT(no)]
-)
+if test "x$asm" == "xyes"; then
+    AC_MSG_CHECKING(whether we can use inline asm code)
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+    ]], [[
+    int a = 42;
+    int *pnt = &a;
+    __asm__ __volatile__ ("" : : "r"(pnt) : "memory");
+    ]])],
+      [AC_MSG_RESULT(yes)
+       AC_DEFINE([HAVE_INLINE_ASM], [1], [inline asm code can be used])],
+      [AC_MSG_RESULT(no)]
+    )
+fi
 
 AC_CHECK_HEADERS([byteswap.h, sys/mman.h])
 
@@ -297,18 +246,18 @@ fi
 #        target and so won't force all object files in the library to be
 #        included in ours - despite the fact that we are making a shared
 #        library and linking to a static one. This is broken and we work
-#        around it by hacking the whole-archive flags into the _LDADD variable
-#        for wallycore.
+#        around it by hacking the secp objects directly into the library
+#        via the _LDADD variable for wallycore.
+#        We previously achieved this by adding the libsecp256k1.a archive,
+#        but changes to libtool and apples linkers mean that
+#        archives-within-archives no longer work.
 #        Because automake tries to police its users very strictly and fails
 #        hard when flags are passed in this way, we have to substitute the
 #        flags here.
 #        Because libtool both intercepts -Wl and arbitrarily re-orders its
 #        command line inputs, we have to concoct a single expression to
 #        enforce linking that cannot be split, hence the below expression.
-LIBADD_SECP256K1="secp256k1/.libs/libsecp256k1.a"
-if test "x$whole_archive" == "xyes"; then
-    LIBADD_SECP256K1="-Wl,--whole-archive,secp256k1/.libs/libsecp256k1.a,--no-whole-archive"
-fi
+LIBADD_SECP256K1="-Wl,secp256k1/src/libsecp256k1_la-secp256k1.${OBJEXT},secp256k1/src/libsecp256k1_precomputed_la-precomputed_ecmult_gen.${OBJEXT},secp256k1/src/libsecp256k1_precomputed_la-precomputed_ecmult.${OBJEXT}"
 AC_SUBST([LIBADD_SECP256K1])
 
 #
@@ -437,6 +386,9 @@ AC_CONFIG_FILES([
 ])
 
 secp_asm="--with-asm=auto"
+if test "x$asm" == "xno"; then
+    secp_asm="--with-asm=no"
+fi
 if test "x$debug" == "xyes"; then
     secp_asm="--with-asm=no"
 fi

setup.py

@@ -1,8 +1,39 @@
 """setuptools config for wallycore """
 from setuptools import setup, Extension
-import platform
+import copy, os, platform
+import distutils.sysconfig
+
+CONFIGURE_ARGS = '--enable-swig-python --enable-python-manylinux --enable-ecmult-static-precomputation'
+CONFIGURE_ARGS += ' --enable-elements --disable-swig-java --disable-tests --disable-dependency-tracking'
+
+distutils_env = distutils.sysconfig.get_config_vars()
+configure_env = copy.deepcopy(os.environ)
 
 is_windows = platform.system() == "Windows"
+arch_flags = os.environ.get('ARCHFLAGS','').split()
+archs = []
+while arch_flags:
+    if arch_flags[0] == '-arch':
+        archs.append(arch_flags[1])
+        arch_flags.pop(0)
+    arch_flags.pop(0)
+
+if os.environ.get('GITHUB_ACTION') and os.environ.get('RUNNER_OS') == 'macOS':
+    # Github CI build on an macOS box
+    is_x86 = os.environ.get('RUNNER_ARCH', '') == 'X64'
+    is_native = not archs or (len(archs) == 1 and (is_x86 == (archs[0] == 'x86_64')))
+    # TODO: Enable builds on M1 macs once github supports them
+    if is_x86 and not is_native:
+        # We are cross-compiling or compiling a univeral2 binary.
+        # Configure our source code as a cross compile to make the build work
+        CONFIGURE_ARGS += ' --host x86_64-apple-darwin'
+        arch = 'universal2' if len(archs) > 1 else archs[0]
+        CONFIGURE_ARGS += ' --target {}-apple-macos'.format(arch)
+        if len(archs) > 1:
+            CONFIGURE_ARGS += ' --with-asm=no'
+        if 'PY_CFLAGS' in distutils_env:
+            configure_env['CFLAGS'] = distutils_env['PY_CFLAGS']
+            configure_env['LDFLAGS'] = distutils_env['PY_LDFLAGS']
 
 if not is_windows:
     # Run the autotools/make build up front to generate our sources,
@@ -15,11 +46,11 @@
     abs_path = os.path.dirname(os.path.abspath(__file__)) + '/'
 
     def call(cmd):
-        subprocess.check_call(cmd.split(' '), cwd=abs_path)
+        subprocess.check_call(cmd.split(' '), cwd=abs_path, env=configure_env)
 
     call('./tools/cleanup.sh')
     call('./tools/autogen.sh')
-    call('./configure --enable-swig-python --enable-python-manylinux --enable-ecmult-static-precomputation --enable-elements --disable-tests')
+    call('./configure {}'.format(CONFIGURE_ARGS))
     call('make -j{}'.format(multiprocessing.cpu_count()))
 
 define_macros=[
@@ -31,7 +62,7 @@ def call(cmd):
     ]
 if is_windows:
     define_macros.append(('USE_ECMULT_STATIC_PRECOMPUTATION', None))
-    define_macros.append(('ECMULT_WINDOW_SIZE', 16))
+    define_macros.append(('ECMULT_WINDOW_SIZE', 15))
 
 include_dirs=[
     './',

src/elements.c

@@ -614,7 +614,7 @@ int wally_asset_pak_whitelistproof(
     }
 
     if (secp256k1_whitelist_sign(ctx, &sig, online_secp_keys, offline_secp_keys, num_keys,
-                                 &pubkey, online_priv_key, summed_key, key_index, NULL, NULL) &&
+                                 &pubkey, online_priv_key, summed_key, key_index) &&
         secp256k1_whitelist_verify(ctx, &sig, online_secp_keys, offline_secp_keys, num_keys, &pubkey) &&
         secp256k1_whitelist_signature_serialize(ctx, bytes_out, &sig_size, &sig)) {
         ret = WALLY_OK;

src/internal.c

@@ -21,7 +21,7 @@ static secp256k1_context *global_ctx = NULL;
 
 int privkey_tweak_add(unsigned char *seckey, const unsigned char *tweak)
 {
-    return secp256k1_ec_privkey_tweak_add(secp256k1_context_no_precomp, seckey, tweak);
+    return secp256k1_ec_seckey_tweak_add(secp256k1_context_no_precomp, seckey, tweak);
 }
 
 int pubkey_combine(secp256k1_pubkey *pubnonce, const secp256k1_pubkey *const *pubnonces, size_t n)

src/wrap_js/src/combined.c

@@ -26,13 +26,17 @@
 #include "wif.c"
 #include "wordlist.c"
 #undef PACKAGE
+#undef PACKAGE_BUGREPORT
 #undef PACKAGE_NAME
 #undef PACKAGE_STRING
 #undef PACKAGE_TARNAME
+#undef PACKAGE_URL
 #undef PACKAGE_VERSION
 #undef VERSION
 #undef SECP256K1_BUILD
 #include "src/secp256k1/src/secp256k1.c"
+#include "src/secp256k1/src/precomputed_ecmult_gen.c"
+#include "src/secp256k1/src/precomputed_ecmult.c"
 #include "ccan/ccan/crypto/sha256/sha256.c"
 
 void wally_silence_unused_warnings(void)
@@ -41,15 +45,16 @@ void wally_silence_unused_warnings(void)
     assert_bip32_assumptions();
     assert_bip38_assumptions();
     assert_tx_assumptions();
+    secp256k1_fe_get_bounds(NULL, 0);
     secp256k1_fe_inv_var(NULL, NULL);
     secp256k1_ge_set_all_gej_var(NULL, NULL, 0);
     secp256k1_gej_has_quad_y_var(NULL);
     secp256k1_ge_is_valid_var(NULL);
     secp256k1_ge_set_infinity(NULL);
-    secp256k1_ec_commit_verify(NULL, NULL, NULL, NULL, NULL, 0);
-    secp256k1_ecmult_multi_var(NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0);
-    secp256k1_ecmult_strauss_batch_single(NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0);
-    secp256k1_ecmult_pippenger_batch_single(NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0);
+    secp256k1_ec_commit_verify(NULL, NULL, NULL, NULL, 0);
+    secp256k1_ecmult_multi_var(NULL, NULL, NULL, NULL, NULL, NULL, 0);
+    secp256k1_ecmult_strauss_batch_single(NULL, NULL, NULL, NULL, NULL, NULL, 0);
+    secp256k1_ecmult_pippenger_batch_single(NULL, NULL, NULL, NULL, NULL, NULL, 0);
     secp256k1_pippenger_scratch_size(0, 0);
     secp256k1_scalar_chacha20(NULL, NULL, NULL, 0);
     secp256k1_sha256_initialize_tagged(NULL, NULL, 0);

src/wrap_js/windows_config/binding.gyp.elements_tmpl

@@ -3,7 +3,7 @@
     {
       "target_name": "deps",
       "sources": [ "src/combined.c", "src/combined_ccan.c", "src/combined_ccan2.c" ],
-      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=16", "BUILD_ELEMENTS" ],
+      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=15", "BUILD_ELEMENTS" ],
       "include_dirs": [ "<(platform_include_dirs)", "<(libwally_dir)", "<(libwally_dir)/src", "<(libwally_dir)/src/secp256k1", "<(libwally_dir)/src/secp256k1/src", "<(libwally_dir)/src/ccan" ],
       "type": "static_library"
     },
@@ -12,7 +12,7 @@
       "dependencies": [ "deps" ],
       "sources": [ "nodejs_wrap.cc" ],
       "include_dirs": [ "<(platform_include_dirs)", "<(libwally_dir)/src", "<!(node -e \"require('nan')\")" ],
-      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=16", "BUILD_ELEMENTS" ],
+      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=15", "BUILD_ELEMENTS" ],
     }
   ],
   "conditions": [

src/wrap_js/windows_config/binding.gyp.tmpl

@@ -3,7 +3,7 @@
     {
       "target_name": "deps",
       "sources": [ "src/combined.c", "src/combined_ccan.c", "src/combined_ccan2.c" ],
-      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=16" ],
+      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=15" ],
       "include_dirs": [ "<(platform_include_dirs)", "<(libwally_dir)", "<(libwally_dir)/src", "<(libwally_dir)/src/secp256k1", "<(libwally_dir)/src/secp256k1/src", "<(libwally_dir)/src/ccan" ],
       "type": "static_library"
     },
@@ -12,7 +12,7 @@
       "dependencies": [ "deps" ],
       "sources": [ "nodejs_wrap.cc" ],
       "include_dirs": [ "<(platform_include_dirs)", "<(libwally_dir)/src", "<!(node -e \"require('nan')\")" ],
-      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=16" ],
+      "defines": [ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "ECMULT_WINDOW_SIZE=15" ],
     }
   ],
   "conditions": [

tools/cleanup.sh

@@ -32,7 +32,7 @@ rm -f src/swig_java/*java
 rm -f src/swig_java/*jar
 rm -rf src/swig_java/src/com/blockstream/libwally
 rm -f src/swig_python/wallycore.py
-rm -f src/swig_python/wallycore/__init__.py
+rm -f src/swig_python/wallycore/__init__.py*
 rm -f src/swig_python/swig_python_wrap.c
 rm -f src/wrap_js/binding.gyp
 rm -rf src/wrap_js/build
@@ -47,6 +47,7 @@ rm -rf src/wrap_js/cordovaplugin/jniLibs/
 rm -f src/wrap_js/wally.js
 rm -rf src/.libs
 rm -f src/secp256k1/build-aux/ltmain.sh-e
+rm -f tools/build-aux/ar-lib
 rm -f tools/build-aux/compile
 rm -f tools/build-aux/config.guess
 rm -f tools/build-aux/config.sub

tools/msvc/build.bat

@@ -21,4 +21,4 @@ if "%ELEMENTS_BUILD%" == "elements" (
 REM Compile everything (wally, ccan, libsecp256k) in one lump.
 REM Define USE_ECMULT_STATIC_PRECOMPUTATION  to pick up the
 REM ecmult_static_context.h file generated previously
-cl /utf-8 /DUSE_ECMULT_STATIC_PRECOMPUTATION /DECMULT_WINDOW_SIZE=16 /DWALLY_CORE_BUILD %ELEMENTS_OPT% /DHAVE_CONFIG_H /DSECP256K1_BUILD /I%LIBWALLY_DIR%\src\wrap_js\windows_config /I%LIBWALLY_DIR% /I%LIBWALLY_DIR%\src /I%LIBWALLY_DIR%\include /I%LIBWALLY_DIR%\src\ccan /I%LIBWALLY_DIR%\src\ccan\base64 /I%LIBWALLY_DIR%\src\secp256k1 /Zi /LD src/aes.c src/anti_exfil.c src/base58.c src/base64.c src/bech32.c src/bip32.c src/bip38.c src/bip39.c src/blech32.c src/ecdh.c src/elements.c src/hex.c src/hmac.c src/internal.c src/mnemonic.c src/pbkdf2.c src/psbt.c src/script.c src/scrypt.c src/sign.c src/symmetric.c src/transaction.c src/wif.c src/wordlist.c src/ccan/ccan/crypto/ripemd160/ripemd160.c src/ccan/ccan/crypto/sha256/sha256.c src/ccan/ccan/crypto/sha512/sha512.c src/ccan/ccan/base64/base64_.c src\ccan\ccan\str\hex\hex_.c src/secp256k1/src/secp256k1.c /Fewally.dll
+cl /utf-8 /DUSE_ECMULT_STATIC_PRECOMPUTATION /DECMULT_WINDOW_SIZE=15 /DWALLY_CORE_BUILD %ELEMENTS_OPT% /DHAVE_CONFIG_H /DSECP256K1_BUILD /I%LIBWALLY_DIR%\src\wrap_js\windows_config /I%LIBWALLY_DIR% /I%LIBWALLY_DIR%\src /I%LIBWALLY_DIR%\include /I%LIBWALLY_DIR%\src\ccan /I%LIBWALLY_DIR%\src\ccan\base64 /I%LIBWALLY_DIR%\src\secp256k1 /Zi /LD src/aes.c src/anti_exfil.c src/base58.c src/base64.c src/bech32.c src/bip32.c src/bip38.c src/bip39.c src/blech32.c src/ecdh.c src/elements.c src/hex.c src/hmac.c src/internal.c src/mnemonic.c src/pbkdf2.c src/psbt.c src/script.c src/scrypt.c src/sign.c src/symmetric.c src/transaction.c src/wif.c src/wordlist.c src/ccan/ccan/crypto/ripemd160/ripemd160.c src/ccan/ccan/crypto/sha256/sha256.c src/ccan/ccan/crypto/sha512/sha512.c src/ccan/ccan/base64/base64_.c src\ccan\ccan\str\hex\hex_.c src/secp256k1/src/secp256k1.c src/secp256k1/src/precomputed_ecmult_gen.c src/secp256k1/src/precomputed_ecmult.c /Fewally.dll

tools/msvc/gen_ecmult_static_context.bat

@@ -1,2 +1,2 @@
-cl /utf-8 /DWALLY_CORE_BUILD /DHAVE_CONFIG_H /DSECP256K1_BUILD /I%LIBWALLY_DIR%\src\wrap_js\windows_config /I%LIBWALLY_DIR%\ /I%LIBWALLY_DIR%\src /I%LIBWALLY_DIR%\include /I%LIBWALLY_DIR%\src\ccan /I%LIBWALLY_DIR%\src\secp256k1 /Zi %LIBWALLY_DIR%\src/secp256k1/src/gen_context.c /Fegen_context.exe
+cl /utf-8 /DWALLY_CORE_BUILD /DHAVE_CONFIG_H /DSECP256K1_BUILD /I%LIBWALLY_DIR%\src\wrap_js\windows_config /I%LIBWALLY_DIR%\ /I%LIBWALLY_DIR%\src /I%LIBWALLY_DIR%\include /I%LIBWALLY_DIR%\src\ccan /I%LIBWALLY_DIR%\src\secp256k1 /Zi %LIBWALLY_DIR%\src/secp256k1/src/precompute_ecmult_gen.c /Fegen_context.exe
 gen_context.exe

tools/travis_install.sh

@@ -16,5 +16,5 @@ if [ "$TRAVIS_OS_NAME" = "windows" ]; then
     npm --version
     npm i -g yarn
 
-    sed -e 's/"defines": \[ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H" \]/"defines": \[ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "USE_ECMULT_STATIC_PRECOMPUTATION", "ECMULT_WINDOW_SIZE=16" \]/g' src/wrap_js/binding.gyp.tmpl > src/wrap_js/binding.gyp
+    sed -e 's/"defines": \[ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H" \]/"defines": \[ "SWIG_JAVASCRIPT_BUILD", "HAVE_CONFIG_H", "USE_ECMULT_STATIC_PRECOMPUTATION", "ECMULT_WINDOW_SIZE=15" \]/g' src/wrap_js/binding.gyp.tmpl > src/wrap_js/binding.gyp
 fi