From 307a8f5e518f8895d7ad54d3301d1548f26c34b6 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 10:58:22 +0000 Subject: [PATCH 01/12] Make al2023 dockerfile Basically just copy Dockerfile. Use dnf instead of yum. Don't install cmake3 use automake instead of automake16 You can get a nice diff by checking out this commit and running: git diff --no-index --word-diff dockerfiles/Dockerfile dockerfiles/Dockerfile.al2023 --- dockerfiles/Dockerfile.al2023 | 424 ++++++++++++++++++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100644 dockerfiles/Dockerfile.al2023 diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 new file mode 100644 index 0000000..696b786 --- /dev/null +++ b/dockerfiles/Dockerfile.al2023 @@ -0,0 +1,424 @@ +FROM public.ecr.aws/lambda/provided:al2023 AS builder + +LABEL maintainer="lambgeo " + +# AL2023 uses dnf instead of yum, and automake instead of automake16 +# cmake is available directly (not cmake3) +RUN dnf update -y && \ + dnf install -y git autoconf libtool flex bison cmake make \ + tar gzip gcc gcc-c++ automake libpng-devel nasm \ + libxml2-devel readline-devel openssl-devel curl-devel && \ + dnf clean all && \ + rm -rf /var/cache/dnf + +ENV PREFIX /opt + +WORKDIR /opt + +ENV LD_LIBRARY_PATH $PREFIX/lib:$LD_LIBRARY_PATH + +# pkg-config +ENV PKGCONFIG_VERSION=0.29.2 +RUN mkdir /tmp/pkg-config \ + && curl -sfL https://pkg-config.freedesktop.org/releases/pkg-config-${PKGCONFIG_VERSION}.tar.gz | tar zxf - -C /tmp/pkg-config --strip-components=1 \ + && cd /tmp/pkg-config \ + && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-internal-glib \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/pkg-config + +ENV PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig/ + +# sqlite +RUN mkdir /tmp/sqlite \ + && curl -sfL https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz | tar zxf - -C /tmp/sqlite --strip-components=1 \ + && cd /tmp/sqlite \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/sqlite + +# libexpat +RUN mkdir /tmp/libexpat \ + && curl -sfL https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.gz | tar zxf - -C /tmp/libexpat --strip-components=1 \ + && cd /tmp/libexpat \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/libexpat + +ENV \ + SQLITE3_LIBS="-L${PREFIX}/lib -lsqlite3" \ + SQLITE3_INCLUDE_DIR="${PREFIX}/include" \ + SQLITE3_CFLAGS="$CFLAGS -I${PREFIX}/include" \ + PATH=${PREFIX}/bin/:$PATH + +# libxml2 +RUN mkdir /tmp/libxml2 \ + && curl -sfL https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.10/libxml2-v2.9.10.tar.gz | tar zxf - -C /tmp/libxml2 --strip-components=1 \ + && cd /tmp/libxml2 \ + && ./autogen.sh --prefix=$PREFIX --with-ftp=no --with-http=no --with-python=no \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libxml2 + +# nghttp2 +ENV NGHTTP2_VERSION=1.42.0 +RUN mkdir /tmp/nghttp2 \ + && curl -sfL https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz | tar zxf - -C /tmp/nghttp2 --strip-components=1 \ + && cd /tmp/nghttp2 \ + && ./configure --enable-lib-only --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/nghttp2 + +# libcurl +ENV CURL_VERSION=7.73.0 +RUN mkdir /tmp/libcurl \ + && curl -sfL https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz | tar zxf - -C /tmp/libcurl --strip-components=1 \ + && cd /tmp/libcurl \ + && ./configure --disable-manual --disable-cookies --with-nghttp2=$PREFIX --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libcurl + +# libdeflate +RUN mkdir /tmp/libdeflate \ + && curl -sfL https://github.com/ebiggers/libdeflate/archive/v1.10.tar.gz | tar zxf - -C /tmp/libdeflate --strip-components=1 \ + && cd /tmp/libdeflate \ + && make -j $(nproc) --silent PREFIX=$PREFIX && make install \ + && rm -rf /tmp/libdeflate + +# libpng +ENV LIBPNG_VERSION=1.6.37 +RUN mkdir /tmp/png \ + && curl -sfL https://github.com/glennrp/libpng/archive/v${LIBPNG_VERSION}.tar.gz | tar zxf - -C /tmp/png --strip-components=1 \ + && cd /tmp/png \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/png + +# libjpeg_turbo +ENV LIBJPEG_TURBO_VERSION=2.1.3 +RUN mkdir /tmp/jpeg \ + && curl -sfL https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${LIBJPEG_TURBO_VERSION}.tar.gz | tar zxf - -C /tmp/jpeg --strip-components=1 \ + && cd /tmp/jpeg \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/jpeg + +# webp +ENV WEBP_VERSION=1.1.0 +RUN mkdir /tmp/webp \ + && curl -sfL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | tar zxf - -C /tmp/webp --strip-components=1 \ + && cd /tmp/webp \ + && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/webp + +# zstd +ENV ZSTD_VERSION=1.4.5 +RUN mkdir /tmp/zstd \ + && curl -sfL https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz | tar zxf - -C /tmp/zstd --strip-components=1 \ + && cd /tmp/zstd \ + && make -j $(nproc) PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 --silent && make install PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 \ + && rm -rf /tmp/zstd + +# lerc +ENV LERC_VERSION=2.2.1 +RUN mkdir /tmp/lerc \ + && curl -sfL https://github.com/Esri/lerc/archive/v${LERC_VERSION}.tar.gz | tar zxf - -C /tmp/lerc --strip-components=1 \ + && cd /tmp/lerc \ + && cmake . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/lerc + +# libtiff +ENV LIBTIFF_VERSION=4.5.0 +RUN mkdir /tmp/libtiff \ + && curl -sfL https://download.osgeo.org/libtiff/tiff-${LIBTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libtiff --strip-components=1 \ + && cd /tmp/libtiff \ + && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --disable-static \ + --enable-rpath \ + --with-lerc-lib-dir=$PREFIX/lib \ + --with-lerc-include-dir=$PREFIX/include \ + --with-libdeflate-lib-dir=$PREFIX/lib \ + --with-libdeflate-include-dir=$PREFIX/include \ + --with-jpeg-lib-dir=$PREFIX/lib \ + --with-jpeg-include-dir=$PREFIX/include \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libtiff + +# postgres +ENV PG_VERSION=15.2 +RUN mkdir /tmp/postgres \ + && curl -sfL https://ftp.postgresql.org/pub/source/v${PG_VERSION}/postgresql-${PG_VERSION}.tar.gz | tar zxf - -C /tmp/postgres --strip-components=1 \ + && cd /tmp/postgres \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-openssl \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/postgres + +# openjpeg +ENV OPENJPEG_VERSION=2.5.0 +RUN mkdir /tmp/openjpeg \ + && curl -sfL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | tar zxf - -C /tmp/openjpeg --strip-components=1 \ + && cd /tmp/openjpeg \ + && mkdir build && cd build \ + && cmake .. \ + -DBUILD_THIRDPARTY:BOOL=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/openjpeg + +# geos +ENV GEOS_VERSION=3.12.1 +RUN mkdir /tmp/geos \ + && curl -sfL https://github.com/libgeos/geos/archive/refs/tags/${GEOS_VERSION}.tar.gz | tar zxf - -C /tmp/geos --strip-components=1 \ + && cd /tmp/geos \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/geos + +# proj +# We still provide proj-datumgrid but the user can choose to set `PROJ_NETWORK=ON` to use network grids +# ENV PROJ_VERSION=9.2.0 +# RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ +# && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ +# && curl -sfL http://download.osgeo.org/proj/proj-datumgrid-latest.tar.gz | tar zxf - -C /tmp/proj/data \ +# && cd /tmp/proj \ +# && aclocal && autoheader && libtoolize --force --copy && automake --add-missing --copy && autoconf \ +# && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static --enable-lto \ +# && make -j $(nproc) --silent && make install \ +# && rm -rf /tmp/proj + +ENV PROJ_VERSION=9.3.1 +RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ + && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ + && cd /tmp/proj \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_INSTALL_INCLUDEDIR:PATH=include \ + -DBUILD_TESTING=OFF \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/proj + +# GeoTIFF +ENV LIBGEOTIFF_VERSION=1.7.1 +RUN mkdir /tmp/libgeotiff \ + && curl -sfL https://github.com/OSGeo/libgeotiff/releases/download/${LIBGEOTIFF_VERSION}/libgeotiff-${LIBGEOTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libgeotiff --strip-components=1 \ + && cd /tmp/libgeotiff \ + && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-proj=$PREFIX \ + --with-jpeg=$PREFIX \ + --with-zip=yes \ + --with-zlib \ + --with-libtiff=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libgeotiff + +# Build blosc (needed for ZARR) +ENV BLOSC_VERSION=1.21.0 +RUN mkdir /tmp/blosc \ + && curl -sfL https://github.com/Blosc/c-blosc/archive/refs/tags/v${BLOSC_VERSION}.tar.gz | tar zxf - -C /tmp/blosc --strip-components=1 \ + && cd /tmp/blosc \ + && cmake -G"Unix Makefiles" \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_INSTALL_RPATH="$ORIGIN" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DBUILD_SHARED=ON \ + -DBUILD_STATIC=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_TESTS=OFF \ + -DPREFER_EXTERNAL_LZ4=ON \ + -DPREFER_EXTERNAL_ZLIB=ON \ + -DPREFER_EXTERNAL_ZSTD=ON \ + -DDEACTIVATE_SNAPPY=OFF . \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/blosc + +# libcrypto (for GPC access) +RUN mkdir /tmp/cryptopp \ + && curl -sfL https://github.com/weidai11/cryptopp/archive/refs/tags/CRYPTOPP_8_6_0.tar.gz | tar zxf - -C /tmp/cryptopp --strip-components=1 \ + && cd /tmp/cryptopp \ + && make -j $(nproc) --silent && make install PREFIX=$PREFIX \ + && rm -rf /tmp/cryptopp + +# szip (for hdf) +ENV SZIP_VERSION=2.1.1 +RUN mkdir /tmp/szip \ + && curl -sfL https://support.hdfgroup.org/ftp/lib-external/szip/${SZIP_VERSION}/src/szip-${SZIP_VERSION}.tar.gz | tar zxf - -C /tmp/szip --strip-components=1 \ + && cd /tmp/szip \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/szip + +# libhdf4 +ENV HDF4_VERSION=4.2.15 +RUN mkdir /tmp/hdf4 \ + && curl -sfL https://support.hdfgroup.org/ftp/HDF/releases/HDF${HDF4_VERSION}/src/hdf-${HDF4_VERSION}.tar | tar xvf - -C /tmp/hdf4 --strip-components=1 \ + && cd /tmp/hdf4 \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-szlib=$PREFIX \ + --with-jpeg=$PREFIX \ + --enable-shared \ + --disable-static \ + --disable-netcdf \ + --disable-fortran \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/hdf4 + +# libhdf5 +ENV HDF5_VERSION=1.12.0 +RUN mkdir /tmp/hdf5 \ + && curl -sfL https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-${HDF5_VERSION}/src/hdf5-$HDF5_VERSION.tar.gz | tar zxf - -C /tmp/hdf5 --strip-components=1 \ + && cd /tmp/hdf5 \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-szlib=$PREFIX \ + --enable-cxx \ + --enable-thread-safe \ + --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/hdf5 + +# NetCDF +ENV NETCDF_VERSION=4.7.4 +RUN mkdir /tmp/netcdf \ + && curl -sfL https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz | tar zxf - -C /tmp/netcdf --strip-components=1 \ + && cd /tmp/netcdf \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" CPPFLAGS="-I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib" ./configure \ + --with-default-chunk-size=67108864 \ + --with-chunk-cache-size=67108864 \ + --prefix=$PREFIX \ + --disable-static \ + --enable-netcdf4 \ + --enable-dap \ + --with-pic \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/netcdf + +# # JPEG-XL +# # libhwy +# RUN mkdir /tmp/libhwy \ +# && curl -sfL https://github.com/google/highway/archive/refs/tags/0.16.0.tar.gz | tar zxf - -C /tmp/libhwy --strip-components=1 \ +# && cd /tmp/libhwy \ +# && mkdir build && cd build \ +# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ +# && make -j $(nproc) install && make clean \ +# && rm -rf /tmp/libhwy + +# # brotli +# ENV BROTLI_VERSION=1.0.9 +# RUN mkdir /tmp/brotli \ +# && curl -sfL https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz | tar zxf - -C /tmp/brotli --strip-components=1 \ +# && cd /tmp/brotli \ +# && mkdir build && cd build \ +# && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ../configure-cmake --prefix=$PREFIX \ +# && make -j $(nproc) --silent && make install && make clean \ +# && rm -rf /tmp/brotli + +# # # JpegXL +# ENV JXL_VERSION=0.8.1 +# RUN mkdir /tmp/jxl \ +# && curl -sfL https://github.com/libjxl/libjxl/archive/refs/tags/v${JXL_VERSION}.tar.gz | tar zxf - -C /tmp/jxl --strip-components=1 \ +# && cd /tmp/jxl \ +# && mkdir build && cd build \ +# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ +# && make -j $(nproc) install && make clean \ +# && rm -rf /tmp/jxl + +# We use commit sha to make sure we are not using `cache` when building the docker image +# "7ca88116f5a46d429251361634eb24629f315076" is the latest commit on release/3.6 branch + + +# GDAL +ARG GDAL_VERSION +RUN mkdir /tmp/gdal \ + && curl -sfL https://github.com/OSGeo/gdal/archive/refs/tags/v${GDAL_VERSION}.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 \ + && cd /tmp/gdal \ + && mkdir build && cd build \ + && cmake .. \ + -DGDAL_USE_EXTERNAL_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DGDAL_SET_INSTALL_RELATIVE_RPATH=ON \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=OFF \ + -DGDAL_USE_LERC_INTERNAL=OFF \ + -DLERC_INCLUDE_DIR=$PREFIX/include \ + -DLERC_LIBRARY=$PREFIX/lib/libLercLib.so \ + -DPNG_PNG_INCLUDE_DIR=$PREFIX/include \ + -DPNG_LIBRARY_RELEASE=$PREFIX/lib/libpng.so \ + -DBUILD_PYTHON_BINDINGS=OFF \ + && cmake --build . -j $(nproc) \ + && cmake --build . --target install \ + && rm -rf /tmp/gdal + +# from https://github.com/pypa/manylinux/blob/d8ef5d47433ba771fa4403fd48f352c586e06e43/docker/build_scripts/build.sh#L133-L138 +# Install patchelf (latest with unreleased bug fixes) +ENV PATCHELF_VERSION=0.10 +RUN mkdir /tmp/patchelf \ + && curl -sfL https://github.com/NixOS/patchelf/archive/${PATCHELF_VERSION}.tar.gz | tar zxf - -C /tmp/patchelf --strip-components=1 \ + && cd /tmp/patchelf && ./bootstrap.sh && ./configure \ + && make -j $(nproc) --silent && make install \ + && cd / && rm -rf /tmp/patchelf + +# # libturbojpeg.so is not used by GDAL. Only libjpeg.so* +RUN rm -f $PREFIX/lib/libturbojpeg.so* \ + # Only libwebp.so is used by GDAL + && rm -f $PREFIX/lib/libwebpmux.so* $PREFIX/lib/libwebpdemux.so* $PREFIX/lib/libwebpdecoder.so* + +# FIX +RUN for i in $PREFIX/bin/*; do patchelf --force-rpath --set-rpath '$ORIGIN/../lib' $i; done +RUN for i in $PREFIX/lib/*.so; do patchelf --force-rpath --set-rpath '$ORIGIN' $i; done + +# Build final image +FROM public.ecr.aws/lambda/provided:al2023 AS runner + +ENV PREFIX /opt +COPY --from=builder /opt/lib/ $PREFIX/lib/ +COPY --from=builder /opt/include/ $PREFIX/include/ +COPY --from=builder /opt/share/ $PREFIX/share/ +COPY --from=builder /opt/bin/ $PREFIX/bin/ + +RUN export GDAL_VERSION=$(gdal-config --version) + +ENV \ + GDAL_DATA=$PREFIX/share/gdal \ + PROJ_LIB=$PREFIX/share/proj \ + GDAL_CONFIG=$PREFIX/bin/gdal-config \ + GEOS_CONFIG=$PREFIX/bin/geos-config \ + PATH=$PREFIX/bin:$PATH + +ENTRYPOINT bash From 3d3b3ad1008418d45eb9c09af11adca97de60f52 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Sun, 14 Dec 2025 19:56:11 +0000 Subject: [PATCH 02/12] Bump nghttp2 version to 1.68.0 This fixes: curl: symbol lookup error: /lib64/libcurl.so.4: undefined symbol: nghttp2_option_set_no_rfc9113_leading_and_trailing_ws_validation --- dockerfiles/Dockerfile.al2023 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 index 696b786..3a9deff 100644 --- a/dockerfiles/Dockerfile.al2023 +++ b/dockerfiles/Dockerfile.al2023 @@ -59,7 +59,7 @@ RUN mkdir /tmp/libxml2 \ && rm -rf /tmp/libxml2 # nghttp2 -ENV NGHTTP2_VERSION=1.42.0 +ENV NGHTTP2_VERSION=1.68.0 RUN mkdir /tmp/nghttp2 \ && curl -sfL https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz | tar zxf - -C /tmp/nghttp2 --strip-components=1 \ && cd /tmp/nghttp2 \ From 917dc0916df0035c650ac23fb824bdfc2a6fa92e Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 11:25:35 +0000 Subject: [PATCH 03/12] Install libtrc and copy *.so files This is for the hdf4 lib --- dockerfiles/Dockerfile.al2023 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 index 3a9deff..0fb433a 100644 --- a/dockerfiles/Dockerfile.al2023 +++ b/dockerfiles/Dockerfile.al2023 @@ -6,7 +6,7 @@ LABEL maintainer="lambgeo " # cmake is available directly (not cmake3) RUN dnf update -y && \ dnf install -y git autoconf libtool flex bison cmake make \ - tar gzip gcc gcc-c++ automake libpng-devel nasm \ + tar gzip gcc gcc-c++ automake libpng-devel nasm libtirpc-devel \ libxml2-devel readline-devel openssl-devel curl-devel && \ dnf clean all && \ rm -rf /var/cache/dnf @@ -403,6 +403,9 @@ RUN rm -f $PREFIX/lib/libturbojpeg.so* \ RUN for i in $PREFIX/bin/*; do patchelf --force-rpath --set-rpath '$ORIGIN/../lib' $i; done RUN for i in $PREFIX/lib/*.so; do patchelf --force-rpath --set-rpath '$ORIGIN' $i; done +# hdf4 needs libtrc headers so copy these over +RUN cp /usr/lib64/libtirpc.so* $PREFIX/lib/ + # Build final image FROM public.ecr.aws/lambda/provided:al2023 AS runner From 30ac5514b50b44ff53a483aa8b59a60a913fc9ff Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 15:04:59 +0000 Subject: [PATCH 04/12] hdf4 libtrc build flags --- dockerfiles/Dockerfile.al2023 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 index 0fb433a..c2729af 100644 --- a/dockerfiles/Dockerfile.al2023 +++ b/dockerfiles/Dockerfile.al2023 @@ -284,7 +284,7 @@ ENV HDF4_VERSION=4.2.15 RUN mkdir /tmp/hdf4 \ && curl -sfL https://support.hdfgroup.org/ftp/HDF/releases/HDF${HDF4_VERSION}/src/hdf-${HDF4_VERSION}.tar | tar xvf - -C /tmp/hdf4 --strip-components=1 \ && cd /tmp/hdf4 \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" CPPFLAGS="-I/usr/include/tirpc" LIBS="-ltirpc" ./configure \ --prefix=$PREFIX \ --with-szlib=$PREFIX \ --with-jpeg=$PREFIX \ From d995e6da596134f5f6a1645bc5992a9638204a0f Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 09:46:31 +0000 Subject: [PATCH 05/12] Build brotli brotli is available systemwide on al2023 so all the packages are linking against it. --- dockerfiles/Dockerfile.al2023 | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 index c2729af..b7b312f 100644 --- a/dockerfiles/Dockerfile.al2023 +++ b/dockerfiles/Dockerfile.al2023 @@ -335,15 +335,21 @@ RUN mkdir /tmp/netcdf \ # && make -j $(nproc) install && make clean \ # && rm -rf /tmp/libhwy -# # brotli -# ENV BROTLI_VERSION=1.0.9 -# RUN mkdir /tmp/brotli \ -# && curl -sfL https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz | tar zxf - -C /tmp/brotli --strip-components=1 \ -# && cd /tmp/brotli \ -# && mkdir build && cd build \ -# && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ../configure-cmake --prefix=$PREFIX \ -# && make -j $(nproc) --silent && make install && make clean \ -# && rm -rf /tmp/brotli +# brotli +ENV BROTLI_VERSION=1.2.0 +RUN mkdir /tmp/brotli \ + && curl -sfL https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz | tar zxf - -C /tmp/brotli --strip-components=1 \ + && cd /tmp/brotli \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && cmake --build . -j $(nproc) \ + && cmake --build . --target install \ + && rm -rf /tmp/brotli # # # JpegXL # ENV JXL_VERSION=0.8.1 From 231522cd604f6c767c068c38b3d5e249f5ac1b71 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 10:37:12 +0000 Subject: [PATCH 06/12] Change tests for gdalwarp libsql based on command gdalwarp is using so file in lib not bin. docker run --platform=linux/amd64 --entrypoint bash \ --rm ghcr.io/lambgeo/lambda-gdal:3.12-al2023 \ -c "ldd /opt/bin/gdalwarp | grep -i sqlite" libsqlite3.so.0 => /opt/lib/libsqlite3.so.0 (0x0000752f34cca000) --- tests/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.sh b/tests/tests.sh index 8870528..24eca33 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -21,7 +21,7 @@ if [[ ! "$(ogrinfo --formats | grep 'DXF')" ]]; then echo "DXF NOK" && exit 1; f echo "OK" echo "Checking sqlite build" -if [[ ! "$(ldd $PREFIX/bin/gdalwarp | grep '/opt/bin/../lib/libsqlite3')" ]]; then echo "gdalwarp libsql NOK" && exit 1; fi +if [[ ! "$(ldd $PREFIX/bin/gdalwarp | grep -E '/opt/(lib|bin/\.\./lib)/libsqlite3')" ]]; then echo "gdalwarp libsql NOK" && exit 1; fi if [[ ! "$(ldd $PREFIX/lib/libgdal.so | grep '/opt/lib/libsqlite3')" ]]; then echo "libgdal libsql NOK" && exit 1; fi if [[ ! "$(ldd $PREFIX/lib/libproj.so | grep '/opt/lib/libsqlite3')" ]]; then echo "libproj libsql NOK" && exit 1; fi if [[ ! "$(ldd $PREFIX/lib/libgeotiff.so | grep '/opt/lib/libsqlite3')" ]]; then echo "libgeotiff libsql NOK" && exit 1; fi From e7d52740dd8a45a54ccd80054de29fae4d19c6b0 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 12:58:26 +0000 Subject: [PATCH 07/12] Parameterise build script Bump numpy to 2.x while we're at it. --- scripts/build.sh | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 55f9356..7a97a61 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,18 +4,34 @@ GDAL_VERSION=$1 GDAL_VERSION_TAG=${GDAL_VERSION%.*} RUNTIME=$2 RUNTIME_VERSION=$3 +BASE_OS=${4:-al2} # Optional: al2 (default) or al2023 -echo "Building image for AWS Lambda | GDAL: ${GDAL_VERSION} | Runtime: ${RUNTIME}:${RUNTIME_VERSION}" +# Select Dockerfile and package installer based on base OS +if [ "$BASE_OS" = "al2023" ]; then + DOCKERFILE="dockerfiles/Dockerfile.al2023" + TAG_SUFFIX="-al2023" + PKG_INSTALLER="dnf" + NUMPY_VERSION="2.3.5" +else + DOCKERFILE="dockerfiles/Dockerfile" + TAG_SUFFIX="" + PKG_INSTALLER="yum" + NUMPY_VERSION="1.25" +fi + +echo "Building image for AWS Lambda | GDAL: ${GDAL_VERSION} | Runtime: ${RUNTIME}:${RUNTIME_VERSION} | Base: ${BASE_OS}" docker buildx build \ --platform=linux/amd64 \ --build-arg GDAL_VERSION=${GDAL_VERSION} \ - -f dockerfiles/Dockerfile \ - -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG} . + -f ${DOCKERFILE} \ + -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}${TAG_SUFFIX} . docker buildx build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION_TAG=${GDAL_VERSION_TAG} \ + --build-arg GDAL_VERSION_TAG=${GDAL_VERSION_TAG}${TAG_SUFFIX} \ --build-arg RUNTIME_VERSION=${RUNTIME_VERSION} \ + --build-arg PKG_INSTALLER=${PKG_INSTALLER} \ + --build-arg NUMPY_VERSION=${NUMPY_VERSION} \ -f dockerfiles/runtimes/${RUNTIME} \ - -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}-${RUNTIME}${RUNTIME_VERSION} . + -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}${TAG_SUFFIX}-${RUNTIME}${RUNTIME_VERSION} . From a3ab453435b3f5b308feed642eb30935d37cd145 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 13:41:07 +0000 Subject: [PATCH 08/12] Update CI to build AL2 and AL2023 images - Build both Dockerfile (AL2) and Dockerfile.al2023 (AL2023) - AL2 images still use GDAL 3.8.3 - AL2023 images use GDAL 3.11.5 - Drop Python 3.9 support - Add Python 3.14 support - Python 3.10-3.11 runtimes built against AL2 base (GDAL 3.8) - Python 3.12-3.14 runtimes built against AL2023 base (GDAL 3.11) - Separate layer deployment for AL2 (tag: 3.8) and AL2023 (tag: 3.11-al2023) - Update numpy to 2.3.5 for AL2023 Python runtimes --- .github/workflows/ci.yml | 102 ++++++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e39b7b..73ffc98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,10 @@ on: pull_request: env: - GDAL_VERSION: 3.8.3 - GDAL_VERSION_TAG: 3.8 + GDAL_VERSION_AL2: 3.8.3 + GDAL_VERSION_TAG_AL2: 3.8 + GDAL_VERSION_AL2023: 3.11.5 + GDAL_VERSION_TAG_AL2023: 3.11 jobs: build: @@ -45,7 +47,7 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build + - name: Build AL2 uses: docker/build-push-action@v6 with: platforms: linux/amd64 @@ -53,18 +55,40 @@ jobs: load: true file: dockerfiles/Dockerfile build-args: | - GDAL_VERSION=${{ env.GDAL_VERSION }} - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max + GDAL_VERSION=${{ env.GDAL_VERSION_AL2 }} + tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 + cache-from: type=gha,scope=al2 + cache-to: type=gha,mode=max,scope=al2 - - name: Test + - name: Build AL2023 + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + load: true + file: dockerfiles/Dockerfile.al2023 + build-args: | + GDAL_VERSION=${{ env.GDAL_VERSION_AL2023 }} + tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 + cache-from: type=gha,scope=al2023 + cache-to: type=gha,mode=max,scope=al2023 + + - name: Test AL2 + run: | + docker run \ + --platform=linux/amd64 \ + --entrypoint bash \ + -v ${{ github.workspace }}:/local \ + --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + /local/tests/tests.sh + + - name: Test AL2023 run: | docker run \ --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} \ + --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ /local/tests/tests.sh deploy: @@ -96,57 +120,81 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Publish docker image + - name: Publish AL2 docker image uses: docker/build-push-action@v6 with: platforms: linux/amd64 context: . file: dockerfiles/Dockerfile build-args: | - GDAL_VERSION=${{ env.GDAL_VERSION }} + GDAL_VERSION=${{ env.GDAL_VERSION_AL2 }} push: true - cache-from: type=gha - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} + cache-from: type=gha,scope=al2 + tags: | + ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 + ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }} - - name: Build and Deploy layers + - name: Publish AL2023 docker image + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + file: dockerfiles/Dockerfile.al2023 + build-args: | + GDAL_VERSION=${{ env.GDAL_VERSION_AL2023 }} + push: true + cache-from: type=gha,scope=al2023 + tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 + + - name: Build and Deploy layers (AL2) run: | python -m pip install boto3 click docker run \ --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} \ + --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + /local/scripts/create-layer.sh + python scripts/deploy.py ${{ env.GDAL_VERSION_TAG_AL2 }} --deploy + + - name: Build and Deploy layers (AL2023) + run: | + docker run \ + --platform=linux/amd64 \ + --entrypoint bash \ + -v ${{ github.workspace }}:/local \ + --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ /local/scripts/create-layer.sh - python scripts/deploy.py ${{ env.GDAL_VERSION_TAG }} --deploy + python scripts/deploy.py ${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 --deploy - - name: Build Runtime and Push Python <=3.11 with yum + - name: Build Runtime and Push Python 3.10-3.11 (AL2) run: | - runtimes='3.9 3.10 3.11' + runtimes='3.10 3.11' for runtime in ${runtimes}; do docker build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG }} \ + --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ --build-arg RUNTIME_VERSION=${runtime} \ --build-arg NUMPY_VERSION=1.25 \ --build-arg PKG_INSTALLER=yum \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} + -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} . + docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} done - - name: Build Runtime and Push Python >= 3.12 with dnf + - name: Build Runtime and Push Python 3.12-3.14 (AL2023) run: | - runtimes='3.12 3.13' + runtimes='3.12 3.13 3.14' for runtime in ${runtimes}; do docker build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG }} \ + --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ --build-arg RUNTIME_VERSION=${runtime} \ - --build-arg NUMPY_VERSION=1.26 \ + --build-arg NUMPY_VERSION=2.3.5 \ --build-arg PKG_INSTALLER=dnf \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} + -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} . + docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} done update-layer: From 76967d9fa608bfa08a56903e53f64d9e8ec2ffee Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 13:56:04 +0000 Subject: [PATCH 09/12] /hack/ deal with uppercase in repo owner name --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73ffc98..5c3f799 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set registry owner + run: | + owner="${{ github.repository_owner }}" + echo "REGISTRY_OWNER=${owner,,}" >> $GITHUB_ENV + - name: Set up Python uses: actions/setup-python@v5 with: @@ -56,7 +61,7 @@ jobs: file: dockerfiles/Dockerfile build-args: | GDAL_VERSION=${{ env.GDAL_VERSION_AL2 }} - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 + tags: ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 cache-from: type=gha,scope=al2 cache-to: type=gha,mode=max,scope=al2 @@ -69,7 +74,7 @@ jobs: file: dockerfiles/Dockerfile.al2023 build-args: | GDAL_VERSION=${{ env.GDAL_VERSION_AL2023 }} - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 + tags: ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 cache-from: type=gha,scope=al2023 cache-to: type=gha,mode=max,scope=al2023 @@ -79,7 +84,7 @@ jobs: --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ /local/tests/tests.sh - name: Test AL2023 @@ -88,7 +93,7 @@ jobs: --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ /local/tests/tests.sh deploy: @@ -98,6 +103,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set registry owner + run: | + owner="${{ github.repository_owner }}" + echo "REGISTRY_OWNER=${owner,,}" >> $GITHUB_ENV + - name: Set up Python uses: actions/setup-python@v5 with: @@ -131,8 +141,8 @@ jobs: push: true cache-from: type=gha,scope=al2 tags: | - ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 - ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }} + ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 + ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }} - name: Publish AL2023 docker image uses: docker/build-push-action@v6 @@ -144,7 +154,7 @@ jobs: GDAL_VERSION=${{ env.GDAL_VERSION_AL2023 }} push: true cache-from: type=gha,scope=al2023 - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 + tags: ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 - name: Build and Deploy layers (AL2) run: | @@ -153,7 +163,7 @@ jobs: --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ /local/scripts/create-layer.sh python scripts/deploy.py ${{ env.GDAL_VERSION_TAG_AL2 }} --deploy @@ -163,7 +173,7 @@ jobs: --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ /local/scripts/create-layer.sh python scripts/deploy.py ${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 --deploy @@ -178,8 +188,8 @@ jobs: --build-arg NUMPY_VERSION=1.25 \ --build-arg PKG_INSTALLER=yum \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} + -t ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} . + docker push ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} done - name: Build Runtime and Push Python 3.12-3.14 (AL2023) @@ -193,8 +203,8 @@ jobs: --build-arg NUMPY_VERSION=2.3.5 \ --build-arg PKG_INSTALLER=dnf \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} + -t ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} . + docker push ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} done update-layer: From d07e9aee3435ee06080f069e6bec649bcef803c9 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 14:41:27 +0000 Subject: [PATCH 10/12] Run ci builds in parallel --- .github/workflows/ci.yml | 53 +++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c3f799..da60604 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ env: GDAL_VERSION_TAG_AL2023: 3.11 jobs: - build: + build-al2: runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, '[skip ci]')" steps: @@ -65,6 +65,46 @@ jobs: cache-from: type=gha,scope=al2 cache-to: type=gha,mode=max,scope=al2 + - name: Test AL2 + run: | + docker run \ + --platform=linux/amd64 \ + --entrypoint bash \ + -v ${{ github.workspace }}:/local \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + /local/tests/tests.sh + + build-al2023: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, '[skip ci]')" + steps: + - uses: actions/checkout@v4 + + - name: Set registry owner + run: | + owner="${{ github.repository_owner }}" + echo "REGISTRY_OWNER=${owner,,}" >> $GITHUB_ENV + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install boto3 click + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Github + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build AL2023 uses: docker/build-push-action@v6 with: @@ -78,15 +118,6 @@ jobs: cache-from: type=gha,scope=al2023 cache-to: type=gha,mode=max,scope=al2023 - - name: Test AL2 - run: | - docker run \ - --platform=linux/amd64 \ - --entrypoint bash \ - -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ - /local/tests/tests.sh - - name: Test AL2023 run: | docker run \ @@ -97,7 +128,7 @@ jobs: /local/tests/tests.sh deploy: - needs: [build] + needs: [build-al2, build-al2023] runs-on: ubuntu-latest if: contains(github.ref, 'tags') && github.event_name == 'push' steps: From 13a6a2185544ed79332b50ea3b2975628f9ca73a Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 15:06:34 +0000 Subject: [PATCH 11/12] Silence docker warnings --- dockerfiles/Dockerfile | 6 +++--- dockerfiles/Dockerfile.al2023 | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 5875f15..4381e5f 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -10,11 +10,11 @@ RUN yum update -y && \ yum clean all && \ rm -rf /var/cache/yum /var/lib/yum/history -ENV PREFIX /opt +ENV PREFIX=/opt WORKDIR /opt -ENV LD_LIBRARY_PATH $PREFIX/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH # pkg-config ENV PKGCONFIG_VERSION=0.29.2 @@ -404,7 +404,7 @@ RUN for i in $PREFIX/lib/*.so; do patchelf --force-rpath --set-rpath '$ORIGIN' $ # Build final image FROM public.ecr.aws/lambda/provided:al2 AS runner -ENV PREFIX /opt +ENV PREFIX=/opt COPY --from=builder /opt/lib/ $PREFIX/lib/ COPY --from=builder /opt/include/ $PREFIX/include/ COPY --from=builder /opt/share/ $PREFIX/share/ diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 index b7b312f..4f30c82 100644 --- a/dockerfiles/Dockerfile.al2023 +++ b/dockerfiles/Dockerfile.al2023 @@ -4,6 +4,7 @@ LABEL maintainer="lambgeo " # AL2023 uses dnf instead of yum, and automake instead of automake16 # cmake is available directly (not cmake3) +# libtirpc-devel provides RPC headers needed by HDF4 RUN dnf update -y && \ dnf install -y git autoconf libtool flex bison cmake make \ tar gzip gcc gcc-c++ automake libpng-devel nasm libtirpc-devel \ @@ -11,11 +12,11 @@ RUN dnf update -y && \ dnf clean all && \ rm -rf /var/cache/dnf -ENV PREFIX /opt +ENV PREFIX=/opt WORKDIR /opt -ENV LD_LIBRARY_PATH $PREFIX/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH # pkg-config ENV PKGCONFIG_VERSION=0.29.2 @@ -415,7 +416,7 @@ RUN cp /usr/lib64/libtirpc.so* $PREFIX/lib/ # Build final image FROM public.ecr.aws/lambda/provided:al2023 AS runner -ENV PREFIX /opt +ENV PREFIX=/opt COPY --from=builder /opt/lib/ $PREFIX/lib/ COPY --from=builder /opt/include/ $PREFIX/include/ COPY --from=builder /opt/share/ $PREFIX/share/ From 635b19426c793315b452822077455f1f8446d5fa Mon Sep 17 00:00:00 2001 From: GeoWill Date: Mon, 15 Dec 2025 16:31:15 +0000 Subject: [PATCH 12/12] Update create-layer to work with yum or dnf --- scripts/create-layer.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/create-layer.sh b/scripts/create-layer.sh index 8d06ba1..ff0de8f 100755 --- a/scripts/create-layer.sh +++ b/scripts/create-layer.sh @@ -3,7 +3,12 @@ echo "-----------------------" echo "Creating lambda layer" echo "-----------------------" -yum install -y zip binutils +# Use dnf if available (AL2023), otherwise yum (AL2) +if command -v dnf &> /dev/null; then + dnf install -y zip binutils findutils +else + yum install -y zip binutils +fi echo "Remove useless files" rm -rdf $PREFIX/share/doc \