diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e39b7b..da60604 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,16 +15,23 @@ on: pull_request: env: - GDAL_VERSION: 3.8.3 - GDAL_VERSION_TAG: 3.8 + GDAL_VERSION_AL2: 3.8.3 + GDAL_VERSION_TAG_AL2: 3.8 + GDAL_VERSION_AL2023: 3.11.5 + GDAL_VERSION_TAG_AL2023: 3.11 jobs: - build: + build-al2: runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, '[skip ci]')" steps: - uses: actions/checkout@v4 + - name: Set registry owner + run: | + owner="${{ github.repository_owner }}" + echo "REGISTRY_OWNER=${owner,,}" >> $GITHUB_ENV + - name: Set up Python uses: actions/setup-python@v5 with: @@ -45,7 +52,7 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build + - name: Build AL2 uses: docker/build-push-action@v6 with: platforms: linux/amd64 @@ -53,27 +60,85 @@ jobs: load: true file: dockerfiles/Dockerfile build-args: | - GDAL_VERSION=${{ env.GDAL_VERSION }} - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max + GDAL_VERSION=${{ env.GDAL_VERSION_AL2 }} + tags: ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 + cache-from: type=gha,scope=al2 + cache-to: type=gha,mode=max,scope=al2 - - name: Test + - name: Test AL2 run: | docker run \ --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + /local/tests/tests.sh + + build-al2023: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, '[skip ci]')" + steps: + - uses: actions/checkout@v4 + + - name: Set registry owner + run: | + owner="${{ github.repository_owner }}" + echo "REGISTRY_OWNER=${owner,,}" >> $GITHUB_ENV + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install boto3 click + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Github + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build AL2023 + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + load: true + file: dockerfiles/Dockerfile.al2023 + build-args: | + GDAL_VERSION=${{ env.GDAL_VERSION_AL2023 }} + tags: ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 + cache-from: type=gha,scope=al2023 + cache-to: type=gha,mode=max,scope=al2023 + + - name: Test AL2023 + run: | + docker run \ + --platform=linux/amd64 \ + --entrypoint bash \ + -v ${{ github.workspace }}:/local \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ /local/tests/tests.sh deploy: - needs: [build] + needs: [build-al2, build-al2023] runs-on: ubuntu-latest if: contains(github.ref, 'tags') && github.event_name == 'push' steps: - uses: actions/checkout@v4 + - name: Set registry owner + run: | + owner="${{ github.repository_owner }}" + echo "REGISTRY_OWNER=${owner,,}" >> $GITHUB_ENV + - name: Set up Python uses: actions/setup-python@v5 with: @@ -96,57 +161,81 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Publish docker image + - name: Publish AL2 docker image uses: docker/build-push-action@v6 with: platforms: linux/amd64 context: . file: dockerfiles/Dockerfile build-args: | - GDAL_VERSION=${{ env.GDAL_VERSION }} + GDAL_VERSION=${{ env.GDAL_VERSION_AL2 }} push: true - cache-from: type=gha - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} + cache-from: type=gha,scope=al2 + tags: | + ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 + ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }} - - name: Build and Deploy layers + - name: Publish AL2023 docker image + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + file: dockerfiles/Dockerfile.al2023 + build-args: | + GDAL_VERSION=${{ env.GDAL_VERSION_AL2023 }} + push: true + cache-from: type=gha,scope=al2023 + tags: ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 + + - name: Build and Deploy layers (AL2) run: | python -m pip install boto3 click docker run \ --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ + /local/scripts/create-layer.sh + python scripts/deploy.py ${{ env.GDAL_VERSION_TAG_AL2 }} --deploy + + - name: Build and Deploy layers (AL2023) + run: | + docker run \ + --platform=linux/amd64 \ + --entrypoint bash \ + -v ${{ github.workspace }}:/local \ + --rm ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ /local/scripts/create-layer.sh - python scripts/deploy.py ${{ env.GDAL_VERSION_TAG }} --deploy + python scripts/deploy.py ${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 --deploy - - name: Build Runtime and Push Python <=3.11 with yum + - name: Build Runtime and Push Python 3.10-3.11 (AL2) run: | - runtimes='3.9 3.10 3.11' + runtimes='3.10 3.11' for runtime in ${runtimes}; do docker build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG }} \ + --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG_AL2 }}-al2 \ --build-arg RUNTIME_VERSION=${runtime} \ --build-arg NUMPY_VERSION=1.25 \ --build-arg PKG_INSTALLER=yum \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} + -t ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} . + docker push ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2 }}-python${runtime} done - - name: Build Runtime and Push Python >= 3.12 with dnf + - name: Build Runtime and Push Python 3.12-3.14 (AL2023) run: | - runtimes='3.12 3.13' + runtimes='3.12 3.13 3.14' for runtime in ${runtimes}; do docker build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG }} \ + --build-arg GDAL_VERSION_TAG=${{ env.GDAL_VERSION_TAG_AL2023 }}-al2023 \ --build-arg RUNTIME_VERSION=${runtime} \ - --build-arg NUMPY_VERSION=1.26 \ + --build-arg NUMPY_VERSION=2.3.5 \ --build-arg PKG_INSTALLER=dnf \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} + -t ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} . + docker push ghcr.io/${{ env.REGISTRY_OWNER }}/lambda-gdal:${{ env.GDAL_VERSION_TAG_AL2023 }}-python${runtime} done update-layer: diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 5875f15..4381e5f 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -10,11 +10,11 @@ RUN yum update -y && \ yum clean all && \ rm -rf /var/cache/yum /var/lib/yum/history -ENV PREFIX /opt +ENV PREFIX=/opt WORKDIR /opt -ENV LD_LIBRARY_PATH $PREFIX/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH # pkg-config ENV PKGCONFIG_VERSION=0.29.2 @@ -404,7 +404,7 @@ RUN for i in $PREFIX/lib/*.so; do patchelf --force-rpath --set-rpath '$ORIGIN' $ # Build final image FROM public.ecr.aws/lambda/provided:al2 AS runner -ENV PREFIX /opt +ENV PREFIX=/opt COPY --from=builder /opt/lib/ $PREFIX/lib/ COPY --from=builder /opt/include/ $PREFIX/include/ COPY --from=builder /opt/share/ $PREFIX/share/ diff --git a/dockerfiles/Dockerfile.al2023 b/dockerfiles/Dockerfile.al2023 new file mode 100644 index 0000000..4f30c82 --- /dev/null +++ b/dockerfiles/Dockerfile.al2023 @@ -0,0 +1,434 @@ +FROM public.ecr.aws/lambda/provided:al2023 AS builder + +LABEL maintainer="lambgeo " + +# AL2023 uses dnf instead of yum, and automake instead of automake16 +# cmake is available directly (not cmake3) +# libtirpc-devel provides RPC headers needed by HDF4 +RUN dnf update -y && \ + dnf install -y git autoconf libtool flex bison cmake make \ + tar gzip gcc gcc-c++ automake libpng-devel nasm libtirpc-devel \ + libxml2-devel readline-devel openssl-devel curl-devel && \ + dnf clean all && \ + rm -rf /var/cache/dnf + +ENV PREFIX=/opt + +WORKDIR /opt + +ENV LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH + +# pkg-config +ENV PKGCONFIG_VERSION=0.29.2 +RUN mkdir /tmp/pkg-config \ + && curl -sfL https://pkg-config.freedesktop.org/releases/pkg-config-${PKGCONFIG_VERSION}.tar.gz | tar zxf - -C /tmp/pkg-config --strip-components=1 \ + && cd /tmp/pkg-config \ + && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-internal-glib \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/pkg-config + +ENV PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig/ + +# sqlite +RUN mkdir /tmp/sqlite \ + && curl -sfL https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz | tar zxf - -C /tmp/sqlite --strip-components=1 \ + && cd /tmp/sqlite \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/sqlite + +# libexpat +RUN mkdir /tmp/libexpat \ + && curl -sfL https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.gz | tar zxf - -C /tmp/libexpat --strip-components=1 \ + && cd /tmp/libexpat \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/libexpat + +ENV \ + SQLITE3_LIBS="-L${PREFIX}/lib -lsqlite3" \ + SQLITE3_INCLUDE_DIR="${PREFIX}/include" \ + SQLITE3_CFLAGS="$CFLAGS -I${PREFIX}/include" \ + PATH=${PREFIX}/bin/:$PATH + +# libxml2 +RUN mkdir /tmp/libxml2 \ + && curl -sfL https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.10/libxml2-v2.9.10.tar.gz | tar zxf - -C /tmp/libxml2 --strip-components=1 \ + && cd /tmp/libxml2 \ + && ./autogen.sh --prefix=$PREFIX --with-ftp=no --with-http=no --with-python=no \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libxml2 + +# nghttp2 +ENV NGHTTP2_VERSION=1.68.0 +RUN mkdir /tmp/nghttp2 \ + && curl -sfL https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz | tar zxf - -C /tmp/nghttp2 --strip-components=1 \ + && cd /tmp/nghttp2 \ + && ./configure --enable-lib-only --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/nghttp2 + +# libcurl +ENV CURL_VERSION=7.73.0 +RUN mkdir /tmp/libcurl \ + && curl -sfL https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz | tar zxf - -C /tmp/libcurl --strip-components=1 \ + && cd /tmp/libcurl \ + && ./configure --disable-manual --disable-cookies --with-nghttp2=$PREFIX --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libcurl + +# libdeflate +RUN mkdir /tmp/libdeflate \ + && curl -sfL https://github.com/ebiggers/libdeflate/archive/v1.10.tar.gz | tar zxf - -C /tmp/libdeflate --strip-components=1 \ + && cd /tmp/libdeflate \ + && make -j $(nproc) --silent PREFIX=$PREFIX && make install \ + && rm -rf /tmp/libdeflate + +# libpng +ENV LIBPNG_VERSION=1.6.37 +RUN mkdir /tmp/png \ + && curl -sfL https://github.com/glennrp/libpng/archive/v${LIBPNG_VERSION}.tar.gz | tar zxf - -C /tmp/png --strip-components=1 \ + && cd /tmp/png \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/png + +# libjpeg_turbo +ENV LIBJPEG_TURBO_VERSION=2.1.3 +RUN mkdir /tmp/jpeg \ + && curl -sfL https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${LIBJPEG_TURBO_VERSION}.tar.gz | tar zxf - -C /tmp/jpeg --strip-components=1 \ + && cd /tmp/jpeg \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/jpeg + +# webp +ENV WEBP_VERSION=1.1.0 +RUN mkdir /tmp/webp \ + && curl -sfL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | tar zxf - -C /tmp/webp --strip-components=1 \ + && cd /tmp/webp \ + && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/webp + +# zstd +ENV ZSTD_VERSION=1.4.5 +RUN mkdir /tmp/zstd \ + && curl -sfL https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz | tar zxf - -C /tmp/zstd --strip-components=1 \ + && cd /tmp/zstd \ + && make -j $(nproc) PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 --silent && make install PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 \ + && rm -rf /tmp/zstd + +# lerc +ENV LERC_VERSION=2.2.1 +RUN mkdir /tmp/lerc \ + && curl -sfL https://github.com/Esri/lerc/archive/v${LERC_VERSION}.tar.gz | tar zxf - -C /tmp/lerc --strip-components=1 \ + && cd /tmp/lerc \ + && cmake . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/lerc + +# libtiff +ENV LIBTIFF_VERSION=4.5.0 +RUN mkdir /tmp/libtiff \ + && curl -sfL https://download.osgeo.org/libtiff/tiff-${LIBTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libtiff --strip-components=1 \ + && cd /tmp/libtiff \ + && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --disable-static \ + --enable-rpath \ + --with-lerc-lib-dir=$PREFIX/lib \ + --with-lerc-include-dir=$PREFIX/include \ + --with-libdeflate-lib-dir=$PREFIX/lib \ + --with-libdeflate-include-dir=$PREFIX/include \ + --with-jpeg-lib-dir=$PREFIX/lib \ + --with-jpeg-include-dir=$PREFIX/include \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libtiff + +# postgres +ENV PG_VERSION=15.2 +RUN mkdir /tmp/postgres \ + && curl -sfL https://ftp.postgresql.org/pub/source/v${PG_VERSION}/postgresql-${PG_VERSION}.tar.gz | tar zxf - -C /tmp/postgres --strip-components=1 \ + && cd /tmp/postgres \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-openssl \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/postgres + +# openjpeg +ENV OPENJPEG_VERSION=2.5.0 +RUN mkdir /tmp/openjpeg \ + && curl -sfL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | tar zxf - -C /tmp/openjpeg --strip-components=1 \ + && cd /tmp/openjpeg \ + && mkdir build && cd build \ + && cmake .. \ + -DBUILD_THIRDPARTY:BOOL=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/openjpeg + +# geos +ENV GEOS_VERSION=3.12.1 +RUN mkdir /tmp/geos \ + && curl -sfL https://github.com/libgeos/geos/archive/refs/tags/${GEOS_VERSION}.tar.gz | tar zxf - -C /tmp/geos --strip-components=1 \ + && cd /tmp/geos \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/geos + +# proj +# We still provide proj-datumgrid but the user can choose to set `PROJ_NETWORK=ON` to use network grids +# ENV PROJ_VERSION=9.2.0 +# RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ +# && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ +# && curl -sfL http://download.osgeo.org/proj/proj-datumgrid-latest.tar.gz | tar zxf - -C /tmp/proj/data \ +# && cd /tmp/proj \ +# && aclocal && autoheader && libtoolize --force --copy && automake --add-missing --copy && autoconf \ +# && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static --enable-lto \ +# && make -j $(nproc) --silent && make install \ +# && rm -rf /tmp/proj + +ENV PROJ_VERSION=9.3.1 +RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ + && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ + && cd /tmp/proj \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_INSTALL_INCLUDEDIR:PATH=include \ + -DBUILD_TESTING=OFF \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/proj + +# GeoTIFF +ENV LIBGEOTIFF_VERSION=1.7.1 +RUN mkdir /tmp/libgeotiff \ + && curl -sfL https://github.com/OSGeo/libgeotiff/releases/download/${LIBGEOTIFF_VERSION}/libgeotiff-${LIBGEOTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libgeotiff --strip-components=1 \ + && cd /tmp/libgeotiff \ + && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-proj=$PREFIX \ + --with-jpeg=$PREFIX \ + --with-zip=yes \ + --with-zlib \ + --with-libtiff=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libgeotiff + +# Build blosc (needed for ZARR) +ENV BLOSC_VERSION=1.21.0 +RUN mkdir /tmp/blosc \ + && curl -sfL https://github.com/Blosc/c-blosc/archive/refs/tags/v${BLOSC_VERSION}.tar.gz | tar zxf - -C /tmp/blosc --strip-components=1 \ + && cd /tmp/blosc \ + && cmake -G"Unix Makefiles" \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_INSTALL_RPATH="$ORIGIN" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DBUILD_SHARED=ON \ + -DBUILD_STATIC=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_TESTS=OFF \ + -DPREFER_EXTERNAL_LZ4=ON \ + -DPREFER_EXTERNAL_ZLIB=ON \ + -DPREFER_EXTERNAL_ZSTD=ON \ + -DDEACTIVATE_SNAPPY=OFF . \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/blosc + +# libcrypto (for GPC access) +RUN mkdir /tmp/cryptopp \ + && curl -sfL https://github.com/weidai11/cryptopp/archive/refs/tags/CRYPTOPP_8_6_0.tar.gz | tar zxf - -C /tmp/cryptopp --strip-components=1 \ + && cd /tmp/cryptopp \ + && make -j $(nproc) --silent && make install PREFIX=$PREFIX \ + && rm -rf /tmp/cryptopp + +# szip (for hdf) +ENV SZIP_VERSION=2.1.1 +RUN mkdir /tmp/szip \ + && curl -sfL https://support.hdfgroup.org/ftp/lib-external/szip/${SZIP_VERSION}/src/szip-${SZIP_VERSION}.tar.gz | tar zxf - -C /tmp/szip --strip-components=1 \ + && cd /tmp/szip \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/szip + +# libhdf4 +ENV HDF4_VERSION=4.2.15 +RUN mkdir /tmp/hdf4 \ + && curl -sfL https://support.hdfgroup.org/ftp/HDF/releases/HDF${HDF4_VERSION}/src/hdf-${HDF4_VERSION}.tar | tar xvf - -C /tmp/hdf4 --strip-components=1 \ + && cd /tmp/hdf4 \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" CPPFLAGS="-I/usr/include/tirpc" LIBS="-ltirpc" ./configure \ + --prefix=$PREFIX \ + --with-szlib=$PREFIX \ + --with-jpeg=$PREFIX \ + --enable-shared \ + --disable-static \ + --disable-netcdf \ + --disable-fortran \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/hdf4 + +# libhdf5 +ENV HDF5_VERSION=1.12.0 +RUN mkdir /tmp/hdf5 \ + && curl -sfL https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-${HDF5_VERSION}/src/hdf5-$HDF5_VERSION.tar.gz | tar zxf - -C /tmp/hdf5 --strip-components=1 \ + && cd /tmp/hdf5 \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-szlib=$PREFIX \ + --enable-cxx \ + --enable-thread-safe \ + --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/hdf5 + +# NetCDF +ENV NETCDF_VERSION=4.7.4 +RUN mkdir /tmp/netcdf \ + && curl -sfL https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz | tar zxf - -C /tmp/netcdf --strip-components=1 \ + && cd /tmp/netcdf \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" CPPFLAGS="-I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib" ./configure \ + --with-default-chunk-size=67108864 \ + --with-chunk-cache-size=67108864 \ + --prefix=$PREFIX \ + --disable-static \ + --enable-netcdf4 \ + --enable-dap \ + --with-pic \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/netcdf + +# # JPEG-XL +# # libhwy +# RUN mkdir /tmp/libhwy \ +# && curl -sfL https://github.com/google/highway/archive/refs/tags/0.16.0.tar.gz | tar zxf - -C /tmp/libhwy --strip-components=1 \ +# && cd /tmp/libhwy \ +# && mkdir build && cd build \ +# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ +# && make -j $(nproc) install && make clean \ +# && rm -rf /tmp/libhwy + +# brotli +ENV BROTLI_VERSION=1.2.0 +RUN mkdir /tmp/brotli \ + && curl -sfL https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz | tar zxf - -C /tmp/brotli --strip-components=1 \ + && cd /tmp/brotli \ + && mkdir build && cd build \ + && cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && cmake --build . -j $(nproc) \ + && cmake --build . --target install \ + && rm -rf /tmp/brotli + +# # # JpegXL +# ENV JXL_VERSION=0.8.1 +# RUN mkdir /tmp/jxl \ +# && curl -sfL https://github.com/libjxl/libjxl/archive/refs/tags/v${JXL_VERSION}.tar.gz | tar zxf - -C /tmp/jxl --strip-components=1 \ +# && cd /tmp/jxl \ +# && mkdir build && cd build \ +# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ +# && make -j $(nproc) install && make clean \ +# && rm -rf /tmp/jxl + +# We use commit sha to make sure we are not using `cache` when building the docker image +# "7ca88116f5a46d429251361634eb24629f315076" is the latest commit on release/3.6 branch + + +# GDAL +ARG GDAL_VERSION +RUN mkdir /tmp/gdal \ + && curl -sfL https://github.com/OSGeo/gdal/archive/refs/tags/v${GDAL_VERSION}.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 \ + && cd /tmp/gdal \ + && mkdir build && cd build \ + && cmake .. \ + -DGDAL_USE_EXTERNAL_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DGDAL_SET_INSTALL_RELATIVE_RPATH=ON \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=OFF \ + -DGDAL_USE_LERC_INTERNAL=OFF \ + -DLERC_INCLUDE_DIR=$PREFIX/include \ + -DLERC_LIBRARY=$PREFIX/lib/libLercLib.so \ + -DPNG_PNG_INCLUDE_DIR=$PREFIX/include \ + -DPNG_LIBRARY_RELEASE=$PREFIX/lib/libpng.so \ + -DBUILD_PYTHON_BINDINGS=OFF \ + && cmake --build . -j $(nproc) \ + && cmake --build . --target install \ + && rm -rf /tmp/gdal + +# from https://github.com/pypa/manylinux/blob/d8ef5d47433ba771fa4403fd48f352c586e06e43/docker/build_scripts/build.sh#L133-L138 +# Install patchelf (latest with unreleased bug fixes) +ENV PATCHELF_VERSION=0.10 +RUN mkdir /tmp/patchelf \ + && curl -sfL https://github.com/NixOS/patchelf/archive/${PATCHELF_VERSION}.tar.gz | tar zxf - -C /tmp/patchelf --strip-components=1 \ + && cd /tmp/patchelf && ./bootstrap.sh && ./configure \ + && make -j $(nproc) --silent && make install \ + && cd / && rm -rf /tmp/patchelf + +# # libturbojpeg.so is not used by GDAL. Only libjpeg.so* +RUN rm -f $PREFIX/lib/libturbojpeg.so* \ + # Only libwebp.so is used by GDAL + && rm -f $PREFIX/lib/libwebpmux.so* $PREFIX/lib/libwebpdemux.so* $PREFIX/lib/libwebpdecoder.so* + +# FIX +RUN for i in $PREFIX/bin/*; do patchelf --force-rpath --set-rpath '$ORIGIN/../lib' $i; done +RUN for i in $PREFIX/lib/*.so; do patchelf --force-rpath --set-rpath '$ORIGIN' $i; done + +# hdf4 needs libtrc headers so copy these over +RUN cp /usr/lib64/libtirpc.so* $PREFIX/lib/ + +# Build final image +FROM public.ecr.aws/lambda/provided:al2023 AS runner + +ENV PREFIX=/opt +COPY --from=builder /opt/lib/ $PREFIX/lib/ +COPY --from=builder /opt/include/ $PREFIX/include/ +COPY --from=builder /opt/share/ $PREFIX/share/ +COPY --from=builder /opt/bin/ $PREFIX/bin/ + +RUN export GDAL_VERSION=$(gdal-config --version) + +ENV \ + GDAL_DATA=$PREFIX/share/gdal \ + PROJ_LIB=$PREFIX/share/proj \ + GDAL_CONFIG=$PREFIX/bin/gdal-config \ + GEOS_CONFIG=$PREFIX/bin/geos-config \ + PATH=$PREFIX/bin:$PATH + +ENTRYPOINT bash diff --git a/scripts/build.sh b/scripts/build.sh index 55f9356..7a97a61 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,18 +4,34 @@ GDAL_VERSION=$1 GDAL_VERSION_TAG=${GDAL_VERSION%.*} RUNTIME=$2 RUNTIME_VERSION=$3 +BASE_OS=${4:-al2} # Optional: al2 (default) or al2023 -echo "Building image for AWS Lambda | GDAL: ${GDAL_VERSION} | Runtime: ${RUNTIME}:${RUNTIME_VERSION}" +# Select Dockerfile and package installer based on base OS +if [ "$BASE_OS" = "al2023" ]; then + DOCKERFILE="dockerfiles/Dockerfile.al2023" + TAG_SUFFIX="-al2023" + PKG_INSTALLER="dnf" + NUMPY_VERSION="2.3.5" +else + DOCKERFILE="dockerfiles/Dockerfile" + TAG_SUFFIX="" + PKG_INSTALLER="yum" + NUMPY_VERSION="1.25" +fi + +echo "Building image for AWS Lambda | GDAL: ${GDAL_VERSION} | Runtime: ${RUNTIME}:${RUNTIME_VERSION} | Base: ${BASE_OS}" docker buildx build \ --platform=linux/amd64 \ --build-arg GDAL_VERSION=${GDAL_VERSION} \ - -f dockerfiles/Dockerfile \ - -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG} . + -f ${DOCKERFILE} \ + -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}${TAG_SUFFIX} . docker buildx build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION_TAG=${GDAL_VERSION_TAG} \ + --build-arg GDAL_VERSION_TAG=${GDAL_VERSION_TAG}${TAG_SUFFIX} \ --build-arg RUNTIME_VERSION=${RUNTIME_VERSION} \ + --build-arg PKG_INSTALLER=${PKG_INSTALLER} \ + --build-arg NUMPY_VERSION=${NUMPY_VERSION} \ -f dockerfiles/runtimes/${RUNTIME} \ - -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}-${RUNTIME}${RUNTIME_VERSION} . + -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}${TAG_SUFFIX}-${RUNTIME}${RUNTIME_VERSION} . diff --git a/scripts/create-layer.sh b/scripts/create-layer.sh index 8d06ba1..ff0de8f 100755 --- a/scripts/create-layer.sh +++ b/scripts/create-layer.sh @@ -3,7 +3,12 @@ echo "-----------------------" echo "Creating lambda layer" echo "-----------------------" -yum install -y zip binutils +# Use dnf if available (AL2023), otherwise yum (AL2) +if command -v dnf &> /dev/null; then + dnf install -y zip binutils findutils +else + yum install -y zip binutils +fi echo "Remove useless files" rm -rdf $PREFIX/share/doc \ diff --git a/tests/tests.sh b/tests/tests.sh index 8870528..24eca33 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -21,7 +21,7 @@ if [[ ! "$(ogrinfo --formats | grep 'DXF')" ]]; then echo "DXF NOK" && exit 1; f echo "OK" echo "Checking sqlite build" -if [[ ! "$(ldd $PREFIX/bin/gdalwarp | grep '/opt/bin/../lib/libsqlite3')" ]]; then echo "gdalwarp libsql NOK" && exit 1; fi +if [[ ! "$(ldd $PREFIX/bin/gdalwarp | grep -E '/opt/(lib|bin/\.\./lib)/libsqlite3')" ]]; then echo "gdalwarp libsql NOK" && exit 1; fi if [[ ! "$(ldd $PREFIX/lib/libgdal.so | grep '/opt/lib/libsqlite3')" ]]; then echo "libgdal libsql NOK" && exit 1; fi if [[ ! "$(ldd $PREFIX/lib/libproj.so | grep '/opt/lib/libsqlite3')" ]]; then echo "libproj libsql NOK" && exit 1; fi if [[ ! "$(ldd $PREFIX/lib/libgeotiff.so | grep '/opt/lib/libsqlite3')" ]]; then echo "libgeotiff libsql NOK" && exit 1; fi