CTest & GitHub Actions Improvements (#381)

* Run tests in parallel in some `GitHub Actions` Workflows * Except for `macOS` & `Coverage` jobs * Update `Node.js 16` actions to `Node.js 20` versions * `actions/upload-artifact` * `actions/cache` * Add server `IP:Port` in `CTest` HTTPD logs * Make `render_speedtest_*` CTest tests require tiles to be downloaded * Add & use tile download functions for CTest * Including `check` & `remove` "tests" * To remove much of the redundant code * Split up HTTPD0/HTTPD1 downloads in CTest * Remove Amazon Linux 2/CentOS 7 from CI workflow * Comment out fedora:rawhide, it's not currently working * Re-enable CentOS Stream tests
2025-07-23 00:28:34 +00:00 · 2024-01-24 21:45:21 -07:00
parent 4143ab23bd
commit 9408604811
12 changed files with 206 additions and 229 deletions
--- a/.github/actions/cmake/build/action.yml
+++ b/.github/actions/cmake/build/action.yml
@ -2,13 +2,6 @@
 runs:
  using: composite
  steps:
-    - name: Create `cmake` symbolic link
-      run: |
-        if ! command -v cmake &> /dev/null && command -v cmake3 &> /dev/null; then
-          ln --symbolic cmake3 /usr/bin/cmake
-        fi
-      shell: bash --noprofile --norc -euxo pipefail {0}
-
    - name: Prepare `build` directory
      run: |
        cmake -B build -S . \
--- a/.github/actions/cmake/test/action.yml
+++ b/.github/actions/cmake/test/action.yml
@ -6,30 +6,16 @@ inputs:
 runs:
  using: composite
  steps:
-    - name: Create `ctest` symbolic link
-      run: |
-        if ! command -v ctest &> /dev/null && command -v ctest3 &> /dev/null; then
-          ln --symbolic ctest3 /usr/bin/ctest
-        fi
-      shell: bash --noprofile --norc -euxo pipefail {0}
-
    - name: Test `mod_tile`
-      run: ctest ${{ inputs.options }}
-      shell: bash --noprofile --norc -euxo pipefail {0}
-      working-directory: build
-
-    - name: Archive test artifacts on failure
-      if: failure()
      run: |
-        TAR_FILENAME=${{ matrix.image || matrix.os || github.job }}-${{ matrix.compiler }}.tar.gz
-        TAR_FILENAME=$(echo "${TAR_FILENAME}" | sed 's/:/-/g')
-        tar -zcf ${TAR_FILENAME} tests
+        export CTEST_PARALLEL_LEVEL=${TEST_PARALLEL_LEVEL:-$(nproc)}
+        ctest ${{ inputs.options }}
      shell: bash --noprofile --norc -euxo pipefail {0}
      working-directory: build

    - name: Upload test artifacts on failure
-      if: failure()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      with:
-        name: Test Artifacts
-        path: build/*.tar.gz
+        name: Test Artifacts - ${{ matrix.image || matrix.os || github.job }}-${{ matrix.compiler }}
+        path: build/tests
+      if: failure()
--- a/.github/actions/dependencies/build-and-install/mapnik/action.yml
+++ b/.github/actions/dependencies/build-and-install/mapnik/action.yml
@ -14,7 +14,7 @@ runs:

    - name: Cache "Download `Mapnik`" & "Build `Mapnik`"
      id: cache-mapnik-src
-      uses: actions/cache@v3
+      uses: actions/cache@v4
      with:
        path: mapnik-src
        key: ${{ matrix.image }}-${{ matrix.compiler }}-mapnik-${{ inputs.version }}
--- a/.github/actions/dependencies/build-and-install/mapnik/latest/action.yml
+++ b/.github/actions/dependencies/build-and-install/mapnik/latest/action.yml
@ -4,7 +4,7 @@ runs:
  steps:
    - name: Cache "Checkout `Mapnik`" & "Build `Mapnik`"
      id: cache-mapnik
-      uses: actions/cache@v3
+      uses: actions/cache@v4
      with:
        path: |
          mapnik-build
--- a/.github/actions/dependencies/install/action.yml
+++ b/.github/actions/dependencies/install/action.yml
@ -197,15 +197,13 @@ inputs:
      memcached
  mapnik-build-version-centos-stream:
    default: 3.1.0
-  mapnik-build-version-amazonlinux2-centos7:
-    default: 3.0.17
  mapnik-build-version-opensuse:
    default: 3.1.0

 runs:
  using: composite
  steps:
-    - name: Install Dependencies (Amazon Linux/CentOS/CentOS Stream)
+    - name: Install Dependencies (CentOS Stream)
      uses: ./.github/actions/dependencies/install/yum
      with:
        dependencies: epel-release
@ -213,13 +211,9 @@ runs:
          ${{ inputs.rhel-build-dependencies }}
          ${{ inputs.rhel-mapnik-build-dependencies }}
          ${{ inputs.rhel-test-dependencies }}
-          ${{ matrix.build_system == 'CMake' && 'cmake3' || 'autoconf automake redhat-rpm-config' }}
+          ${{ matrix.build_system == 'CMake' && 'cmake' || 'autoconf automake redhat-rpm-config' }}
          ${{ matrix.compiler == 'LLVM' && 'clang' || 'gcc gcc-c++' }}
-          ${{ matrix.image == 'centos:7' && matrix.compiler == 'LLVM' && 'gcc-c++' || '' }}
-      if: |
-        startsWith(matrix.image, 'amazonlinux:') ||
-        startsWith(matrix.image, 'centos:') ||
-        startsWith(matrix.image, 'quay.io/centos/centos:stream')
+      if: startsWith(matrix.image, 'quay.io/centos/centos:stream')

    - name: Install Dependencies (Debian)
      uses: ./.github/actions/dependencies/install/apt-get
@ -299,12 +293,6 @@ runs:
      shell: bash --noprofile --norc -euxo pipefail {0}
      if: startsWith(matrix.image, 'opensuse/')

-    - name: Build & Install `mapnik` (Amazon Linux 2/CentOS 7)
-      uses: ./.github/actions/dependencies/build-and-install/mapnik
-      with:
-        version: ${{ inputs.mapnik-build-version-amazonlinux2-centos7 }}
-      if: matrix.image == 'amazonlinux:2' || matrix.image == 'centos:7'
-
    - name: Build & Install `mapnik` (CentOS Stream)
      uses: ./.github/actions/dependencies/build-and-install/mapnik
      with:
--- a/.github/actions/dependencies/install/yum/action.yml
+++ b/.github/actions/dependencies/install/yum/action.yml
@ -28,11 +28,6 @@ runs:
      shell: bash --noprofile --norc -euxo pipefail {0}
      if: matrix.image == 'quay.io/centos/centos:stream9'

-    - name: Enable EPEL repository (Amazon Linux 2)
-      run: amazon-linux-extras install epel -y
-      shell: bash --noprofile --norc -euxo pipefail {0}
-      if: matrix.image == 'amazonlinux:2' && contains(inputs.dependencies, 'epel-release')
-
    - name: Install dependency package(s)
      run: |
        if [ -n "${{ inputs.dependencies }}" ]; then
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@ -15,14 +15,15 @@ jobs:
    strategy:
      matrix:
        image:
-          - "centos:7"
          - "debian:11"
          - "debian:12"
          - "debian:testing"
          - "fedora:38"
          - "fedora:39"
-          - "fedora:rawhide"
+          # - "fedora:rawhide"
          - "opensuse/leap:15"
+          - "quay.io/centos/centos:stream8"
+          - "quay.io/centos/centos:stream9"
          - "ubuntu:20.04"
          - "ubuntu:rolling"
        build_system:
@ -53,21 +54,12 @@ jobs:
        CXX: ${{ matrix.compiler == 'LLVM' && 'clang++' || 'g++' }}
      image: ${{ matrix.image }}
    steps:
-      - name: Install `git` (Amazon Linux 2)
-        run: yum --assumeyes install git
-        if: matrix.image == 'amazonlinux:2'
-
      - name: Install `git` (openSUSE)
        run: zypper --non-interactive install git
        if: startsWith(matrix.image, 'opensuse/')

-      - name: Checkout code (CentOS 7)
-        uses: actions/checkout@v3
-        if: matrix.image == 'centos:7'
-
      - name: Checkout code
        uses: actions/checkout@v4
-        if: matrix.image != 'centos:7'

      - name: Install dependencies
        uses: ./.github/actions/dependencies/install
@ -127,6 +119,7 @@ jobs:
      CFLAGS: -Wno-implicit-function-declaration
      LDFLAGS: -undefined dynamic_lookup
      LIBRARY_PATH: /usr/local/lib
+      TEST_PARALLEL_LEVEL: 1
    name: >-
      ${{ matrix.os }}
      (${{ matrix.build_system }})
@ -178,6 +171,7 @@ jobs:
      BUILD_PARALLEL_LEVEL: 2
      CTEST_HOST: localhost
      LIBRARY_PATH: /usr/local/lib
+      TEST_PARALLEL_LEVEL: 2
      TMPDIR: /tmp
    name: >-
      ${{ matrix.box_generic }}
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@ -19,6 +19,7 @@ jobs:
      BUILD_TYPE: Debug
      CFLAGS: --coverage
      CXXFLAGS: --coverage
+      TEST_PARALLEL_LEVEL: 1
    strategy:
      matrix:
        compiler:
@ -66,12 +67,13 @@ jobs:

      - name: Generate `mod_tile` coverage results artifact
        run: |
-          mkdir -p coverage
-          genhtml coverage.info --output-directory coverage/${{ matrix.mapnik_latest && 'mapnik_latest' || 'mapnik' }}
+          genhtml coverage.info --output-directory coverage
        working-directory: build

      - name: Upload `mod_tile` coverage results artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
        with:
-          name: Coverage Results
-          path: build/coverage
+          name: Coverage Results - ${{ matrix.mapnik_latest && 'mapnik_latest' || 'mapnik' }}
+          path: |
+            build/coverage
+            build/coverage.info
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@ -56,7 +56,7 @@ jobs:
          git diff --patch > ArtisticStyleFormattingFixes.patch
      - name: Upload `ArtisticStyleFormattingFixes.patch` file
        if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
        with:
          name: ArtisticStyleFormattingFixes.patch
          path: ArtisticStyleFormattingFixes.patch
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@ -64,13 +64,7 @@ set(TESTS_LOGS_DIR "${PROJECT_BINARY_DIR}/tests/logs")
 set(TESTS_RUN_DIR "${PROJECT_BINARY_DIR}/tests/run")
 set(TESTS_TILES_DIR "${PROJECT_BINARY_DIR}/tests/tiles")

-set(TILE_JPG_SHA256SUM "e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7")
-set(TILE_PNG256_SHA256SUM "dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08")
-set(TILE_PNG32_SHA256SUM "1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b")
-set(TILE_WEBP_SHA256SUM_01 "ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc") # libwebp.so.4
-set(TILE_WEBP_SHA256SUM_02 "96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5") # libwebp.so.6
-set(TILE_WEBP_SHA256SUM_03 "a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4") # libwebp.so.7
-set(TILE_WEBP_SHA256SUM_04 "904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af") # libwebp.so.7
+file(COPY tiles.sha256sum DESTINATION ${PROJECT_BINARY_DIR}/tests)

 set(TILE_ZXY "9/297/191")

@ -97,6 +91,76 @@ endif()
 list(LENGTH STORAGE_BACKENDS STORAGE_BACKENDS_LENGTH)
 math(EXPR STORAGE_BACKENDS_LENGTH "${STORAGE_BACKENDS_LENGTH} - 1")

+#-----------------------------------------------------------------------------
+#
+#  Test functions
+#
+#-----------------------------------------------------------------------------
+
+function(add_bad_tile_download_test TEST_NAME TEST_URL REQUIRED_FIXTURE_NAME)
+  add_test(
+    NAME bad_download_tile_${TEST_NAME}
+    COMMAND ${BASH} -c "
+      echo 'Downloading ${TEST_URL}'
+      while true; do
+        HTTP_CODE=$(${CURL_CMD} --write-out '%{http_code}' ${TEST_URL})
+        echo \"HTTP Code: '\${HTTP_CODE}'\"
+        if [ \"\${HTTP_CODE}\" == \"404\" ] || [ \"\${HTTP_CODE}\" == \"500\" ]; then
+          exit 0;
+        fi
+        echo 'Sleeping 1s';
+        sleep 1;
+      done
+    "
+    WORKING_DIRECTORY tests
+  )
+  set_tests_properties(bad_download_tile_${TEST_NAME} PROPERTIES
+    FIXTURES_REQUIRED ${REQUIRED_FIXTURE_NAME}
+    TIMEOUT 10 # Needs to be more than ModTileMissingRequestTimeout in httpd.conf
+  )
+endfunction()
+
+function(add_good_tile_download_test TEST_NAME TEST_URL FILE_NAME FIXTURE_NAME REQUIRED_FIXTURE_NAME)
+  add_test(
+    NAME download_tile_${TEST_NAME}
+    COMMAND ${BASH} -c "
+      echo 'Downloading ${TEST_URL}'
+      until $(${CURL_CMD} ${TEST_URL} --output ${FILE_NAME}); do
+        echo 'Sleeping 1s';
+        sleep 1;
+      done
+    "
+    WORKING_DIRECTORY tests
+  )
+  set_tests_properties(download_tile_${TEST_NAME} PROPERTIES
+    FIXTURES_REQUIRED ${REQUIRED_FIXTURE_NAME}
+    FIXTURES_SETUP ${FIXTURE_NAME}
+    TIMEOUT 10
+  )
+  add_test(
+    NAME check_tile_${TEST_NAME}
+    COMMAND ${BASH} -c "
+      ${SHA256SUM_EXECUTABLE} -c tiles.sha256sum | ${GREP_EXECUTABLE} ${FILE_NAME} | ${GREP_EXECUTABLE} OK
+    "
+    WORKING_DIRECTORY tests
+  )
+  set_tests_properties(check_tile_${TEST_NAME} PROPERTIES
+    DEPENDS_ON download_tile_${TEST_NAME}
+    FIXTURES_REQUIRED ${FIXTURE_NAME}
+    REQUIRED_FILES ${FILE_NAME}
+  )
+  add_test(
+    NAME remove_tile_${TEST_NAME}
+    COMMAND ${RM} -v ${FILE_NAME}
+    WORKING_DIRECTORY tests
+  )
+  set_tests_properties(remove_tile_${TEST_NAME} PROPERTIES
+    DEPENDS_ON download_tile_${TEST_NAME}
+    FIXTURES_CLEANUP ${FIXTURE_NAME}
+    REQUIRED_FILES ${FILE_NAME}
+  )
+endfunction()
+
 #-----------------------------------------------------------------------------
 #
 #  Tests
@ -269,7 +333,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
      WORKING_DIRECTORY tests
    )
    set_tests_properties(render_expired_delete_${SOCKET_TYPE}_${STORAGE_BACKEND} PROPERTIES
-      DEPENDS "render_expired_touch_${SOCKET_TYPE}_${STORAGE_BACKEND};render_speedtest_${SOCKET_TYPE}_${STORAGE_BACKEND}"
+      DEPENDS render_expired_touch_${SOCKET_TYPE}_${STORAGE_BACKEND}
      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
      TIMEOUT 60
    )
@ -369,7 +433,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
      WORKING_DIRECTORY tests
    )
    set_tests_properties(render_speedtest_${SOCKET_TYPE}_${STORAGE_BACKEND} PROPERTIES
-      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+      FIXTURES_REQUIRED "services_started_${STORAGE_BACKEND};tiles_downloaded_${STORAGE_BACKEND}"
      TIMEOUT 60
    )
    add_test(
@ -440,29 +504,16 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
      endif()
      set(TILE_URL_PATH "/${BAD_ADD_CONFIG}_${SOCKET_TYPE}/${TILE_ZXY}.${EXTENSION}")
      set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
-      set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
-      add_test(
-        NAME bad_download_tile_${BAD_ADD_CONFIG}_${SOCKET_TYPE}_${STORAGE_BACKEND}
-        COMMAND ${BASH} -c "
-          echo \"Downloading ${HTTPD0_URL}\"
-          if ${CURL_CMD} ${HTTPD0_URL}; then
-            echo \"Unexpected success.\"
-            exit 1;
-          else
-            exit 0;
-          fi
-          echo \"Downloading ${HTTPD1_URL}\"
-          if ${CURL_CMD} ${HTTPD1_URL}; then
-            echo \"Unexpected success.\"
-            exit 1;
-          else
-            exit 0;
-          fi
-        "
-        WORKING_DIRECTORY tests
+      add_bad_tile_download_test(
+        ${BAD_ADD_CONFIG}_${SOCKET_TYPE}_${STORAGE_BACKEND}_0
+        ${HTTPD0_URL}
+        services_started_${STORAGE_BACKEND}
      )
-      set_tests_properties(bad_download_tile_${BAD_ADD_CONFIG}_${SOCKET_TYPE}_${STORAGE_BACKEND} PROPERTIES
-        FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+      set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
+      add_bad_tile_download_test(
+        ${BAD_ADD_CONFIG}_${SOCKET_TYPE}_${STORAGE_BACKEND}_1
+        ${HTTPD1_URL}
+        services_started_${STORAGE_BACKEND}
      )
    endforeach()
  endforeach()
@ -470,28 +521,23 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
  foreach(MAP_NAME IN LISTS MAP_NAMES)
    # Set EXTENSION from MAP_NAME (only works for map names containing an actual extension)
    string(REGEX REPLACE "[0-9]+" "" EXTENSION ${MAP_NAME})
+    set(TILE_FILE_NAME "tile.${MAP_NAME}.${STORAGE_BACKEND}")
    set(TILE_URL_PATH "/tiles/${MAP_NAME}/${TILE_ZXY}.${EXTENSION}")
    set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
-    set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
-    set(TILE_FILE_NAME "tile.${MAP_NAME}.${STORAGE_BACKEND}")
-    add_test(
-      NAME download_tile_${MAP_NAME}_${STORAGE_BACKEND}
-      COMMAND ${BASH} -c "
-        until $(${CURL_CMD} ${HTTPD0_URL} --output ${TILE_FILE_NAME}.0); do
-          echo 'Sleeping 1s (${TILE_FILE_NAME}.0)';
-          sleep 1;
-        done
-        until $(${CURL_CMD} ${HTTPD1_URL} --output ${TILE_FILE_NAME}.1); do
-          echo 'Sleeping 1s (${TILE_FILE_NAME}.1)';
-          sleep 1;
-        done
-      "
-      WORKING_DIRECTORY tests
+    add_good_tile_download_test(
+      ${MAP_NAME}_${STORAGE_BACKEND}_0
+      ${HTTPD0_URL}
+      ${TILE_FILE_NAME}.0
+      tiles_downloaded_${STORAGE_BACKEND}
+      services_started_${STORAGE_BACKEND}
    )
-    set_tests_properties(download_tile_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
-      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
-      FIXTURES_SETUP tiles_downloaded_${STORAGE_BACKEND}
-      TIMEOUT 10
+    set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
+    add_good_tile_download_test(
+      ${MAP_NAME}_${STORAGE_BACKEND}_1
+      ${HTTPD1_URL}
+      ${TILE_FILE_NAME}.1
+      tiles_downloaded_${STORAGE_BACKEND}
+      services_started_${STORAGE_BACKEND}
    )
    add_test(
      NAME status_and_dirty_${MAP_NAME}_${STORAGE_BACKEND}
@ -528,85 +574,30 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
      WORKING_DIRECTORY tests
    )
    set_tests_properties(status_and_dirty_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
-      DEPENDS_ON download_tile_${MAP_NAME}_${STORAGE_BACKEND}
-      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+      FIXTURES_REQUIRED "services_started_${STORAGE_BACKEND};tiles_downloaded_${STORAGE_BACKEND}"
      TIMEOUT 20
    )
-    add_test(
-      NAME remove_tile_${MAP_NAME}_${STORAGE_BACKEND}
-      COMMAND ${RM} -v ${TILE_FILE_NAME}.0 ${TILE_FILE_NAME}.1
-      WORKING_DIRECTORY tests
-    )
-    set_tests_properties(remove_tile_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
-      DEPENDS_ON download_tile_${MAP_NAME}_${STORAGE_BACKEND}
-      FIXTURES_CLEANUP tiles_downloaded_${STORAGE_BACKEND}
-      REQUIRED_FILES "${TILE_FILE_NAME}.0;${TILE_FILE_NAME}.1"
-    )
  endforeach()

+  set(TILE_FILE_NAME "tile.add_tile_config.${STORAGE_BACKEND}")
  set(TILE_URL_PATH "/download_add_tile_config/${TILE_ZXY}.png")
  set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
+  add_good_tile_download_test(
+    add_tile_config_${STORAGE_BACKEND}_0
+    ${HTTPD0_URL}
+    ${TILE_FILE_NAME}.0
+    tiles_downloaded_${STORAGE_BACKEND}
+    services_started_${STORAGE_BACKEND}
+  )
  set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
-  set(TILE_FILE_NAME "tile.add_tile_config.${STORAGE_BACKEND}")
-  add_test(
-    NAME download_tile_add_tile_config_${STORAGE_BACKEND}
-    COMMAND ${BASH} -c "
-      until $(${CURL_CMD} ${HTTPD0_URL} --output ${TILE_FILE_NAME}.0); do
-        echo 'Sleeping 1s (${TILE_FILE_NAME}.0)';
-        sleep 1;
-      done
-      until $(${CURL_CMD} ${HTTPD1_URL} --output ${TILE_FILE_NAME}.1); do
-        echo 'Sleeping 1s (${TILE_FILE_NAME}.1)';
-        sleep 1;
-      done
-    "
-    WORKING_DIRECTORY tests
-  )
-  set_tests_properties(download_tile_add_tile_config_${STORAGE_BACKEND} PROPERTIES
-    FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
-    FIXTURES_SETUP tiles_downloaded_${STORAGE_BACKEND}
-    TIMEOUT 10
-  )
-  add_test(
-    NAME remove_tile_add_tile_config_${STORAGE_BACKEND}
-    COMMAND ${RM} -v ${TILE_FILE_NAME}.0 ${TILE_FILE_NAME}.1
-    WORKING_DIRECTORY tests
-  )
-  set_tests_properties(remove_tile_add_tile_config_${STORAGE_BACKEND} PROPERTIES
-    DEPENDS_ON download_tile_add_tile_config_${STORAGE_BACKEND}
-    FIXTURES_CLEANUP tiles_downloaded_${STORAGE_BACKEND}
-    REQUIRED_FILES "${TILE_FILE_NAME}.0;${TILE_FILE_NAME}.1"
+  add_good_tile_download_test(
+    add_tile_config_${STORAGE_BACKEND}_1
+    ${HTTPD1_URL}
+    ${TILE_FILE_NAME}.1
+    tiles_downloaded_${STORAGE_BACKEND}
+    services_started_${STORAGE_BACKEND}
  )

-  add_test(
-    NAME check_tiles_${STORAGE_BACKEND}
-    COMMAND ${BASH} -c "
-      (echo '${TILE_JPG_SHA256SUM}  tile.jpg.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_JPG_SHA256SUM}  tile.jpg.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_PNG256_SHA256SUM}  tile.add_tile_config.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_PNG256_SHA256SUM}  tile.add_tile_config.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_PNG256_SHA256SUM}  tile.png256.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_PNG256_SHA256SUM}  tile.png256.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_PNG32_SHA256SUM}  tile.png32.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) && \
-      (echo '${TILE_PNG32_SHA256SUM}  tile.png32.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) && \
-      ( \
-        (echo '${TILE_WEBP_SHA256SUM_01}  tile.webp.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) || \
-        (echo '${TILE_WEBP_SHA256SUM_02}  tile.webp.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) || \
-        (echo '${TILE_WEBP_SHA256SUM_03}  tile.webp.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) || \
-        (echo '${TILE_WEBP_SHA256SUM_04}  tile.webp.${STORAGE_BACKEND}.0' | ${SHA256SUM_EXECUTABLE} -c) \
-      ) && \
-      ( \
-        (echo '${TILE_WEBP_SHA256SUM_01}  tile.webp.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) || \
-        (echo '${TILE_WEBP_SHA256SUM_02}  tile.webp.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) || \
-        (echo '${TILE_WEBP_SHA256SUM_03}  tile.webp.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) || \
-        (echo '${TILE_WEBP_SHA256SUM_04}  tile.webp.${STORAGE_BACKEND}.1' | ${SHA256SUM_EXECUTABLE} -c) \
-      )
-    "
-    WORKING_DIRECTORY tests
-  )
-  set_tests_properties(check_tiles_${STORAGE_BACKEND} PROPERTIES
-    FIXTURES_REQUIRED tiles_downloaded_${STORAGE_BACKEND}
-  )
  add_test(
    NAME stats_urls_${STORAGE_BACKEND}
    COMMAND ${BASH} -c "
@ -675,49 +666,23 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
  )

  if(STORAGE_BACKEND STREQUAL file)
+    set(TILE_FILE_NAME "tile.parameterization.${STORAGE_BACKEND}")
    set(TILE_URL_PATH "/tiles/${DEFAULT_MAP_NAME}/en,de,_/${TILE_ZXY}.png")
    set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
+    add_good_tile_download_test(
+      parameterization_${STORAGE_BACKEND}_0
+      ${HTTPD0_URL}
+      ${TILE_FILE_NAME}.0
+      tiles_downloaded_${STORAGE_BACKEND}
+      services_started_${STORAGE_BACKEND}
+    )
    set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
-    set(TILE_FILE_NAME "tile.parameterization.${STORAGE_BACKEND}")
-    add_test(
-      NAME download_tile_parameterization_${STORAGE_BACKEND}
-      COMMAND ${BASH} -c "
-        until $(${CURL_CMD} ${HTTPD0_URL} --output ${TILE_FILE_NAME}.0); do
-          echo 'Sleeping 1s (${TILE_FILE_NAME}.0)';
-          sleep 1;
-        done
-        until $(${CURL_CMD} ${HTTPD1_URL} --output ${TILE_FILE_NAME}.1); do
-          echo 'Sleeping 1s (${TILE_FILE_NAME}.1)';
-          sleep 1;
-        done
-      "
-      WORKING_DIRECTORY tests
-    )
-    set_tests_properties(download_tile_parameterization_${STORAGE_BACKEND} PROPERTIES
-      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
-      FIXTURES_SETUP tiles_downloaded_${STORAGE_BACKEND}
-      TIMEOUT 10
-    )
-    add_test(
-      NAME check_tiles_parameterization_${STORAGE_BACKEND}
-      COMMAND ${BASH} -c "
-        (echo '${TILE_PNG256_SHA256SUM}  ${TILE_FILE_NAME}.0' | ${SHA256SUM_EXECUTABLE} -c) && \
-        (echo '${TILE_PNG256_SHA256SUM}  ${TILE_FILE_NAME}.1' | ${SHA256SUM_EXECUTABLE} -c)
-      "
-      WORKING_DIRECTORY tests
-    )
-    set_tests_properties(check_tiles_parameterization_${STORAGE_BACKEND} PROPERTIES
-      FIXTURES_REQUIRED tiles_downloaded_${STORAGE_BACKEND}
-    )
-    add_test(
-      NAME remove_tile_parameterization_${STORAGE_BACKEND}
-      COMMAND ${RM} -v ${TILE_FILE_NAME}.0 ${TILE_FILE_NAME}.1
-      WORKING_DIRECTORY tests
-    )
-    set_tests_properties(remove_tile_parameterization_${STORAGE_BACKEND} PROPERTIES
-      DEPENDS_ON download_tile_parameterization_${STORAGE_BACKEND}
-      FIXTURES_CLEANUP tiles_downloaded_${STORAGE_BACKEND}
-      REQUIRED_FILES "${TILE_FILE_NAME}.0;${TILE_FILE_NAME}.1"
+    add_good_tile_download_test(
+      parameterization_${STORAGE_BACKEND}_1
+      ${HTTPD1_URL}
+      ${TILE_FILE_NAME}.1
+      tiles_downloaded_${STORAGE_BACKEND}
+      services_started_${STORAGE_BACKEND}
    )
  endif()

--- a/tests/httpd.conf.in
+++ b/tests/httpd.conf.in
@ -34,7 +34,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
  ModTileEnableTileThrottlingXForward 0
  ModTileMaxLoadMissing 5
  ModTileMaxLoadOld 2
-  ModTileMissingRequestTimeout 10
+  ModTileMissingRequestTimeout 2
  ModTileRenderdSocketName @RENDERD0_SOCKET@
  ModTileRequestTimeout 3
  ModTileThrottlingRenders 128 0.2
@ -62,7 +62,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
  ModTileEnableTileThrottlingXForward 0
  ModTileMaxLoadMissing 5
  ModTileMaxLoadOld 2
-  ModTileMissingRequestTimeout 10
+  ModTileMissingRequestTimeout 3
  ModTileRenderdSocketAddr @RENDERD1_HOST@ @RENDERD1_PORT@
  ModTileRequestTimeout 3
  ModTileThrottlingRenders 128 0.2
@ -70,7 +70,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
  ModTileTileDir @TILE_DIR@
 </VirtualHost>

-CustomLog @HTTPD_LOG_ACCESS@ "%h %l %u %t \"%r\" %>s %b"
+CustomLog @HTTPD_LOG_ACCESS@ "%A:%p %t \"%r\" %>s %b"
 ErrorLog @HTTPD_LOG_ERROR@
 Group @WWW_GROUP_NAME@
 Listen @HTTPD0_HOST@:@HTTPD0_PORT@
--- a/tests/tiles.sha256sum
+++ b/tests/tiles.sha256sum
@ -0,0 +1,54 @@
+1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b  tile.png32.file.0
+1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b  tile.png32.file.1
+1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b  tile.png32.memcached_custom.0
+1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b  tile.png32.memcached_custom.1
+1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b  tile.png32.memcached_default.0
+1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b  tile.png32.memcached_default.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.add_tile_config.file.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.add_tile_config.file.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.add_tile_config.memcached_custom.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.add_tile_config.memcached_custom.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.add_tile_config.memcached_default.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.add_tile_config.memcached_default.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.parameterization.file.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.parameterization.file.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.png256.file.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.png256.file.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.png256.memcached_custom.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.png256.memcached_custom.1
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.png256.memcached_default.0
+dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08  tile.png256.memcached_default.1
+e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7  tile.jpg.file.0
+e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7  tile.jpg.file.1
+e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7  tile.jpg.memcached_custom.0
+e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7  tile.jpg.memcached_custom.1
+e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7  tile.jpg.memcached_default.0
+e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7  tile.jpg.memcached_default.1
+# libwebp.so.4
+ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc  tile.webp.file.0
+ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc  tile.webp.file.1
+ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc  tile.webp.memcached_custom.0
+ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc  tile.webp.memcached_custom.1
+ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc  tile.webp.memcached_default.0
+ef3862a57831b21ec69c15be196e1e2b4fea66246c361142631b9fa22b85decc  tile.webp.memcached_default.1
+# libwebp.so.6
+96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5  tile.webp.file.0
+96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5  tile.webp.file.1
+96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5  tile.webp.memcached_custom.0
+96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5  tile.webp.memcached_custom.1
+96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5  tile.webp.memcached_default.0
+96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5  tile.webp.memcached_default.1
+# libwebp.so.7
+a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4  tile.webp.file.0
+a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4  tile.webp.file.1
+a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4  tile.webp.memcached_custom.0
+a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4  tile.webp.memcached_custom.1
+a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4  tile.webp.memcached_default.0
+a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4  tile.webp.memcached_default.1
+# libwebp.so.7
+904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af  tile.webp.file.0
+904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af  tile.webp.file.1
+904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af  tile.webp.memcached_custom.0
+904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af  tile.webp.memcached_custom.1
+904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af  tile.webp.memcached_default.0
+904593e291cce2561138bd83b704588c02c16630b8c133d78d535b8986e901af  tile.webp.memcached_default.1