Additional tests (#394)

* Add tests for `ModTileCacheExtendedHostName` and expired tileOld/TileVeryOld * Add tests for `ModTileEnableTileThrottling`/`ModTileEnableTileThrottlingXForward`
2025-07-23 00:28:34 +00:00 · 2024-02-22 21:29:10 -07:00
parent c31d526441
commit a0e9681f51
4 changed files with 93 additions and 49 deletions
--- a/includes/store_file_utils.h
+++ b/includes/store_file_utils.h
@ -32,18 +32,18 @@ extern "C" {
 */
 int mkdirp(const char *path);

-/* File path hashing. Used by both mod_tile and render daemon
- * The two must both agree on the file layout for meta-tiling
- * to work
- */
-void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
-
 int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz);

 #ifdef METATILE
 /* New meta-tile storage functions */
 /* Returns the path to the meta-tile and the offset within the meta-tile */
 int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
+#else
+/* File path hashing. Used by both mod_tile and render daemon
+ * The two must both agree on the file layout for meta-tiling
+ * to work
+ */
+void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
 #endif

 #ifdef __cplusplus
--- a/src/store_file_utils.c
+++ b/src/store_file_utils.c
@ -117,27 +117,6 @@ static int check_xyz(int x, int y, int z)
 	return oob;
 }

-void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
-{
-#ifdef DIRECTORY_HASH
-	// We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
-	// Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
-	// 4 bits of x & y are used per byte of output
-	unsigned char i, hash[5];
-
-	for (i = 0; i < 5; i++) {
-		hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
-		x >>= 4;
-		y >>= 4;
-	}
-
-	snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
-#else
-	snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
-#endif
-	return;
-}
-
 int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz)
 {
 #ifdef DIRECTORY_HASH
@ -224,7 +203,7 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
 		snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.meta", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
 	}

-#else
+#else // DIRECTORY_HASH

 	if (strlen(options)) {
 		snprintf(path, len, "%s/%s/%d/%u/%u.%s.meta", tile_dir, xmlconfig, z, x, y, options);
@ -232,7 +211,28 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
 		snprintf(path, len, "%s/%s/%d/%u/%u.meta", tile_dir, xmlconfig, z, x, y);
 	}

-#endif
+#endif // DIRECTORY_HASH
 	return offset;
 }
-#endif
+#else // METATILE
+void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
+{
+#ifdef DIRECTORY_HASH
+	// We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
+	// Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
+	// 4 bits of x & y are used per byte of output
+	unsigned char i, hash[5];
+
+	for (i = 0; i < 5; i++) {
+		hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
+		x >>= 4;
+		y >>= 4;
+	}
+
+	snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
+#else // DIRECTORY_HASH
+	snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
+#endif // DIRECTORY_HASH
+	return;
+}
+#endif // METATILE
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@ -438,7 +438,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
    )
    add_test(NAME add_tile_config_${SOCKET_TYPE}_${STORAGE_BACKEND}
      COMMAND ${BASH} -c "
-        CONFIG_NAME=\"add_tile_config_${SOCKET_TYPE}\"
+        CONFIG_NAME=\"bad_tile_config_${SOCKET_TYPE}\"
        SEARCH_LINE=\$(${GREP_EXECUTABLE} -m1 \"Loading tile config \${CONFIG_NAME}\" ${HTTPD_LOG})
        SEARCH_STRS=(
          \" at /\${CONFIG_NAME}/ \"
@ -461,7 +461,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
    add_test(NAME add_tile_mime_config_${SOCKET_TYPE}_${STORAGE_BACKEND}
      COMMAND ${BASH} -c "
        for SEARCH_CONFIG in js png; do
-          CONFIG_NAME=\"add_tile_mime_config_\${SEARCH_CONFIG}_${SOCKET_TYPE}\"
+          CONFIG_NAME=\"bad_tile_mime_config_\${SEARCH_CONFIG}_${SOCKET_TYPE}\"
          MIME_TYPE=image/png
          if [ \"\${SEARCH_CONFIG}\" = \"js\" ]; then
            MIME_TYPE=text/javascript
@ -492,12 +492,12 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
    set_tests_properties(add_tile_mime_config_${SOCKET_TYPE}_${STORAGE_BACKEND} PROPERTIES
      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
    )
-    foreach(BAD_ADD_CONFIG add_tile_config add_tile_mime_config_js add_tile_mime_config_png)
-      if(BAD_ADD_CONFIG STREQUAL add_tile_config)
+    foreach(BAD_ADD_CONFIG bad_tile_config bad_tile_mime_config_js bad_tile_mime_config_png)
+      if(BAD_ADD_CONFIG STREQUAL bad_tile_config)
        set(EXTENSION jpg)
-      elseif(BAD_ADD_CONFIG STREQUAL add_tile_mime_config_js)
+      elseif(BAD_ADD_CONFIG STREQUAL bad_tile_mime_config_js)
        set(EXTENSION js)
-      elseif(BAD_ADD_CONFIG STREQUAL add_tile_mime_config_png)
+      elseif(BAD_ADD_CONFIG STREQUAL bad_tile_mime_config_png)
        set(EXTENSION png)
      endif()
      # Generate URL path for tiles
@ -529,6 +529,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
    # Generate tile URLs
    set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
    set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
+    set(HTTPD2_URL "http://${HTTPD2_HOST}:${HTTPD2_PORT}${TILE_URL_PATH}")
    # Generate tile dirty and status URLs
    set(DIRTY_OFF_URL "${HTTPD1_URL}/dirty")
    set(DIRTY_ON_URL "${HTTPD0_URL}/dirty")
@ -624,11 +625,47 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
    set_tests_properties(stats_urls_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
      FIXTURES_REQUIRED "services_started_${STORAGE_BACKEND};tiles_downloaded_${STORAGE_BACKEND}"
    )
+    add_test(NAME tile_expired_${MAP_NAME}_${STORAGE_BACKEND}
+      COMMAND ${BASH} -c "
+        ${TOUCH_EXECUTABLE} -d '-1 month' ${TEST_TILES_DIR}/planet-import-complete
+        if ! ${CURL_CMD} --output /dev/null ${HTTPD1_URL}; then
+          echo \"Past import: ${HTTPD1_URL}\";
+        fi
+        if ! ${CURL_CMD} --output /dev/null ${HTTPD2_URL}; then
+          echo \"Past import: ${HTTPD2_URL}\";
+        fi
+        ${TOUCH_EXECUTABLE} -d '+1 month' ${TEST_TILES_DIR}/planet-import-complete
+        if ! ${CURL_CMD} --output /dev/null ${HTTPD1_URL}; then
+          echo \"Future import: ${HTTPD1_URL}\";
+        fi
+        if ! ${CURL_CMD} --output /dev/null ${HTTPD2_URL}; then
+          echo \"Future import: ${HTTPD2_URL}\";
+        fi
+      "
+      WORKING_DIRECTORY tests
+    )
+    set_tests_properties(tile_expired_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
+      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+    )
+    add_test(NAME throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND}
+      COMMAND ${BASH} -c "
+        if ! ${CURL_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, ${CTEST_HOST}\" --output /dev/null ${HTTPD1_URL}; then
+          echo \"${HTTPD1_URL}\";
+        fi
+        if ! ${CURL_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, ${CTEST_HOST}\" --output /dev/null ${HTTPD2_URL}; then
+          echo \"${HTTPD2_URL}\";
+        fi
+      "
+      WORKING_DIRECTORY tests
+    )
+    set_tests_properties(throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES
+      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+    )
  endforeach()

  # Generate file and URL paths for tiles
  set(TILE_FILE_NAME "tile.add_tile_config.${STORAGE_BACKEND}")
-  set(TILE_URL_PATH "/download_add_tile_config/${TILE_ZXY}.png")
+  set(TILE_URL_PATH "/good_add_tile_config/${TILE_ZXY}.png")
  # Generate tile URLs
  set(HTTPD0_URL "http://${HTTPD0_HOST}:${HTTPD0_PORT}${TILE_URL_PATH}")
  set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}")
@ -768,6 +805,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
          exit 1;
        fi
      "
+      WORKING_DIRECTORY tests
    )
    set_tests_properties(cors_all_${STORAGE_BACKEND} PROPERTIES
      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
@ -802,6 +840,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
          exit 1;
        fi
      "
+      WORKING_DIRECTORY tests
    )
    set_tests_properties(cors_localhost_${STORAGE_BACKEND} PROPERTIES
      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
@ -820,6 +859,7 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
          exit 1;
        fi
      "
+      WORKING_DIRECTORY tests
    )
    set_tests_properties(max_load_missing_${STORAGE_BACKEND} PROPERTIES
      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
--- a/tests/httpd.conf.in
+++ b/tests/httpd.conf.in
@ -15,10 +15,10 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
 </IfModule>

 <VirtualHost @HTTPD0_HOST@:@HTTPD0_PORT@>
-  AddTileConfig /add_tile_config_sock/ add_tile_config_sock extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
-  AddTileConfig /download_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
-  AddTileMimeConfig /add_tile_mime_config_js_sock/ add_tile_mime_config_js_sock js
-  AddTileMimeConfig /add_tile_mime_config_png_sock/ add_tile_mime_config_png_sock png
+  AddTileConfig /bad_tile_config_sock/ bad_tile_config_sock extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
+  AddTileConfig /good_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
+  AddTileMimeConfig /bad_tile_mime_config_js_sock/ bad_tile_mime_config_js_sock js
+  AddTileMimeConfig /bad_tile_mime_config_png_sock/ bad_tile_mime_config_png_sock png
  LoadTileConfigFile @RENDERD_CONF@
  ModTileBulkMode Off
  ModTileCacheDurationDirty 900
@ -41,10 +41,10 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
 </VirtualHost>

 <VirtualHost @HTTPD1_HOST@:@HTTPD1_PORT@>
-  AddTileConfig /add_tile_config_tcp/ add_tile_config_tcp extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
-  AddTileConfig /download_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
-  AddTileMimeConfig /add_tile_mime_config_js_tcp/ add_tile_mime_config_js_tcp js
-  AddTileMimeConfig /add_tile_mime_config_png_tcp/ add_tile_mime_config_png_tcp png
+  AddTileConfig /bad_tile_config_tcp/ bad_tile_config_tcp extension=jpg maxzoom=15 mimetype=image/jpeg minzoom=10 tile_dir=@TILE_DIR@
+  AddTileConfig /good_add_tile_config/ @DEFAULT_MAP_NAME@_htcp extension=png maxzoom=20 mimetype=image/png minzoom=0 tile_dir=@TILE_DIR@
+  AddTileMimeConfig /bad_tile_mime_config_js_tcp/ bad_tile_mime_config_js_tcp js
+  AddTileMimeConfig /bad_tile_mime_config_png_tcp/ bad_tile_mime_config_png_tcp png
  LoadTileConfigFile @RENDERD_CONF@
  ModTileBulkMode Off
  ModTileCacheDurationDirty 900
@ -52,18 +52,21 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
  ModTileCacheDurationMax 604800
  ModTileCacheDurationMediumZoom 13 86400
  ModTileCacheDurationMinimum 10800
+  ModTileCacheExtendedDuration 2592000
+  ModTileCacheExtendedHostName @HTTPD1_HOST@
  ModTileCacheLastModifiedFactor 0.20
  ModTileEnableDirtyURL Off
  ModTileEnableStats Off
  ModTileEnableStatusURL Off
-  ModTileEnableTileThrottling Off
-  ModTileEnableTileThrottlingXForward 0
+  ModTileEnableTileThrottling On
+  ModTileEnableTileThrottlingXForward 1
  ModTileMissingRequestTimeout 3
  ModTileRenderdSocketAddr @RENDERD1_HOST@ @RENDERD1_PORT@
  ModTileRequestTimeout 3
  ModTileThrottlingRenders 128 0.2
  ModTileThrottlingTiles 10000 1
  ModTileTileDir @TILE_DIR@
+  ModTileVeryOldThreshold 2678400
 </VirtualHost>

 <VirtualHost @HTTPD2_HOST@:@HTTPD2_PORT@>
@ -78,8 +81,8 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
  ModTileEnableDirtyURL Off
  ModTileEnableStats Off
  ModTileEnableStatusURL Off
-  ModTileEnableTileThrottling Off
-  ModTileEnableTileThrottlingXForward 0
+  ModTileEnableTileThrottling On
+  ModTileEnableTileThrottlingXForward 2
  ModTileMaxLoadMissing 0
  ModTileMaxLoadOld 0
  ModTileMissingRequestTimeout 3
@ -88,6 +91,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d
  ModTileThrottlingRenders 128 0.2
  ModTileThrottlingTiles 10000 1
  ModTileTileDir @TILE_DIR@
+  ModTileVeryOldThreshold -10000000
 </VirtualHost>

 CustomLog @HTTPD_LOG_ACCESS@ "%A:%p %t \"%r\" %>s %b"