mod_tile: Implement directory hashing. Improve reconnection between mod_tile and renderer. Use new Mapnik native code for cutting and saving the tiles in 256 colours. Removes ImageMagick dependency.

Makefile

@@ -25,13 +25,13 @@ clean:
 RENDER_CPPFLAGS += -g -O2 -Wall
 RENDER_CPPFLAGS += -I/usr/local/include/mapnik
 RENDER_CPPFLAGS += $(shell pkg-config --cflags freetype2)
-RENDER_CPPFLAGS += $(shell Magick++-config --cxxflags --cppflags)
+#RENDER_CPPFLAGS += $(shell Magick++-config --cxxflags --cppflags)
 RENDER_CPPFLAGS += $(shell pkg-config --cflags libagg)
 
 RENDER_LDFLAGS += -g
 RENDER_LDFLAGS += -lmapnik -L/usr/local/lib64
 RENDER_LDFLAGS += $(shell pkg-config --libs freetype2)
-RENDER_LDFLAGS += $(shell Magick++-config --ldflags --libs)
+#RENDER_LDFLAGS += $(shell Magick++-config --ldflags --libs)
 RENDER_LDFLAGS += $(shell pkg-config --libs libagg)
 
 renderd: daemon.c gen_tile.cpp

gen_tile.cpp

@@ -18,6 +18,10 @@
 #include "gen_tile.h"
 #include "protocol.h"
 
+
+#undef USE_RENDER_OFFSET
+
+
 using namespace mapnik;
 
 #define DEG_TO_RAD (M_PIl/180)
@@ -27,7 +31,7 @@ static const int minZoom = 0;
 static const int maxZoom = 18;
 static const char *mapfile = "/home/jburgess/osm/svn.openstreetmap.org/applications/rendering/mapnik/osm-jb-merc.xml";
 
-
+#if 0
 static void postProcess(const char *path)
 {
     // Convert the 32bit RGBA image to one with indexed colours
@@ -53,7 +57,7 @@ static void postProcess(const char *path)
         std::cerr << "Caught exception: " << error_.what() << std::endl;
     }
 }
-
+#endif
 
 
 static double minmax(double a, double b, double c)
@@ -154,11 +158,16 @@ static enum protoCmd render(Map &m, Image32 &buf, int x, int y, int z, const cha
     bbox.width(bbox.width() * 2);
     bbox.height(bbox.height() * 2);
     m.zoomToBox(bbox);
-
+#ifdef USE_RENDER_OFFSET
     agg_renderer<Image32> ren(m,buf, 128,128);
     ren.apply();
-    buf.saveToFile(filename,"png");
-
+    buf.saveToFile(filename,"png256");
+#else
+    agg_renderer<Image32> ren(m,buf);
+    ren.apply();
+    image_view<ImageData32> vw(128,128,256,256, buf.data());
+    save_to_file(filename,"png256", vw);
+#endif
     return cmdDone; // OK
 }
 
@@ -177,7 +186,11 @@ void render_init(void)
 void *render_thread(__attribute__((unused)) void *unused)
 {
     Map m(2 * 256, 2 * 256);
+#ifdef USE_RENDER_OFFSET
     Image32 buf(256, 256);
+#else
+    Image32 buf(512, 512);
+#endif
 
     load_map(m,mapfile);
 
@@ -189,7 +202,7 @@ void *render_thread(__attribute__((unused)) void *unused)
             //pthread_mutex_lock(&map_lock);
             ret = render(m, buf, req->x, req->y, req->z, req->path);
             //pthread_mutex_unlock(&map_lock);
-            postProcess(req->path);
+            //postProcess(req->path);
             send_response(item, ret);
             delete_request(item);
         } else

mod_tile.c

@@ -49,9 +49,11 @@ module AP_MODULE_DECLARE_DATA tile_module;
 // IMG_PATH must have blank.png etc.
 #define WWW_ROOT "/var/www/html"
 #define IMG_PATH "/img"
-// TILE_PATH must have tile z directory z(0..18)/x/y.png
+// TILE_PATH is where Openlayers with try to fetch the "z/x/y.png" tiles from
 #define TILE_PATH "/osm_tiles2"
-//#define TILE_PATH "/tile"
+// With directory hashing enabled we rewrite the path so that tiles are really stored here instead
+#define DIRECTORY_HASH
+#define HASH_PATH "/direct"
 // MAX_LOAD_OLD: if tile is out of date, don't re-render it if past this load threshold (users gets old tile)
 #define MAX_LOAD_OLD 5
 // MAX_LOAD_OLD: if tile is missing, don't render it if past this load threshold (user gets 404 error)
@@ -69,7 +71,7 @@ module AP_MODULE_DECLARE_DATA tile_module;
 #define PLANET_TIMESTAMP "/tmp/planet-import-complete"
 
 // Timeout before giving for a tile to be rendered
-#define REQUEST_TIMEOUT (3)
+#define REQUEST_TIMEOUT (5)
 #define FD_INVALID (-1)
 
 
@@ -92,7 +94,8 @@ static int error_message(request_rec *r, const char *format, ...)
     len = vasprintf(&msg, format, ap);
 
     if (msg) {
-        ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "%s", msg);
+        //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "%s", msg);
+        ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "%s", msg);
         r->content_type = "text/plain";
         if (!r->header_only)
             ap_rputs(msg, r);
@@ -136,7 +139,8 @@ int request_tile(request_rec *r, int x, int y, int z, const char *filename, int
     //struct pollfd fds[1];
     static int fd = FD_INVALID;
     int ret = 0;
-
+    int retry = 1;
+ 
     if (fd == FD_INVALID) {
         fd = socket_init(r);
 
@@ -144,7 +148,7 @@ int request_tile(request_rec *r, int x, int y, int z, const char *filename, int
             //fprintf(stderr, "Failed to connect to renderer\n");
             return 0;
         } else {
-        ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "Connected to renderer");
+            ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "Connected to renderer");
         }
     }
 
@@ -158,16 +162,22 @@ int request_tile(request_rec *r, int x, int y, int z, const char *filename, int
     strcpy(cmd.path, filename);
 
     //fprintf(stderr, "Requesting tile(%d,%d,%d)\n", z,x,y);
-    ret = send(fd, &cmd, sizeof(cmd), 0);
-    if (ret != sizeof(cmd)) {
-        if (errno == EPIPE) {
-            close(fd);
-            fd = FD_INVALID;
-        }
+    do {
+        ret = send(fd, &cmd, sizeof(cmd), 0);
+
+        if (ret == sizeof(cmd))
+            break;
+
+        if (errno != EPIPE)
+            return 0;
+ 
+         close(fd);
+         fd = socket_init(r);
+         if (fd == FD_INVALID)
+             return 0;
+         ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "Reconnected to renderer");
+    } while (retry--);
 
-        //perror("send error");
-        return 0;
-    }
     if (!dirtyOnly) {
         struct timeval tv = { REQUEST_TIMEOUT, 0 };
         fd_set rx;
@@ -278,16 +288,22 @@ static apr_status_t expires_filter(ap_filter_t *f, apr_bucket_brigade *b)
     return ap_pass_brigade(f->next, b);
 }
 
-static int serve_blank(request_rec *r)
+
+static int serve_tile(request_rec *r, const char *rel_path)
 {
-    // Redirect request to blank tile
+    // Redirect request to the tile
     r->method = apr_pstrdup(r->pool, "GET");
     r->method_number = M_GET;
     apr_table_unset(r->headers_in, "Content-Length");
-    ap_internal_redirect_handler(IMG_PATH "/blank-000000.png", r);
+    ap_internal_redirect_handler(rel_path, r);
     return OK;
 }
 
+static int serve_blank(request_rec *r)
+{
+    return serve_tile(r, IMG_PATH "/blank-000000.png");
+}
+
 int get_load_avg(request_rec *r)
 {
     FILE *loadavg = fopen("/proc/loadavg", "r");
@@ -316,7 +332,7 @@ static int tile_dirty(request_rec *r, int x, int y, int z, const char *path)
 
 
 
-static int get_tile(request_rec *r, int x, int y, int z, const char *path)
+static int get_tile(request_rec *r, int x, int y, int z, const char *abs_path, const char *rel_path)
 {
     int avg = get_load_avg(r);
     enum tileState state;
@@ -326,39 +342,31 @@ static int get_tile(request_rec *r, int x, int y, int z, const char *path)
         return error_message(r, "error: Load above MAX_LOAD_ANY threshold %d > %d", avg, MAX_LOAD_ANY);
     }
 
-    state = tile_state(r, path);
-
-    // Note: We rely on the default Apache handler to return the files from the filesystem
-    // hence we return DECLINED in order to return the tile to the client
-    // or OK if we want to send something else.
+    state = tile_state(r, abs_path);
 
     switch (state) {
         case tileCurrent:
-            return DECLINED;
+            return serve_tile(r, rel_path);
             break;
         case tileOld:
             if (avg > MAX_LOAD_OLD) {
                // Too much load to render it now, mark dirty but return old tile
-               tile_dirty(r, x, y, z, path);
-               return DECLINED;
+               tile_dirty(r, x, y, z, abs_path);
+               return serve_tile(r, rel_path);
             }
             break;
         case tileMissing:
             if (avg > MAX_LOAD_MISSING) {
-               tile_dirty(r, x, y, z, path);
+               tile_dirty(r, x, y, z, abs_path);
                return error_message(r, "error: File missing and load above MAX_LOAD_MISSING threshold %d > %d", avg, MAX_LOAD_MISSING);
             }
             break;
     }
 
-    if (request_tile(r, x,y,z,path, 0)) {
-        // Need to make apache try accessing this tile again (since it may have been missing)
-        // TODO: Instead of redirect, maybe we can update fileinfo for new tile, but is this sufficient?
-        apr_table_unset(r->headers_in, "Content-Length");
-        ap_internal_redirect_handler(r->uri, r);
-        return OK;
-    }
-    return error_message(r, "rendering failed for %s", path);
+    if (request_tile(r, x,y,z,abs_path, 0))
+        return serve_tile(r, rel_path);
+
+    return error_message(r, "rendering failed for %s", rel_path);
 }
 
 static int tile_status(request_rec *r, int x, int y, int z, const char *path)
@@ -390,13 +398,38 @@ static int tile_status(request_rec *r, int x, int y, int z, const char *path)
     return error_message(r, "Tile is %s. Last rendered at %s. Last accessed at %s", old ? "due to be rendered" : "clean", MtimeStr, AtimeStr);
 }
 
+static const char *xyz_to_path(char *path, size_t len, int x, int y, int z)
+{
+#ifdef DIRECTORY_HASH
+    // Directory hashing is optimised for z18 max (2^18 tiles split into 2 levels of 2^9)
+    //snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d/%03d/%03d/%03d/%03d.png", z,
+    //                 x/512 x%512, y/512, y%512);
+
+    // We attempt to cluseter the tiles so that a 16x16 square of tiles will be in a single directory
+    // Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
+    // 4 bits of x & y are used per byte of output
+    unsigned char i, hash[5];
+
+    for (i=0; i<5; i++) {
+        hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
+        x >>= 4;
+        y >>= 4;
+    }
+    snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d/%u/%u/%u/%u/%u.png", z, hash[4], hash[3], hash[2], hash[1], hash[0]);
+#else
+    snprintf(path, PATH_MAX, WWW_ROOT TILE_PATH "/%d/%d/%d.png", z, x, y);
+#endif
+    return path + strlen(WWW_ROOT);
+}
+
 
 static int tile_handler(request_rec *r)
 {
     int x, y, z, n, limit;
     char option[11];
     int oob;
-    char path[PATH_MAX];
+    char abs_path[PATH_MAX];
+    const char *rel_path;
 
     option[0] = '\0';
 
@@ -413,10 +446,10 @@ static int tile_handler(request_rec *r)
         return DECLINED;
     }
 
-    // Generate the tile filename.
-    // This may differ from r->filename in some cases (e.g. if a parent directory is missing)
-    snprintf(path, PATH_MAX, WWW_ROOT TILE_PATH "/%d/%d/%d.png", z, x, y);
-
+    // Generate the tile filename
+    rel_path = xyz_to_path(abs_path, sizeof(abs_path), x, y, z);
+    //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "abs_path(%s), rel_path(%s)", abs_path, rel_path);
+    
     // Validate tile co-ordinates
     oob = (z < 0 || z > MAX_ZOOM);
     if (!oob) {
@@ -427,16 +460,16 @@ static int tile_handler(request_rec *r)
 
     if (n == 3) {
         ap_add_output_filter("MOD_TILE", NULL, r, r->connection);
-        return oob ? serve_blank(r) : get_tile(r, x, y, z, path);
+        return oob ? serve_blank(r) : get_tile(r, x, y, z, abs_path, rel_path);
     }
 
     if (n == 4) {
         if (oob)
             return error_message(r, "The tile co-ordinates that you specified are invalid");
         if (!strcmp(option, "status"))
-            return tile_status(r, x, y, z, path);
+            return tile_status(r, x, y, z, abs_path);
         if (!strcmp(option, "dirty"))
-            return tile_dirty(r, x, y, z, path);
+            return tile_dirty(r, x, y, z, abs_path);
         return error_message(r, "Unknown option");
     }
     return DECLINED;

readme.txt

@@ -31,10 +31,17 @@ and background requests (updating tiles which have expired)
 
 Tile serving
 ============
-Tiles are served from the filesystem using Apache from 
-/var/www/html/osm_tiles2/[Z]/[X]/[Y].png
+To avoid problems with directories becoming too large the
+files are stored in a different layout to that presented
+by the web server. The tiles are now stored under
+/var/www/html/direct/[Z]/nnn/nnn/nnn/nnn/nnn.png
 
-where X,Y,Z are the standard OSM tile co-ordinates.
+Where nnn is derived from a combination of the X and Y
+OSM tile co-ordinates.
+
+Apache serves the files as if they were present
+under "/osm_tiles2/Z/X/Y.png" wiht the path being
+converted automatically.
 
 An Apache module called mod_tile enhances the regular
 Apache file serving mechanisms to provide:
@@ -46,6 +53,8 @@ cache expiry headers
 2) When tiles have expired it requests the rendering
 daemon to render (or re-render) the tile.
 
+3) Remapping of the file path to the hashed layout
+
 There is an attempt to make the mod_tile code aware of
 the load on the server so that it backs off the rendering
 if the machine is under heavy load.
@@ -72,7 +81,9 @@ LoadModule tile_module modules/mod_tile.so
 
 --------------
 
-Create the directory /var/www/html/osm_tiles2/
+Create the directories:
+ /var/www/html/osm_tiles2/
+ /var/www/html/direct/
 
 Run the rendering daemon 'renderd'
 
@@ -92,8 +103,11 @@ The render daemon should have produce a message like:
 Got incoming connection, fd 7, number 1
 Render z(0), x(0), y(0), path(/var/www/html/osm_tiles2/0/0/0.png)
 
-After a few seconds you should see a tile of the world
-in your browser window.
+The disk should start thrashing as Mapnik tries to pull
+in data for the first time. After a few seconds you'll
+probably see a 404 error. Wait for the disk activity to
+cease and then reload the tile. With a bit of luck you
+should see a tile of the world in your browser window.
 
 To get a complete slippy map you should install a copy
 of the OpenLayers based OSM slippy map and point this to
@@ -130,10 +144,15 @@ Unfortunately if you reduce the block size to 1 or 2kB then this also
 has a significant impact on the maximum file system size and number of
 inodes available.
 
+
+**  Note: The issues below have been worked around in the current
+code by using the hashed directory path. 
+
 The simple z/x/y.png filesystem layout means that at high zoom levels
 there can be large numbers of files in a single directory
  
   Zoom 18 = 2^18 = 256k files in a single directory.
 
 If ext2/3 is being used then you really need to have directory indexing
-enabled.
+enabled.
+