diff --git a/Makefile b/Makefile index 9910b4d..b7e455f 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ top_builddir = ${top_dir} include ${top_builddir}/build/special.mk CXX := g++ +CXXFLAGS += -Wall APXS = apxs APACHECTL = apachectl @@ -16,7 +17,7 @@ EXTRA_CFLAGS = -I$(builddir) EXTRA_CPPFLAGS += -g -O2 -Wall EXTRA_LDFLAGS += $(shell pkg-config --libs libagg) -all: local-shared-build renderd +all: local-shared-build renderd speedtest render_list clean: rm -f *.o *.lo *.slo *.la .libs/* @@ -34,9 +35,12 @@ RENDER_LDFLAGS += $(shell pkg-config --libs freetype2) #RENDER_LDFLAGS += $(shell Magick++-config --ldflags --libs) RENDER_LDFLAGS += $(shell pkg-config --libs libagg) -renderd: daemon.c gen_tile.cpp +renderd: daemon.c gen_tile.cpp dir_utils.c protocol.h render_config.h dir_utils.h $(CXX) -o $@ $^ $(RENDER_LDFLAGS) $(RENDER_CPPFLAGS) +speedtest: render_config.h protocol.h dir_utils.c dir_utils.h + +render_list: render_config.h protocol.h dir_utils.c dir_utils.h MYSQL_CFLAGS += -g -O2 -Wall MYSQL_CFLAGS += $(shell mysql_config --cflags) diff --git a/daemon.c b/daemon.c index 2b523a1..c61bbd6 100644 --- a/daemon.c +++ b/daemon.c @@ -14,69 +14,12 @@ #include "gen_tile.h" #include "protocol.h" - -#define QUEUE_MAX (32) -#define MAX_CONNECTIONS (256) - -#define MAX(a,b) ((a) > (b) ? (a) : (b)) - -#define FD_INVALID (-1) -#define REQ_LIMIT (32) -#define DIRTY_LIMIT (1000 * 1000) -#define NUM_THREADS (4) +#include "render_config.h" +#include "dir_utils.h" static pthread_t render_threads[NUM_THREADS]; static struct sigaction sigPipeAction; - -void pipe_handler(__attribute__((unused)) int sigNum) -{ - // Needed in case the client closes the connection - // before we send a response. - // FIXME: is fprintf really safe in signal handler? - //fprintf(stderr, "Caught SIGPIPE\n"); -} - -// Build parent directories for the specified file name -// Note: the part following the trailing / is ignored -// e.g. mkdirp("/a/b/foo.png") == shell mkdir -p /a/b -static int mkdirp(const char *path) { - struct stat s; - char tmp[PATH_MAX]; - char *p; - - strncpy(tmp, path, sizeof(tmp)); - - // Look for parent directory - p = strrchr(tmp, '/'); - if (!p) - return 0; - - *p = '\0'; - - if (!stat(tmp, &s)) - return !S_ISDIR(s.st_mode); - *p = '/'; - // Walk up the path making sure each element is a directory - p = tmp; - if (!*p) - return 0; - p++; // Ignore leading / - while (*p) { - if (*p == '/') { - *p = '\0'; - if (!stat(tmp, &s)) { - if (!S_ISDIR(s.st_mode)) - return 1; - } else if (mkdir(tmp, 0777)) - return 1; - *p = '/'; - } - p++; - } - return 0; -} - static struct item reqHead, dirtyHead, renderHead; static int reqNum, dirtyNum; static pthread_mutex_t qLock; @@ -109,51 +52,39 @@ struct item *fetch_request(void) return item; } -void delete_request(struct item *item) -{ - pthread_mutex_lock(&qLock); - - item->next->prev = item->prev; - item->prev->next = item->next; - - pthread_mutex_unlock(&qLock); - free(item); -} - void clear_requests(int fd) { - struct item *item; + struct item *item, *dupes; pthread_mutex_lock(&qLock); item = reqHead.next; while (item != &reqHead) { if (item->fd == fd) item->fd = FD_INVALID; + + dupes = item->duplicates; + while (dupes) { + if (dupes->fd == fd) + dupes->fd = FD_INVALID; + dupes = dupes->duplicates; + } item = item->next; } + item = renderHead.next; while (item != &renderHead) { if (item->fd == fd) item->fd = FD_INVALID; + + dupes = item->duplicates; + while (dupes) { + if (dupes->fd == fd) + dupes->fd = FD_INVALID; + dupes = dupes->duplicates; + } item = item->next; } - pthread_mutex_unlock(&qLock); -} -void send_response(struct item *item, enum protoCmd rsp) -{ - struct protocol *req = &item->req; - int ret; - - pthread_mutex_lock(&qLock); - - if ((item->fd != FD_INVALID) && (req->cmd == cmdRender)) { - req->cmd = rsp; - //fprintf(stderr, "Sending message %d to %d\n", rsp, item->fd); - ret = send(item->fd, req, sizeof(*req), 0); - if (ret != sizeof(*req)) - perror("send error during send_done"); - } pthread_mutex_unlock(&qLock); } @@ -169,45 +100,83 @@ static inline const char *cmdStr(enum protoCmd c) } } -int pending(struct item *test) +void send_response(struct item *item, enum protoCmd rsp) +{ + struct protocol *req = &item->req; + int ret; + + pthread_mutex_lock(&qLock); + item->next->prev = item->prev; + item->prev->next = item->next; + pthread_mutex_unlock(&qLock); + + while (item) { + struct item *prev = item; + req = &item->req; + if ((item->fd != FD_INVALID) && (req->cmd == cmdRender)) { + req->cmd = rsp; + //fprintf(stderr, "Sending message %s to %d\n", cmdStr(rsp), item->fd); + ret = send(item->fd, req, sizeof(*req), 0); + if (ret != sizeof(*req)) + perror("send error during send_done"); + } + item = item->duplicates; + free(prev); + } +} + + +enum protoCmd pending(struct item *test) { // check all queues and render list to see if this request already queued + // If so, add this new request as a duplicate // call with qLock held struct item *item; - item = reqHead.next; - while (item != &reqHead) { - if ((item->req.x == test->req.x) && (item->req.y == test->req.y) && (item->req.z == test->req.z)) - return 1; - item = item->next; - } - item = dirtyHead.next; - while (item != &dirtyHead) { - if ((item->req.x == test->req.x) && (item->req.y == test->req.y) && (item->req.z == test->req.z)) - return 1; - item = item->next; - } item = renderHead.next; while (item != &renderHead) { - if ((item->req.x == test->req.x) && (item->req.y == test->req.y) && (item->req.z == test->req.z)) - return 1; + if ((item->mx == test->mx) && (item->my == test->my) && (item->req.z == test->req.z)) { + // Add new test item in the list of duplicates + test->duplicates = item->duplicates; + item->duplicates = test; + return cmdIgnore; + } item = item->next; } - return 0; + item = reqHead.next; + while (item != &reqHead) { + if ((item->mx == test->mx) && (item->my == test->my) && (item->req.z == test->req.z)) { + // Add new test item in the list of duplicates + test->duplicates = item->duplicates; + item->duplicates = test; + return cmdIgnore; + } + item = item->next; + } + + item = dirtyHead.next; + while (item != &dirtyHead) { + if ((item->mx == test->mx) && (item->my == test->my) && (item->req.z == test->req.z)) + return cmdNotDone; + item = item->next; + } + + return cmdRender; } enum protoCmd rx_request(const struct protocol *req, int fd) { struct item *list = NULL, *item; + enum protoCmd pend; if (req->ver != 1) { fprintf(stderr, "Bad protocol version %d\n", req->ver); return cmdIgnore; } - fprintf(stderr, "%s z(%d), x(%d), y(%d), path(%s)\n", - cmdStr(req->cmd), req->z, req->x, req->y, req->path); + //fprintf(stderr, "%s fd(%d) z(%d), x(%d), y(%d), path(%s)\n", + // cmdStr(req->cmd), fd, req->z, req->x, req->y, req->path); if ((req->cmd != cmdRender) && (req->cmd != cmdDirty)) return cmdIgnore; @@ -220,24 +189,47 @@ enum protoCmd rx_request(const struct protocol *req, int fd) fprintf(stderr, "malloc failed\n"); return cmdNotDone; } + item->req = *req; + item->duplicates = NULL; + item->fd = (req->cmd == cmdRender) ? fd : FD_INVALID; + +#ifdef METATILE + /* Round down request co-ordinates to the neareast N (should be a power of 2) + * Note: request path is no longer consistent but this will be recalculated + * when the metatile is being rendered. + */ + item->mx = item->req.x & ~(METATILE-1); + item->my = item->req.y & ~(METATILE-1); +#else + item->mx = item->req.x; + item->my = item->req.y; +#endif pthread_mutex_lock(&qLock); - if (pending(item)) { + // Check for a matching request in the current rendering or dirty queues + pend = pending(item); + if (pend == cmdNotDone) { + // We found a match in the dirty queue, can not wait for it pthread_mutex_unlock(&qLock); free(item); - return cmdNotDone; // No way to wait on a pending tile + return cmdNotDone; + } + if (pend == cmdIgnore) { + // Found a match in render queue, item added as duplicate + pthread_mutex_unlock(&qLock); + return cmdIgnore; } + // New request, add it to render or dirty queue if ((req->cmd == cmdRender) && (reqNum < REQ_LIMIT)) { list = &reqHead; reqNum++; - item->fd = fd; } else if (dirtyNum < DIRTY_LIMIT) { list = &dirtyHead; dirtyNum++; - item->fd = FD_INVALID; // No response after render + item->fd = FD_INVALID; // No response after render } if (list) { diff --git a/dir_utils.c b/dir_utils.c new file mode 100644 index 0000000..0bb751c --- /dev/null +++ b/dir_utils.c @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include +#include +#include + + +#include "protocol.h" +#include "render_config.h" +#include "dir_utils.h" + +// Build parent directories for the specified file name +// Note: the part following the trailing / is ignored +// e.g. mkdirp("/a/b/foo.png") == shell mkdir -p /a/b +int mkdirp(const char *path) { + struct stat s; + char tmp[PATH_MAX]; + char *p; + + strncpy(tmp, path, sizeof(tmp)); + + // Look for parent directory + p = strrchr(tmp, '/'); + if (!p) + return 0; + + *p = '\0'; + + if (!stat(tmp, &s)) + return !S_ISDIR(s.st_mode); + *p = '/'; + // Walk up the path making sure each element is a directory + p = tmp; + if (!*p) + return 0; + p++; // Ignore leading / + while (*p) { + if (*p == '/') { + *p = '\0'; + if (!stat(tmp, &s)) { + if (!S_ISDIR(s.st_mode)) + return 1; + } else if (mkdir(tmp, 0777)) + return 1; + *p = '/'; + } + p++; + } + return 0; +} + + + +/* File path hashing. Used by both mod_tile and render daemon + * The two must both agree on the file layout for meta-tiling + * to work + */ + +const char *xyz_to_path(char *path, size_t len, int x, int y, int z) +{ +#ifdef DIRECTORY_HASH + // Directory hashing is optimised for z18 max (2^18 tiles split into 2 levels of 2^9) + //snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d/%03d/%03d/%03d/%03d.png", z, + // x/512 x%512, y/512, y%512); + + // We attempt to cluseter the tiles so that a 16x16 square of tiles will be in a single directory + // Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates + // 4 bits of x & y are used per byte of output + unsigned char i, hash[5]; + + for (i=0; i<5; i++) { + hash[i] = ((x & 0x0f) << 4) | (y & 0x0f); + x >>= 4; + y >>= 4; + } + snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d/%u/%u/%u/%u/%u.png", z, hash[4], hash[3], hash[2], hash[1], hash[0]); +#else + snprintf(path, PATH_MAX, WWW_ROOT TILE_PATH "/%d/%d/%d.png", z, x, y); +#endif + return path + strlen(WWW_ROOT); +} diff --git a/dir_utils.h b/dir_utils.h new file mode 100644 index 0000000..60be1d8 --- /dev/null +++ b/dir_utils.h @@ -0,0 +1,28 @@ +#ifndef DIR_UTILS_H +#define DIR_UTILS_H + +#include + +#ifdef __cplusplus + extern "C" { +#endif + + +/* Build parent directories for the specified file name + * Note: the part following the trailing / is ignored + * e.g. mkdirp("/a/b/foo.png") == shell mkdir -p /a/b + */ +int mkdirp(const char *path); + +/* File path hashing. Used by both mod_tile and render daemon + * The two must both agree on the file layout for meta-tiling + * to work + */ +const char *xyz_to_path(char *path, size_t len, int x, int y, int z); + +#ifdef __cplusplus + } +#endif + + +#endif diff --git a/gen_tile.cpp b/gen_tile.cpp index f9b199f..09a62fb 100644 --- a/gen_tile.cpp +++ b/gen_tile.cpp @@ -17,9 +17,8 @@ #include "gen_tile.h" #include "protocol.h" - - -#undef USE_RENDER_OFFSET +#include "render_config.h" +#include "dir_utils.h" using namespace mapnik; @@ -27,37 +26,23 @@ using namespace mapnik; #define DEG_TO_RAD (M_PIl/180) #define RAD_TO_DEG (180/M_PIl) +#ifdef METATILE +#define RENDER_SIZE (256 * (METATILE + 1)) +#else +#define RENDER_SIZE (512) +#endif + +pthread_mutex_t map_lock; + + static const int minZoom = 0; static const int maxZoom = 18; -static const char *mapfile = "/home/jburgess/osm/svn.openstreetmap.org/applications/rendering/mapnik/osm-jb-merc.xml"; +//static const char *mapfile = "/home/jburgess/osm/svn.openstreetmap.org/applications/rendering/mapnik/osm-jb-merc.xml"; +static const char *mapfile = "/home/jburgess/osm/svn.openstreetmap.org/applications/rendering/mapnik/osm-local-fast-sphere.xml"; +//static const char *mapfile = "/home/jburgess/osm/svn.openstreetmap.org/applications/rendering/mapnik/osm-local-fast.xml"; -#if 0 -static void postProcess(const char *path) -{ - // Convert the 32bit RGBA image to one with indexed colours - // TODO: Ideally this would work on the Mapnik Image32 instead of requiring the intermediate image - // Or have a post-process thread with queueing - - char tmp[PATH_MAX]; - Magick::Image image; - snprintf(tmp, sizeof(tmp), "%s.tmp", path); - - try { - image.read(path); - - image.matte(0); - image.quantizeDither(0); - image.quantizeColors(255); - image.quantize(); - image.modulusDepth(8); - image.write(tmp); - rename(tmp, path); - } - catch( Magick::Exception &error_ ) { - std::cerr << "Caught exception: " << error_.what() << std::endl; - } -} -#endif +//static projection prj("+proj=merc +datum=WGS84 +k=1.0 +units=m +over +no_defs"); +static projection prj("+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over"); static double minmax(double a, double b, double c) @@ -139,10 +124,67 @@ static void load_fonts(const char *font_dir, int recurse) } static GoogleProjection gprj(maxZoom+1); -static projection prj("+proj=merc +datum=WGS84"); -static enum protoCmd render(Map &m, Image32 &buf, int x, int y, int z, const char *filename) +#ifdef METATILE +static enum protoCmd render(Map &m, int x, int y, int z, unsigned int size) { + int render_size = 256 * (size + 1); + double p0x = x * 256; + double p0y = (y + size) * 256; + double p1x = (x + size) * 256; + double p1y = y * 256; + + //std::cout << "META TILE " << z << " " << x << "-" << x+size-1 << " " << y << "-" << y+size-1 << "\n"; + + // FIXME: Without a lock around the projection and rendering we seem + // to get corrupted tiles. The current evidence is pointing towards + // a problem with libproj + pthread_mutex_lock(&map_lock); + + gprj.fromPixelToLL(p0x, p0y, z); + gprj.fromPixelToLL(p1x, p1y, z); + + prj.forward(p0x, p0y); + prj.forward(p1x, p1y); + + Envelope bbox(p0x, p0y, p1x,p1y); + bbox.height(bbox.height() * (size+1.0)/size); + bbox.width(bbox.width() * (size+1.0)/size); + m.resize(render_size, render_size); + m.zoomToBox(bbox); + //m.zoom(size+1); + + Image32 buf(render_size, render_size); + agg_renderer ren(m,buf); + ren.apply(); + + pthread_mutex_unlock(&map_lock); + + // Split the meta tile into an NxN grid of tiles + unsigned int xx, yy; + for (yy = 0; yy < size; yy++) { + for (xx = 0; xx < size; xx++) { + int yoff = 128 + yy * 256; + int xoff = 128 + xx * 256; + image_view vw(xoff, yoff, 256, 256, buf.data()); + + char filename[PATH_MAX]; + char tmp[PATH_MAX]; + xyz_to_path(filename, sizeof(filename), x+xx, y+yy, z); + mkdirp(filename); + snprintf(tmp, sizeof(tmp), "%s.tmp", filename); + //std::cout << "Render " << z << " " << x << "(" << xx << ") " << y << "(" << yy << ") " << filename << "\n"; + save_to_file(tmp,"png256", vw); + rename(tmp, filename); + } + } + std::cout << "DONE TILE " << z << " " << x << "-" << x+size-1 << " " << y << "-" << y+size-1 << "\n"; + return cmdDone; // OK +} +#else +static enum protoCmd render(Map &m, int x, int y, int z, const char *filename) +{ + double p0x = x * 256.0; double p0y = (y + 1) * 256.0; double p1x = (x + 1) * 256.0; @@ -158,20 +200,19 @@ static enum protoCmd render(Map &m, Image32 &buf, int x, int y, int z, const cha bbox.width(bbox.width() * 2); bbox.height(bbox.height() * 2); m.zoomToBox(bbox); -#ifdef USE_RENDER_OFFSET - agg_renderer ren(m,buf, 128,128); - ren.apply(); - buf.saveToFile(filename,"png256"); -#else + + Image32 buf(RENDER_SIZE, RENDER_SIZE); agg_renderer ren(m,buf); ren.apply(); + image_view vw(128,128,256,256, buf.data()); save_to_file(filename,"png256", vw); -#endif + + return cmdDone; // OK } +#endif -pthread_mutex_t map_lock; void render_init(void) { @@ -185,26 +226,28 @@ void render_init(void) void *render_thread(__attribute__((unused)) void *unused) { - Map m(2 * 256, 2 * 256); -#ifdef USE_RENDER_OFFSET - Image32 buf(256, 256); -#else - Image32 buf(512, 512); -#endif + Map m(RENDER_SIZE, RENDER_SIZE); + pthread_mutex_lock(&map_lock); load_map(m,mapfile); + pthread_mutex_unlock(&map_lock); while (1) { enum protoCmd ret; struct item *item = fetch_request(); if (item) { struct protocol *req = &item->req; - //pthread_mutex_lock(&map_lock); - ret = render(m, buf, req->x, req->y, req->z, req->path); - //pthread_mutex_unlock(&map_lock); - //postProcess(req->path); +#ifdef METATILE + // At very low zoom the whole world may be smaller than METATILE + unsigned int size = MIN(METATILE, 1 << req->z); + //pthread_mutex_lock(&map_lock); + ret = render(m, item->mx, item->my, req->z, size); + //pthread_mutex_unlock(&map_lock); + +#else + ret = render(m, req->x, req->y, req->z, req->path); +#endif send_response(item, ret); - delete_request(item); } else sleep(1); // TODO: Use an event to indicate there are new requests } diff --git a/gen_tile.h b/gen_tile.h index 7d09ca7..d88eff4 100644 --- a/gen_tile.h +++ b/gen_tile.h @@ -11,7 +11,9 @@ struct item { struct item *next; struct item *prev; struct protocol req; + int mx, my; int fd; + struct item *duplicates; }; //int render(Map &m, int x, int y, int z, const char *filename); diff --git a/mod_tile.c b/mod_tile.c index de6566c..4a255dc 100644 --- a/mod_tile.c +++ b/mod_tile.c @@ -42,42 +42,7 @@ module AP_MODULE_DECLARE_DATA tile_module; #include "gen_tile.h" #include "protocol.h" - -#define MAX_ZOOM 18 -// MAX_SIZE is the biggest file which we will return to the user -#define MAX_SIZE (1 * 1024 * 1024) -// IMG_PATH must have blank.png etc. -#define WWW_ROOT "/var/www/html" -#define IMG_PATH "/img" -// TILE_PATH is where Openlayers with try to fetch the "z/x/y.png" tiles from -#define TILE_PATH "/osm_tiles2" -// With directory hashing enabled we rewrite the path so that tiles are really stored here instead -#define DIRECTORY_HASH -#define HASH_PATH "/direct" -// MAX_LOAD_OLD: if tile is out of date, don't re-render it if past this load threshold (users gets old tile) -#define MAX_LOAD_OLD 5 -// MAX_LOAD_OLD: if tile is missing, don't render it if past this load threshold (user gets 404 error) -#define MAX_LOAD_MISSING 10 -// MAX_LOAD_ANY: give up serving any data if beyond this load (user gets 404 error) -#define MAX_LOAD_ANY 100 -// Maximum tile age in seconds -// TODO: this mechanism should really be a hard cutoff on planet update time. -#define MAX_AGE (48 * 60 * 60) - -// Typical interval between planet imports, used as basis for tile expiry times -#define PLANET_INTERVAL (7 * 24 * 60 * 60) - -// Planet import should touch this file when complete -#define PLANET_TIMESTAMP "/tmp/planet-import-complete" - -// Timeout before giving for a tile to be rendered -#define REQUEST_TIMEOUT (5) -#define FD_INVALID (-1) - - -#define MIN(x,y) ((x)<(y)?(x):(y)) -#define MAX(x,y) ((x)>(y)?(x):(y)) - +#include "render_config.h" enum tileState { tileMissing, tileOld, tileCurrent }; @@ -449,7 +414,7 @@ static int tile_handler(request_rec *r) // Generate the tile filename rel_path = xyz_to_path(abs_path, sizeof(abs_path), x, y, z); //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "abs_path(%s), rel_path(%s)", abs_path, rel_path); - + // Validate tile co-ordinates oob = (z < 0 || z > MAX_ZOOM); if (!oob) { diff --git a/render_config.h b/render_config.h new file mode 100644 index 0000000..3f74c81 --- /dev/null +++ b/render_config.h @@ -0,0 +1,51 @@ +#ifndef RENDER_CONFIG_H +#define RENDER_CONFIG_H + +#define MAX_ZOOM 18 +// MAX_SIZE is the biggest file which we will return to the user +#define MAX_SIZE (1 * 1024 * 1024) +// IMG_PATH must have blank.png etc. +#define WWW_ROOT "/var/www/html" +#define IMG_PATH "/img" +// TILE_PATH is where Openlayers with try to fetch the "z/x/y.png" tiles from +#define TILE_PATH "/osm_tiles2" +// With directory hashing enabled we rewrite the path so that tiles are really stored here instead +#define DIRECTORY_HASH +#define HASH_PATH "/direct" +// MAX_LOAD_OLD: if tile is out of date, don't re-render it if past this load threshold (users gets old tile) +#define MAX_LOAD_OLD 5 +// MAX_LOAD_OLD: if tile is missing, don't render it if past this load threshold (user gets 404 error) +#define MAX_LOAD_MISSING 10 +// MAX_LOAD_ANY: give up serving any data if beyond this load (user gets 404 error) +#define MAX_LOAD_ANY 100 +// Maximum tile age in seconds +// TODO: this mechanism should really be a hard cutoff on planet update time. +#define MAX_AGE (48 * 60 * 60) + +// Typical interval between planet imports, used as basis for tile expiry times +#define PLANET_INTERVAL (7 * 24 * 60 * 60) + +// Planet import should touch this file when complete +#define PLANET_TIMESTAMP "/tmp/planet-import-complete" + +// Timeout before giving for a tile to be rendered +#define REQUEST_TIMEOUT (5) +#define FD_INVALID (-1) + + +#define MIN(x,y) ((x)<(y)?(x):(y)) +#define MAX(x,y) ((x)>(y)?(x):(y)) + +#define QUEUE_MAX (32) +#define MAX_CONNECTIONS (256) + +#define REQ_LIMIT (32) +#define DIRTY_LIMIT (1000 * 1000) +#define NUM_THREADS (4) + +// Use this to enable meta-tiles which will render NxN tiles at once +// Note: This should be a power of 2 (2, 4, 8, 16 ...) +#define METATILE (8) +//#undef METATILE + +#endif diff --git a/render_list.c b/render_list.c new file mode 100644 index 0000000..861b8e8 --- /dev/null +++ b/render_list.c @@ -0,0 +1,152 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gen_tile.h" +#include "protocol.h" +#include "render_config.h" +#include "dir_utils.h" + +#define DEG_TO_RAD (M_PIl/180) +#define RAD_TO_DEG (180/M_PIl) + +static const int minZoom = 0; +static const int maxZoom = 18; + + +void display_rate(struct timeval start, struct timeval end, int num) +{ + int d_s, d_us; + float sec; + + d_s = end.tv_sec - start.tv_sec; + d_us = end.tv_usec - start.tv_usec; + + sec = d_s + d_us / 1000000.0; + + printf("Rendered %d tiles in %.2f seconds (%.2f tiles/s)\n", num, sec, num / sec); + fflush(NULL); +} + + +int process_loop(int fd, int x, int y, int z) +{ + struct protocol cmd, rsp; + //struct pollfd fds[1]; + int ret = 0; + + bzero(&cmd, sizeof(cmd)); + + cmd.ver = 1; + cmd.cmd = cmdRender; + cmd.z = z; + cmd.x = x; + cmd.y = y; + //strcpy(cmd.path, "/tmp/foo.png"); + + //printf("Sending request\n"); + ret = send(fd, &cmd, sizeof(cmd), 0); + if (ret != sizeof(cmd)) { + perror("send error"); + } + //printf("Waiting for response\n"); + bzero(&rsp, sizeof(rsp)); + ret = recv(fd, &rsp, sizeof(rsp), 0); + if (ret != sizeof(rsp)) { + perror("recv error"); + return 0; + } + //printf("Got response\n"); + + if (!ret) + perror("Socket send error"); + return ret; +} + + +int main(int argc, char **argv) +{ + const char *spath = RENDER_SOCKET; + int fd; + struct sockaddr_un addr; + int ret=0; + int x, y, z; + char name[PATH_MAX]; + struct timeval start, end; + int num_render = 0, num_all = 0; + + fprintf(stderr, "Rendering client\n"); + + fd = socket(PF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + fprintf(stderr, "failed to create unix socket\n"); + exit(2); + } + + bzero(&addr, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, spath, sizeof(addr.sun_path)); + + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + fprintf(stderr, "socket connect failed for: %s\n", spath); + close(fd); + exit(3); + } + + gettimeofday(&start, NULL); + + while(!feof(stdin)) { + struct stat s; + int n = fscanf(stdin, "%d %d %d %*s %*s", &x, &y, &z); + + if (n != 3) { + // Discard input line + char tmp[1024]; + char *r = fgets(tmp, sizeof(tmp), stdin); + if (!r) + continue; + // fprintf(stderr, "bad line %d: %s", num_all, tmp); + continue; + } +// printf("got: x(%d) y(%d) z(%d)\n", x, y, z); + + num_all++; + xyz_to_path(name, sizeof(name), x, y, z); + + if (stat(name, &s) < 0) { + // Assume error is that file doesn't exist + // so render it + ret = process_loop(fd, x, y, z); + num_render++; + if (!(num_render % 10)) { + gettimeofday(&end, NULL); + printf("\n"); + printf("Meta tiles rendered: "); + display_rate(start, end, num_render); + printf("Total tiles rendered: "); + display_rate(start, end, num_render * METATILE * METATILE); + printf("Total tiles handled from input: "); + display_rate(start, end, num_all); + } + } + } + gettimeofday(&end, NULL); + printf("\nTotal for all tiles rendered\n"); + printf("Meta tiles rendered: "); + display_rate(start, end, num_render); + printf("Total tiles rendered: "); + display_rate(start, end, num_render * METATILE * METATILE); + printf("Total tiles handled: "); + display_rate(start, end, num_all); + + close(fd); + return ret; +} diff --git a/speedtest.cpp b/speedtest.cpp new file mode 100644 index 0000000..ee11018 --- /dev/null +++ b/speedtest.cpp @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gen_tile.h" +#include "protocol.h" +#include "render_config.h" +#include "dir_utils.h" + +#define DEG_TO_RAD (M_PIl/180) +#define RAD_TO_DEG (180/M_PIl) + +static const int minZoom = 0; +static const int maxZoom = 18; + +#if 1 +static double boundx0=-0.5; +static double boundy0=51.25; +static double boundx1=0.5; +static double boundy1=51.75; +#endif +#if 0 +// bbox = (-6.0, 50.0,3.0,58.0) +static double boundx0=-6.0; +static double boundy0=50.0; +static double boundx1=3.0; +static double boundy1=58.0; +#endif +#if 0 +// UK: 49.7,-7.6, 58.8, 3.2 +static double boundx0=-7.6; +static double boundy0=49.7; +static double boundx1=3.2; +static double boundy1=58.8; +#endif + + +static double minmax(double a, double b, double c) +{ +#define MIN(x,y) ((x)<(y)?(x):(y)) +#define MAX(x,y) ((x)>(y)?(x):(y)) + a = MAX(a,b); + a = MIN(a,c); + return a; +} + +class GoogleProjection +{ + double *Ac, *Bc, *Cc, *zc; + + public: + GoogleProjection(int levels=18) { + Ac = new double[levels]; + Bc = new double[levels]; + Cc = new double[levels]; + zc = new double[levels]; + int d, c = 256; + for (d=0; dver, req->cmd, req->z, req->x, req->y); + return 0; +} + +int process_loop(int fd, int x, int y, int z) +{ + struct protocol cmd, rsp; + //struct pollfd fds[1]; + int ret = 0; + + bzero(&cmd, sizeof(cmd)); + + cmd.ver = 1; + cmd.cmd = cmdRender; + cmd.z = z; + cmd.x = x; + cmd.y = y; + //strcpy(cmd.path, "/tmp/foo.png"); + + //printf("Sending request\n"); + ret = send(fd, &cmd, sizeof(cmd), 0); + if (ret != sizeof(cmd)) { + perror("send error"); + } + //printf("Waiting for response\n"); + bzero(&rsp, sizeof(rsp)); + ret = recv(fd, &rsp, sizeof(rsp), 0); + if (ret != sizeof(rsp)) { + perror("recv error"); + return 0; + } + //printf("Got response\n"); + + if (!ret) + perror("Socket send error"); + return ret; +} + + +int main(int argc, char **argv) +{ + const char *spath = RENDER_SOCKET; + int fd; + struct sockaddr_un addr; + int ret=0; + int z; + char name[PATH_MAX]; + struct timeval start, end; + struct timeval start_all, end_all; + int num, num_all = 0; + + + fprintf(stderr, "Rendering client\n"); + + fd = socket(PF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + fprintf(stderr, "failed to create unix socket\n"); + exit(2); + } + + bzero(&addr, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, spath, sizeof(addr.sun_path)); + + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + fprintf(stderr, "socket connect failed for: %s\n", spath); + close(fd); + exit(3); + } + + // Render something to counter act the startup costs + // of obtaining the Postgis table extents + + printf("Initial startup costs\n"); + gettimeofday(&start, NULL); + process_loop(fd, 0,0,0); + gettimeofday(&end, NULL); + display_rate(start, end, 1); + + gettimeofday(&start_all, NULL); + + for (z=minZoom; z<=maxZoom; z++) { + double px0 = boundx0; + double py0 = boundy1; + double px1 = boundx1; + double py1 = boundy0; + gprj.fromLLtoPixel(px0, py0, z); + gprj.fromLLtoPixel(px1, py1, z); + + int x, xmin, xmax; + xmin = (int)(px0/256.0); + xmax = (int)(px1/256.0); + + int y, ymin, ymax; + ymin = (int)(py0/256.0); + ymax = (int)(py1/256.0); + + num = (xmax - xmin + 1) * (ymax - ymin + 1); +// if (!num) { +// printf("No tiles at zoom(%d)\n", z); +// continue; +// } + + printf("\nZoom(%d) Now rendering %d tiles\n", z, num); + num_all += num; + gettimeofday(&start, NULL); + + for (x=xmin; x<=xmax; x++) { + for (y=ymin; y<=ymax; y++) { + struct stat s; + xyz_to_path(name, sizeof(name), x, y, z); + if (stat(name, &s) < 0) { + // File doesn't exist + ret = process_loop(fd, x, y, z); + } + //printf("."); + fflush(NULL); + } + } + //printf("\n"); + gettimeofday(&end, NULL); + display_rate(start, end, num); + } + gettimeofday(&end_all, NULL); + printf("\nTotal for all tiles rendered\n"); + display_rate(start_all, end_all, num_all); + + close(fd); + return ret; +}