#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gen_tile.h" #include "render_config.h" #include "daemon.h" #include "protocol.h" #include "dir_utils.h" #include "store.h" #ifdef HTCP_EXPIRE_CACHE #include #include #endif #if MAPNIK_VERSION >= 800 #include #define Image32 image_32 #define ImageData32 image_data_32 #define Envelope box2d #define zoomToBox zoom_to_box #else #include #endif using namespace mapnik; #ifndef DEG_TO_RAD #define DEG_TO_RAD (M_PI/180) #endif #ifndef RAD_TO_DEG #define RAD_TO_DEG (180/M_PI) #endif #ifdef METATILE #define RENDER_SIZE (256 * (METATILE + 1)) #else #define RENDER_SIZE (512) #endif static const int minZoom = 0; static const int maxZoom = MAX_ZOOM; typedef struct { char xmlname[XMLCONFIG_MAX]; char xmlfile[PATH_MAX]; char tile_dir[PATH_MAX]; Map map; projection prj; char xmluri[PATH_MAX]; char host[PATH_MAX]; char htcphost[PATH_MAX]; int htcpsock; int ok; } xmlmapconfig; class SphericalProjection { double *Ac, *Bc, *Cc, *zc; public: SphericalProjection(int levels=MAX_ZOOM) { Ac = new double[levels]; Bc = new double[levels]; Cc = new double[levels]; zc = new double[levels]; int d, c = 256; for (d=0; d(y)?(x):(y)) a = MAX(a,b); a = MIN(a,c); return a; } }; static SphericalProjection tiling(maxZoom+1); static void load_fonts(const char *font_dir, int recurse) { DIR *fonts = opendir(font_dir); struct dirent *entry; char path[PATH_MAX]; // FIXME: Eats lots of stack space when recursive if (!fonts) { syslog(LOG_CRIT, "Unable to open font directory: %s", font_dir); return; } while ((entry = readdir(fonts))) { struct stat b; char *p; if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; snprintf(path, sizeof(path), "%s/%s", font_dir, entry->d_name); if (stat(path, &b)) continue; if (S_ISDIR(b.st_mode)) { if (recurse) load_fonts(path, recurse); continue; } p = strrchr(path, '.'); if (p && !strcmp(p, ".ttf")) { syslog(LOG_DEBUG, "DEBUG: Loading font: %s", path); freetype_engine::register_font(path); } } closedir(fonts); } #ifdef HTCP_EXPIRE_CACHE /** * This function sends a HTCP cache clr request for a given * URL. * RFC for HTCP can be found at http://www.htcp.org/ */ void cache_expire_url(int sock, char * url) { char * buf; if (sock < 0) { return; } int idx = 0; int url_len; url_len = strlen(url); buf = (char *) malloc(12 + 22 + url_len); if (!buf) { return; } idx = 0; //16 bit: Overall length of the datagram packet, including this header *((uint16_t *) (&buf[idx])) = htons(12 + 22 + url_len); idx += 2; //HTCP version. Currently at 0.0 buf[idx++] = 0; //Major version buf[idx++] = 0; //Minor version //Length of HTCP data, including this field *((uint16_t *) (&buf[idx])) = htons(8 + 22 + url_len); idx += 2; //HTCP opcode CLR=4 buf[idx++] = 4; //Reserved buf[idx++] = 0; //32 bit transaction id; *((uint32_t *) (&buf[idx])) = htonl(255); idx += 4; buf[idx++] = 0; buf[idx++] = 0; //HTCP reason //Length of the Method string *((uint16_t *) (&buf[idx])) = htons(4); idx += 2; ///Method string memcpy(&buf[idx], "HEAD", 4); idx += 4; //Length of the url string *((uint16_t *) (&buf[idx])) = htons(url_len); idx += 2; //Url string memcpy(&buf[idx], url, url_len); idx += url_len; //Length of version string *((uint16_t *) (&buf[idx])) = htons(8); idx += 2; //version string memcpy(&buf[idx], "HTTP/1.1", 8); idx += 8; //Length of request headers. Currently 0 as we don't have any headers to send *((uint16_t *) (&buf[idx])) = htons(0); if (send(sock, (void *) buf, (12 + 22 + url_len), 0) < (12 + 22 + url_len)) { syslog(LOG_ERR, "Failed to send HTCP purge for %s\n", url); }; free(buf); } void cache_expire(int sock, char * host, char * uri, int x, int y, int z) { if (sock < 0) { return; } char * url = (char *)malloc(1024); sprintf(url,"http://%s%s%i/%i/%i.png", host, uri, z,x,y); cache_expire_url(sock, url); free(url); } int init_cache_expire(char * htcphost) { struct addrinfo hints; struct addrinfo *result, *rp; int sfd, s; /* Obtain address(es) matching host/port */ memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */ hints.ai_flags = 0; hints.ai_protocol = 0; /* Any protocol */ s = getaddrinfo(htcphost, HTCP_EXPIRE_CACHE_PORT, &hints, &result); if (s != 0) { syslog(LOG_ERR, "Failed to lookup HTCP cache host: %s", gai_strerror(s)); return -1; } /* getaddrinfo() returns a list of address structures. Try each address until we successfully connect(2). If socket(2) (or connect(2)) fails, we (close the socket and) try the next address. */ for (rp = result; rp != NULL; rp = rp->ai_next) { sfd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); if (sfd == -1) continue; if (connect(sfd, rp->ai_addr, rp->ai_addrlen) != -1) break; /* Success */ close(sfd); } if (rp == NULL) { /* No address succeeded */ syslog(LOG_ERR, "Failed to create HTCP cache socket"); return -1; } freeaddrinfo(result); /* No longer needed */ return sfd; } #endif #ifdef METATILE class metaTile { public: metaTile(const std::string &xmlconfig, int x, int y, int z): x_(x), y_(y), z_(z), xmlconfig_(xmlconfig) { clear(); } void clear() { for (int x = 0; x < METATILE; x++) for (int y = 0; y < METATILE; y++) tile[x][y] = ""; } void set(int x, int y, const std::string &data) { tile[x][y] = data; } const std::string get(int x, int y) { return tile[x][y]; } // Returns the offset within the meta-tile index table int xyz_to_meta_offset(int x, int y, int z) { unsigned char mask = METATILE - 1; return (x & mask) * METATILE + (y & mask); } void save(const char *tile_dir) { int ox, oy, limit; size_t offset; struct meta_layout m; char meta_path[PATH_MAX]; struct entry offsets[METATILE * METATILE]; memset(&m, 0, sizeof(m)); memset(&offsets, 0, sizeof(offsets)); xyz_to_meta(meta_path, sizeof(meta_path), tile_dir, xmlconfig_.c_str(), x_, y_, z_); std::stringstream ss; ss << std::string(meta_path) << "." << pthread_self(); std::string tmp(ss.str()); if (mkdirp(tmp.c_str())) throw std::runtime_error("An error when creating tile directory"); try { std::ofstream file; file.exceptions(std::ios_base::failbit | std::ios_base::badbit); file.open(tmp.c_str(), std::ios::out | std::ios::binary | std::ios::trunc); // Create and write header m.count = METATILE * METATILE; memcpy(m.magic, META_MAGIC, strlen(META_MAGIC)); m.x = x_; m.y = y_; m.z = z_; file.write((const char *)&m, sizeof(m)); offset = header_size; limit = (1 << z_); limit = MIN(limit, METATILE); // Generate offset table for (ox=0; ox < limit; ox++) { for (oy=0; oy < limit; oy++) { int mt = xyz_to_meta_offset(x_ + ox, y_ + oy, z_); offsets[mt].offset = offset; offsets[mt].size = tile[ox][oy].size(); offset += offsets[mt].size; } } file.write((const char *)&offsets, sizeof(offsets)); // Write tiles for (ox=0; ox < limit; ox++) { for (oy=0; oy < limit; oy++) { file.write((const char *)tile[ox][oy].data(), tile[ox][oy].size()); } } file.close(); rename(tmp.c_str(), meta_path); } catch (std::ios_base::failure) { // Remove the temporary file unlink(tmp.c_str()); throw std::runtime_error("An error occured when writing a metatile\n"); } //printf("Produced .meta: %s\n", meta_path); } #ifdef HTCP_EXPIRE_CACHE void expire_tiles(int sock, char * host, char * uri) { if (sock < 0) { return; } syslog(LOG_INFO, "Purging metatile via HTCP cache expiry"); int ox, oy; int limit = (1 << z_); limit = MIN(limit, METATILE); // Generate offset table for (ox=0; ox < limit; ox++) { for (oy=0; oy < limit; oy++) { cache_expire(sock, host, uri, (x_ + ox), (y_ + oy), z_); } } } #endif int x_, y_, z_; std::string xmlconfig_; std::string tile[METATILE][METATILE]; static const int header_size = sizeof(struct meta_layout) + (sizeof(struct entry) * (METATILE * METATILE)); }; static enum protoCmd render(Map &m, char *xmlname, projection &prj, int x, int y, int z, unsigned int size, metaTile &tiles) { int render_size = 256 * size; double p0x = x * 256; double p0y = (y + size) * 256; double p1x = (x + size) * 256; double p1y = y * 256; //std::cout << "META TILE " << z << " " << x << "-" << x+size-1 << " " << y << "-" << y+size-1 << "\n"; tiling.fromPixelToLL(p0x, p0y, z); tiling.fromPixelToLL(p1x, p1y, z); prj.forward(p0x, p0y); prj.forward(p1x, p1y); Envelope bbox(p0x, p0y, p1x,p1y); m.resize(render_size, render_size); m.zoomToBox(bbox); m.set_buffer_size(128); //m.zoom(size+1); Image32 buf(render_size, render_size); agg_renderer ren(m,buf); ren.apply(); // Split the meta tile into an NxN grid of tiles unsigned int xx, yy; for (yy = 0; yy < size; yy++) { for (xx = 0; xx < size; xx++) { image_view vw(xx * 256, yy * 256, 256, 256, buf.data()); tiles.set(xx, yy, save_to_string(vw, "png256")); } } // std::cout << "DONE TILE " << xmlname << " " << z << " " << x << "-" << x+size-1 << " " << y << "-" << y+size-1 << "\n"; syslog(LOG_DEBUG, "DEBUG: DONE TILE %s %d %d-%d %d-%d", xmlname, z, x, x+size-1, y, y+size-1); return cmdDone; // OK } #else static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z) { char filename[PATH_MAX]; char tmp[PATH_MAX]; double p0x = x * 256.0; double p0y = (y + 1) * 256.0; double p1x = (x + 1) * 256.0; double p1y = y * 256.0; tiling.fromPixelToLL(p0x, p0y, z); tiling.fromPixelToLL(p1x, p1y, z); prj.forward(p0x, p0y); prj.forward(p1x, p1y); Envelope bbox(p0x, p0y, p1x,p1y); bbox.width(bbox.width() * 2); bbox.height(bbox.height() * 2); m.zoomToBox(bbox); Image32 buf(RENDER_SIZE, RENDER_SIZE); agg_renderer ren(m,buf); ren.apply(); xyz_to_path(filename, sizeof(filename), tile_dir, xmlname, x, y, z); if (mkdirp(filename)) return cmdNotDone; snprintf(tmp, sizeof(tmp), "%s.tmp", filename); image_view vw(128, 128, 256, 256, buf.data()); //std::cout << "Render " << z << " " << x << " " << y << " " << filename << "\n"; save_to_file(vw, tmp, "png256"); if (rename(tmp, filename)) { perror(tmp); return cmdNotDone; } return cmdDone; // OK } #endif void render_init(const char *plugins_dir, const char* font_dir, int font_dir_recurse) { datasource_cache::instance()->register_datasources(plugins_dir); load_fonts(font_dir, font_dir_recurse); } void *render_thread(void * arg) { xmlconfigitem * parentxmlconfig = (xmlconfigitem *)arg; xmlmapconfig maps[XMLCONFIGS_MAX]; int i,iMaxConfigs; for (iMaxConfigs = 0; iMaxConfigs < XMLCONFIGS_MAX; ++iMaxConfigs) { if (parentxmlconfig[iMaxConfigs].xmlname[0] == 0 || parentxmlconfig[iMaxConfigs].xmlfile[0] == 0) break; strcpy(maps[iMaxConfigs].xmlname, parentxmlconfig[iMaxConfigs].xmlname); strcpy(maps[iMaxConfigs].xmlfile, parentxmlconfig[iMaxConfigs].xmlfile); strcpy(maps[iMaxConfigs].tile_dir, parentxmlconfig[iMaxConfigs].tile_dir); maps[iMaxConfigs].map = Map(RENDER_SIZE, RENDER_SIZE); maps[iMaxConfigs].ok = 1; try { load_map(maps[iMaxConfigs].map, maps[iMaxConfigs].xmlfile); } catch (mapnik::config_error &ex) { syslog(LOG_ERR, "An error occurred while loading the map layer '%s': %s", maps[iMaxConfigs].xmlname, ex.what()); maps[iMaxConfigs].ok = 0; } maps[iMaxConfigs].prj = projection(maps[iMaxConfigs].map.srs()); #ifdef HTCP_EXPIRE_CACHE strcpy(maps[iMaxConfigs].xmluri, parentxmlconfig[iMaxConfigs].xmluri); strcpy(maps[iMaxConfigs].host, parentxmlconfig[iMaxConfigs].host); strcpy(maps[iMaxConfigs].htcphost, parentxmlconfig[iMaxConfigs].htcpip); if (strlen(maps[iMaxConfigs].htcphost) > 0) { maps[iMaxConfigs].htcpsock = init_cache_expire( maps[iMaxConfigs].htcphost); if (maps[iMaxConfigs].htcpsock > 0) { syslog(LOG_INFO, "Successfully opened socket for HTCP cache expiry"); } else { syslog(LOG_ERR, "Failed to opened socket for HTCP cache expiry"); } } else { maps[iMaxConfigs].htcpsock = -1; } #endif } while (1) { enum protoCmd ret; struct item *item = fetch_request(); if (item) { struct protocol *req = &item->req; #ifdef METATILE // At very low zoom the whole world may be smaller than METATILE unsigned int size = MIN(METATILE, 1 << req->z); for (i = 0; i < iMaxConfigs; ++i) { if (!strcmp(maps[i].xmlname, req->xmlname)) { metaTile tiles(req->xmlname, item->mx, item->my, req->z); if (maps[i].ok) { timeval tim; gettimeofday(&tim, NULL); long t1=tim.tv_sec*1000+(tim.tv_usec/1000); ret = render(maps[i].map, req->xmlname, maps[i].prj, item->mx, item->my, req->z, size, tiles); gettimeofday(&tim, NULL); long t2=tim.tv_sec*1000+(tim.tv_usec/1000); syslog(LOG_DEBUG, "DEBUG: DONE TILE %s %d %d-%d %d-%d in %.3lf seconds", req->xmlname, req->z, item->mx, item->mx+size-1, item->my, item->my+size-1, (t2 - t1)/1000.0); statsRenderFinish(req->z, t2 - t1); } else { syslog(LOG_ERR, "Received request for map layer '%s' which failed to load", req->xmlname); ret = cmdNotDone; } if (ret == cmdDone) { try { tiles.save(maps[i].tile_dir); } catch (...) { // Treat any error as fatal and request end of processing syslog(LOG_ERR, "Received error when writing metatile to disk, requesting exit."); ret = cmdNotDone; request_exit(); } #ifdef HTCP_EXPIRE_CACHE tiles.expire_tiles(maps[i].htcpsock,maps[i].host,maps[i].xmluri); #endif } #else ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z); #ifdef HTCP_EXPIRE_CACHE cache_expire(maps[i].htcpsock,maps[i].host, maps[i].xmluri, req->x,req->y,req->z); #endif #endif send_response(item, ret); break; } } if (i == iMaxConfigs){ syslog(LOG_ERR, "No map for: %s", req->xmlname); } } else { sleep(1); // TODO: Use an event to indicate there are new requests } } return NULL; }