VladPolskiy/openstreetmap-mod_tile-python-implementation
1
0
Fork 0
mirror of https://github.com/openstreetmap/mod_tile.git synced 2025-07-25 15:04:30 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
fa3969cd22ae767be4f37fe2a464ba0b751ae3f5
openstreetmap-mod_tile-pyth…/gen_tile.cpp

609 lines
18 KiB
C++
Raw Blame History

#include <mapnik/version.hpp>
#include <mapnik/map.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/agg_renderer.hpp>
#include <mapnik/filter_factory.hpp>
#include <mapnik/color_factory.hpp>
#include <mapnik/load_map.hpp>
#include <mapnik/image_util.hpp>
#include <iostream>
#include <fstream>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <dirent.h>
#include <syslog.h>
#include <unistd.h>
#include <pthread.h>
#include "gen_tile.h"
#include "render_config.h"
#include "daemon.h"
#include "protocol.h"
#include "dir_utils.h"
#include "store.h"
#ifdef HTCP_EXPIRE_CACHE
#include <sys/socket.h>
#include <netdb.h>
#endif
#if MAPNIK_VERSION >= 800
#include <mapnik/box2d.hpp>
#define Image32 image_32
#define ImageData32 image_data_32
#define Envelope box2d
#define zoomToBox zoom_to_box
#else
#include <mapnik/envelope.hpp>
#endif
using namespace mapnik;
#ifndef DEG_TO_RAD
#define DEG_TO_RAD (M_PI/180)
#endif
#ifndef RAD_TO_DEG
#define RAD_TO_DEG (180/M_PI)
#endif
#ifdef METATILE
#define RENDER_SIZE (256 * (METATILE + 1))
#else
#define RENDER_SIZE (512)
#endif
static const int minZoom = 0;
static const int maxZoom = MAX_ZOOM;
typedef struct {
char xmlname[XMLCONFIG_MAX];
char xmlfile[PATH_MAX];
char tile_dir[PATH_MAX];
Map map;
projection prj;
char xmluri[PATH_MAX];
char host[PATH_MAX];
char htcphost[PATH_MAX];
int htcpsock;
int ok;
} xmlmapconfig;
class SphericalProjection
{
double *Ac, *Bc, *Cc, *zc;
public:
SphericalProjection(int levels=MAX_ZOOM) {
Ac = new double[levels];
Bc = new double[levels];
Cc = new double[levels];
zc = new double[levels];
int d, c = 256;
for (d=0; d<levels; d++) {
int e = c/2;
Bc[d] = c/360.0;
Cc[d] = c/(2 * M_PI);
zc[d] = e;
Ac[d] = c;
c *=2;
}
}
void fromLLtoPixel(double &x, double &y, int zoom) {
double d = zc[zoom];
double f = minmax(sin(DEG_TO_RAD * y),-0.9999,0.9999);
x = round(d + x * Bc[zoom]);
y = round(d + 0.5*log((1+f)/(1-f))*-Cc[zoom]);
}
void fromPixelToLL(double &x, double &y, int zoom) {
double e = zc[zoom];
double g = (y - e)/-Cc[zoom];
x = (x - e)/Bc[zoom];
y = RAD_TO_DEG * ( 2 * atan(exp(g)) - 0.5 * M_PI);
}
private:
double minmax(double a, double b, double c)
{
#define MIN(x,y) ((x)<(y)?(x):(y))
#define MAX(x,y) ((x)>(y)?(x):(y))
a = MAX(a,b);
a = MIN(a,c);
return a;
}
};
static SphericalProjection tiling(maxZoom+1);
static void load_fonts(const char *font_dir, int recurse)
{
DIR *fonts = opendir(font_dir);
struct dirent *entry;
char path[PATH_MAX]; // FIXME: Eats lots of stack space when recursive
if (!fonts) {
syslog(LOG_CRIT, "Unable to open font directory: %s", font_dir);
return;
}
while ((entry = readdir(fonts))) {
struct stat b;
char *p;
if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
continue;
snprintf(path, sizeof(path), "%s/%s", font_dir, entry->d_name);
if (stat(path, &b))
continue;
if (S_ISDIR(b.st_mode)) {
if (recurse)
load_fonts(path, recurse);
continue;
}
p = strrchr(path, '.');
if (p && !strcmp(p, ".ttf")) {
syslog(LOG_DEBUG, "DEBUG: Loading font: %s", path);
freetype_engine::register_font(path);
}
}
closedir(fonts);
}
#ifdef HTCP_EXPIRE_CACHE
/**
* This function sends a HTCP cache clr request for a given
* URL.
* RFC for HTCP can be found at http://www.htcp.org/
*/
void cache_expire_url(int sock, char * url) {
char * buf;
if (sock < 0) {
return;
}
int idx = 0;
int url_len;
url_len = strlen(url);
buf = (char *) malloc(12 + 22 + url_len);
if (!buf) {
return;
}
idx = 0;
//16 bit: Overall length of the datagram packet, including this header
*((uint16_t *) (&buf[idx])) = htons(12 + 22 + url_len);
idx += 2;
//HTCP version. Currently at 0.0
buf[idx++] = 0; //Major version
buf[idx++] = 0; //Minor version
//Length of HTCP data, including this field
*((uint16_t *) (&buf[idx])) = htons(8 + 22 + url_len);
idx += 2;
//HTCP opcode CLR=4
buf[idx++] = 4;
//Reserved
buf[idx++] = 0;
//32 bit transaction id;
*((uint32_t *) (&buf[idx])) = htonl(255);
idx += 4;
buf[idx++] = 0;
buf[idx++] = 0; //HTCP reason
//Length of the Method string
*((uint16_t *) (&buf[idx])) = htons(4);
idx += 2;
///Method string
memcpy(&buf[idx], "HEAD", 4);
idx += 4;
//Length of the url string
*((uint16_t *) (&buf[idx])) = htons(url_len);
idx += 2;
//Url string
memcpy(&buf[idx], url, url_len);
idx += url_len;
//Length of version string
*((uint16_t *) (&buf[idx])) = htons(8);
idx += 2;
//version string
memcpy(&buf[idx], "HTTP/1.1", 8);
idx += 8;
//Length of request headers. Currently 0 as we don't have any headers to send
*((uint16_t *) (&buf[idx])) = htons(0);
if (send(sock, (void *) buf, (12 + 22 + url_len), 0) < (12 + 22 + url_len)) {
syslog(LOG_ERR, "Failed to send HTCP purge for %s\n", url);
};
free(buf);
}
void cache_expire(int sock, char * host, char * uri, int x, int y, int z) {
if (sock < 0) {
return;
}
char * url = (char *)malloc(1024);
sprintf(url,"http://%s%s%i/%i/%i.png", host, uri, z,x,y);
cache_expire_url(sock, url);
free(url);
}
int init_cache_expire(char * htcphost) {
struct addrinfo hints;
struct addrinfo *result, *rp;
int sfd, s;
/* Obtain address(es) matching host/port */
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */
hints.ai_flags = 0;
hints.ai_protocol = 0; /* Any protocol */
s = getaddrinfo(htcphost, HTCP_EXPIRE_CACHE_PORT, &hints, &result);
if (s != 0) {
syslog(LOG_ERR, "Failed to lookup HTCP cache host: %s", gai_strerror(s));
return -1;
}
/* getaddrinfo() returns a list of address structures.
Try each address until we successfully connect(2).
If socket(2) (or connect(2)) fails, we (close the socket
and) try the next address. */
for (rp = result; rp != NULL; rp = rp->ai_next) {
sfd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
if (sfd == -1)
continue;
if (connect(sfd, rp->ai_addr, rp->ai_addrlen) != -1)
break; /* Success */
close(sfd);
}
if (rp == NULL) { /* No address succeeded */
syslog(LOG_ERR, "Failed to create HTCP cache socket");
return -1;
}
freeaddrinfo(result); /* No longer needed */
return sfd;
}
#endif
#ifdef METATILE
class metaTile {
public:
metaTile(const std::string &xmlconfig, int x, int y, int z):
x_(x), y_(y), z_(z), xmlconfig_(xmlconfig)
{
clear();
}
void clear()
{
for (int x = 0; x < METATILE; x++)
for (int y = 0; y < METATILE; y++)
tile[x][y] = "";
}
void set(int x, int y, const std::string &data)
{
tile[x][y] = data;
}
const std::string get(int x, int y)
{
return tile[x][y];
}
// Returns the offset within the meta-tile index table
int xyz_to_meta_offset(int x, int y, int z)
{
unsigned char mask = METATILE - 1;
return (x & mask) * METATILE + (y & mask);
}
void save(const char *tile_dir)
{
int ox, oy, limit;
size_t offset;
struct meta_layout m;
char meta_path[PATH_MAX];
struct entry offsets[METATILE * METATILE];
memset(&m, 0, sizeof(m));
memset(&offsets, 0, sizeof(offsets));
xyz_to_meta(meta_path, sizeof(meta_path), tile_dir, xmlconfig_.c_str(), x_, y_, z_);
std::stringstream ss;
ss << std::string(meta_path) << "." << pthread_self();
std::string tmp(ss.str());
if (mkdirp(tmp.c_str()))
throw std::runtime_error("An error when creating tile directory");
try {
std::ofstream file;
file.exceptions(std::ios_base::failbit | std::ios_base::badbit);
file.open(tmp.c_str(), std::ios::out | std::ios::binary | std::ios::trunc);
// Create and write header
m.count = METATILE * METATILE;
memcpy(m.magic, META_MAGIC, strlen(META_MAGIC));
m.x = x_;
m.y = y_;
m.z = z_;
file.write((const char *)&m, sizeof(m));
offset = header_size;
limit = (1 << z_);
limit = MIN(limit, METATILE);
// Generate offset table
for (ox=0; ox < limit; ox++) {
for (oy=0; oy < limit; oy++) {
int mt = xyz_to_meta_offset(x_ + ox, y_ + oy, z_);
offsets[mt].offset = offset;
offsets[mt].size = tile[ox][oy].size();
offset += offsets[mt].size;
}
}
file.write((const char *)&offsets, sizeof(offsets));
// Write tiles
for (ox=0; ox < limit; ox++) {
for (oy=0; oy < limit; oy++) {
file.write((const char *)tile[ox][oy].data(), tile[ox][oy].size());
}
}
file.close();
rename(tmp.c_str(), meta_path);
} catch (std::ios_base::failure) {
// Remove the temporary file
unlink(tmp.c_str());
throw std::runtime_error("An error occured when writing a metatile\n");
}
//printf("Produced .meta: %s\n", meta_path);
}
#ifdef HTCP_EXPIRE_CACHE
void expire_tiles(int sock, char * host, char * uri) {
if (sock < 0) {
return;
}
syslog(LOG_INFO, "Purging metatile via HTCP cache expiry");
int ox, oy;
int limit = (1 << z_);
limit = MIN(limit, METATILE);
// Generate offset table
for (ox=0; ox < limit; ox++) {
for (oy=0; oy < limit; oy++) {
cache_expire(sock, host, uri, (x_ + ox), (y_ + oy), z_);
}
}
}
#endif
int x_, y_, z_;
std::string xmlconfig_;
std::string tile[METATILE][METATILE];
static const int header_size = sizeof(struct meta_layout) + (sizeof(struct entry) * (METATILE * METATILE));
};
static enum protoCmd render(Map &m, char *xmlname, projection &prj, int x, int y, int z, unsigned int size, metaTile &tiles)
{
int render_size = 256 * size;
double p0x = x * 256;
double p0y = (y + size) * 256;
double p1x = (x + size) * 256;
double p1y = y * 256;
//std::cout << "META TILE " << z << " " << x << "-" << x+size-1 << " " << y << "-" << y+size-1 << "\n";
tiling.fromPixelToLL(p0x, p0y, z);
tiling.fromPixelToLL(p1x, p1y, z);
prj.forward(p0x, p0y);
prj.forward(p1x, p1y);
Envelope<double> bbox(p0x, p0y, p1x,p1y);
m.resize(render_size, render_size);
m.zoomToBox(bbox);
m.set_buffer_size(128);
//m.zoom(size+1);
Image32 buf(render_size, render_size);
agg_renderer<Image32> ren(m,buf);
ren.apply();
// Split the meta tile into an NxN grid of tiles
unsigned int xx, yy;
for (yy = 0; yy < size; yy++) {
for (xx = 0; xx < size; xx++) {
image_view<ImageData32> vw(xx * 256, yy * 256, 256, 256, buf.data());
tiles.set(xx, yy, save_to_string(vw, "png256"));
}
}
// std::cout << "DONE TILE " << xmlname << " " << z << " " << x << "-" << x+size-1 << " " << y << "-" << y+size-1 << "\n";
syslog(LOG_DEBUG, "DEBUG: DONE TILE %s %d %d-%d %d-%d", xmlname, z, x, x+size-1, y, y+size-1);
return cmdDone; // OK
}
#else
static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z)
{
char filename[PATH_MAX];
char tmp[PATH_MAX];
double p0x = x * 256.0;
double p0y = (y + 1) * 256.0;
double p1x = (x + 1) * 256.0;
double p1y = y * 256.0;
tiling.fromPixelToLL(p0x, p0y, z);
tiling.fromPixelToLL(p1x, p1y, z);
prj.forward(p0x, p0y);
prj.forward(p1x, p1y);
Envelope<double> bbox(p0x, p0y, p1x,p1y);
bbox.width(bbox.width() * 2);
bbox.height(bbox.height() * 2);
m.zoomToBox(bbox);
Image32 buf(RENDER_SIZE, RENDER_SIZE);
agg_renderer<Image32> ren(m,buf);
ren.apply();
xyz_to_path(filename, sizeof(filename), tile_dir, xmlname, x, y, z);
if (mkdirp(filename))
return cmdNotDone;
snprintf(tmp, sizeof(tmp), "%s.tmp", filename);
image_view<ImageData32> vw(128, 128, 256, 256, buf.data());
//std::cout << "Render " << z << " " << x << " " << y << " " << filename << "\n";
save_to_file(vw, tmp, "png256");
if (rename(tmp, filename)) {
perror(tmp);
return cmdNotDone;
}
return cmdDone; // OK
}
#endif
void render_init(const char *plugins_dir, const char* font_dir, int font_dir_recurse)
{
datasource_cache::instance()->register_datasources(plugins_dir);
load_fonts(font_dir, font_dir_recurse);
}
void *render_thread(void * arg)
{
xmlconfigitem * parentxmlconfig = (xmlconfigitem *)arg;
xmlmapconfig maps[XMLCONFIGS_MAX];
int i,iMaxConfigs;
for (iMaxConfigs = 0; iMaxConfigs < XMLCONFIGS_MAX; ++iMaxConfigs) {
if (parentxmlconfig[iMaxConfigs].xmlname[0] == 0 || parentxmlconfig[iMaxConfigs].xmlfile[0] == 0) break;
strcpy(maps[iMaxConfigs].xmlname, parentxmlconfig[iMaxConfigs].xmlname);
strcpy(maps[iMaxConfigs].xmlfile, parentxmlconfig[iMaxConfigs].xmlfile);
strcpy(maps[iMaxConfigs].tile_dir, parentxmlconfig[iMaxConfigs].tile_dir);
maps[iMaxConfigs].map = Map(RENDER_SIZE, RENDER_SIZE);
maps[iMaxConfigs].ok = 1;
try {
load_map(maps[iMaxConfigs].map, maps[iMaxConfigs].xmlfile);
} catch (mapnik::config_error &ex) {
syslog(LOG_ERR, "An error occurred while loading the map layer '%s': %s", maps[iMaxConfigs].xmlname, ex.what());
maps[iMaxConfigs].ok = 0;
}
maps[iMaxConfigs].prj = projection(maps[iMaxConfigs].map.srs());
#ifdef HTCP_EXPIRE_CACHE
strcpy(maps[iMaxConfigs].xmluri, parentxmlconfig[iMaxConfigs].xmluri);
strcpy(maps[iMaxConfigs].host, parentxmlconfig[iMaxConfigs].host);
strcpy(maps[iMaxConfigs].htcphost, parentxmlconfig[iMaxConfigs].htcpip);
if (strlen(maps[iMaxConfigs].htcphost) > 0) {
maps[iMaxConfigs].htcpsock = init_cache_expire(
maps[iMaxConfigs].htcphost);
if (maps[iMaxConfigs].htcpsock > 0) {
syslog(LOG_INFO, "Successfully opened socket for HTCP cache expiry");
} else {
syslog(LOG_ERR, "Failed to opened socket for HTCP cache expiry");
}
} else {
maps[iMaxConfigs].htcpsock = -1;
}
#endif
}
while (1) {
enum protoCmd ret;
struct item *item = fetch_request();
if (item) {
struct protocol *req = &item->req;
#ifdef METATILE
// At very low zoom the whole world may be smaller than METATILE
unsigned int size = MIN(METATILE, 1 << req->z);
for (i = 0; i < iMaxConfigs; ++i) {
if (!strcmp(maps[i].xmlname, req->xmlname)) {
metaTile tiles(req->xmlname, item->mx, item->my, req->z);
if (maps[i].ok) {
timeval tim;
gettimeofday(&tim, NULL);
long t1=tim.tv_sec*1000+(tim.tv_usec/1000);
ret = render(maps[i].map, req->xmlname, maps[i].prj, item->mx, item->my, req->z, size, tiles);
gettimeofday(&tim, NULL);
long t2=tim.tv_sec*1000+(tim.tv_usec/1000);
syslog(LOG_DEBUG, "DEBUG: DONE TILE %s %d %d-%d %d-%d in %.3lf seconds",
req->xmlname, req->z, item->mx, item->mx+size-1, item->my, item->my+size-1, (t2 - t1)/1000.0);
statsRenderFinish(req->z, t2 - t1);
} else {
syslog(LOG_ERR, "Received request for map layer '%s' which failed to load", req->xmlname);
ret = cmdNotDone;
}
if (ret == cmdDone) {
try {
tiles.save(maps[i].tile_dir);
} catch (...) {
// Treat any error as fatal and request end of processing
syslog(LOG_ERR, "Received error when writing metatile to disk, requesting exit.");
ret = cmdNotDone;
request_exit();
}
#ifdef HTCP_EXPIRE_CACHE
tiles.expire_tiles(maps[i].htcpsock,maps[i].host,maps[i].xmluri);
#endif
}
#else
ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z);
#ifdef HTCP_EXPIRE_CACHE
cache_expire(maps[i].htcpsock,maps[i].host, maps[i].xmluri, req->x,req->y,req->z);
#endif
#endif
send_response(item, ret);
break;
}
}
if (i == iMaxConfigs){
syslog(LOG_ERR, "No map for: %s", req->xmlname);
}
} else {
sleep(1); // TODO: Use an event to indicate there are new requests
}
}
return NULL;
}
Reference in New Issue View Git Blame Copy Permalink