More changes to make the tile cache directory configurable at runtime

* -t|--tile-dir option for render_list and convert_meta
* renderd reads it from renderd.conf section "[renderd]" setting "tile_dir"
* mod_tile reads from Apache conf ModTileTileDir directive
There are still some places in store.c left, where HASH_DIR is used directly. More fixes needed.
This commit is contained in:
Jochen Topf
2009-06-10 16:26:12 +00:00
parent fb751bf8d4
commit 29080ad2ec
10 changed files with 84 additions and 50 deletions

View File

@ -135,7 +135,7 @@ int main(int argc, char **argv)
case 'm':
map=strdup(optarg);
break;
case 'p':
case 't':
tile_dir=strdup(optarg);
break;
case 'u':
@ -147,11 +147,11 @@ int main(int argc, char **argv)
case 'h':
fprintf(stderr, "Usage: convert_meta [OPTION] ...\n");
fprintf(stderr, "Convert the rendered PNGs into the more efficient .meta format\n");
fprintf(stderr, " -m, --map convert tiles in this map (default is 'default')\n");
fprintf(stderr, " -t, --tile-dir tile cache directory (default is '" HASH_PATH "')\n");
fprintf(stderr, " -u, --unpack unpack the .meta files back to PNGs\n");
fprintf(stderr, " -m, --map convert tiles in this map (default is \"default\")\n");
fprintf(stderr, " -t, --tile-dir tile cache directory (default is \"" HASH_PATH "\")\n");
fprintf(stderr, " -z, --min-zoom only process tiles greater or equal this zoom level (default 0)\n");
fprintf(stderr, " -Z, --max-zoom only process tiles less than or equal to this zoom level (default 18)\n");
fprintf(stderr, " -z, --min-zoom only process tiles greater or equal to this zoom level (default is 0)\n");
fprintf(stderr, " -Z, --max-zoom only process tiles less than or equal to this zoom level (default is 18)\n");
return -1;
default:
fprintf(stderr, "unhandled char '%c'\n", c);

View File

@ -449,6 +449,7 @@ int main(int argc, char **argv)
exit(7);
}
strcpy(maps[iconf].xmlfile, ini_xmlpath);
strcpy(maps[iconf].tile_dir, config.tile_dir);
}
}

View File

@ -20,6 +20,7 @@ typedef struct {
char xmlname[XMLCONFIG_MAX];
char xmlfile[PATH_MAX];
char xmluri[PATH_MAX];
char tile_dir[PATH_MAX];
} xmlconfigitem;
#endif

View File

@ -62,10 +62,10 @@ int mkdirp(const char *path) {
* to work
*/
void xyz_to_path(char *path, size_t len, const char *xmlconfig, int x, int y, int z)
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
#ifdef DIRECTORY_HASH
// We attempt to cluseter the tiles so that a 16x16 square of tiles will be in a single directory
// We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
// Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
// 4 bits of x & y are used per byte of output
unsigned char i, hash[5];
@ -75,7 +75,7 @@ void xyz_to_path(char *path, size_t len, const char *xmlconfig, int x, int y, in
x >>= 4;
y >>= 4;
}
snprintf(path, len, HASH_PATH "/%s/%d/%u/%u/%u/%u/%u.png", xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
#else
snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
#endif
@ -130,7 +130,7 @@ int path_to_xyz(const char *path, char *xmlconfig, int *px, int *py, int *pz)
#ifdef METATILE
// Returns the path to the meta-tile and the offset within the meta-tile
int xyz_to_meta(char *path, size_t len, const char *xmlconfig, int x, int y, int z)
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
unsigned char i, hash[5], offset, mask;
@ -146,7 +146,7 @@ int xyz_to_meta(char *path, size_t len, const char *xmlconfig, int x, int y, int
x >>= 4;
y >>= 4;
}
snprintf(path, len, HASH_PATH "/%s/%d/%u/%u/%u/%u/%u.meta", xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.meta", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
return offset;
}
#endif

View File

@ -19,7 +19,7 @@ int mkdirp(const char *path);
* The two must both agree on the file layout for meta-tiling
* to work
*/
void xyz_to_path(char *path, size_t len, const char *xmlconfig, int x, int y, int z);
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
int check_xyz(int x, int y, int z);
int path_to_xyz(const char *path, char *xmlconfig, int *px, int *py, int *pz);
@ -27,7 +27,7 @@ int path_to_xyz(const char *path, char *xmlconfig, int *px, int *py, int *pz);
#ifdef METATILE
/* New meta-tile storage functions */
/* Returns the path to the meta-tile and the offset within the meta-tile */
int xyz_to_meta(char *path, size_t len, const char *xmlconfig, int x, int y, int z);
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
#endif
#ifdef __cplusplus

View File

@ -40,6 +40,7 @@ static const int maxZoom = 18;
typedef struct {
char xmlname[XMLCONFIG_MAX];
char xmlfile[PATH_MAX];
char tile_dir[PATH_MAX];
Map map;
projection prj;
} xmlmapconfig;
@ -161,7 +162,7 @@ class metaTile {
return (x & mask) * METATILE + (y & mask);
}
void save()
void save(const char *tile_dir)
{
int ox, oy, limit;
size_t offset;
@ -172,7 +173,7 @@ class metaTile {
memset(&m, 0, sizeof(m));
memset(&offsets, 0, sizeof(offsets));
xyz_to_meta(meta_path, sizeof(meta_path), xmlconfig_.c_str(), x_, y_, z_);
xyz_to_meta(meta_path, sizeof(meta_path), tile_dir, xmlconfig_.c_str(), x_, y_, z_);
std::stringstream ss;
ss << std::string(meta_path) << "." << pthread_self();
std::string tmp(ss.str());
@ -263,7 +264,7 @@ static enum protoCmd render(Map &m, char *xmlname, projection &prj, int x, int y
return cmdDone; // OK
}
#else
static enum protoCmd render(Map &m, char *xmlname, projection &prj, int x, int y, int z)
static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z)
{
char filename[PATH_MAX];
char tmp[PATH_MAX];
@ -287,14 +288,14 @@ static enum protoCmd render(Map &m, char *xmlname, projection &prj, int x, int y
agg_renderer<Image32> ren(m,buf);
ren.apply();
xyz_to_path(filename, sizeof(filename), xmlname, x, y, z);
xyz_to_path(filename, sizeof(filename), tile_dir, xmlname, x, y, z);
if (mkdirp(filename))
return cmdNotDone;
snprintf(tmp, sizeof(tmp), "%s.tmp", filename);
image_view<ImageData32> vw(128,128,256,256, buf.data());
image_view<ImageData32> vw(128, 128, 256, 256, buf.data());
//std::cout << "Render " << z << " " << x << " " << y << " " << filename << "\n";
save_to_file(vw, tmp,"png256");
save_to_file(vw, tmp, "png256");
if (rename(tmp, filename)) {
perror(tmp);
return cmdNotDone;
@ -321,6 +322,7 @@ void *render_thread(void * arg)
if (parentxmlconfig[iMaxConfigs].xmlname[0] == 0 || parentxmlconfig[iMaxConfigs].xmlfile[0] == 0) break;
strcpy(maps[iMaxConfigs].xmlname, parentxmlconfig[iMaxConfigs].xmlname);
strcpy(maps[iMaxConfigs].xmlfile, parentxmlconfig[iMaxConfigs].xmlfile);
strcpy(maps[iMaxConfigs].tile_dir, parentxmlconfig[iMaxConfigs].tile_dir);
maps[iMaxConfigs].map = Map(RENDER_SIZE, RENDER_SIZE);
load_map(maps[iMaxConfigs].map, maps[iMaxConfigs].xmlfile);
maps[iMaxConfigs].prj = projection(maps[iMaxConfigs].map.srs());
@ -340,9 +342,9 @@ void *render_thread(void * arg)
ret = render(maps[i].map, req->xmlname, maps[i].prj, item->mx, item->my, req->z, size, tiles);
if (ret == cmdDone)
tiles.save();
tiles.save(maps[i].tile_dir);
#else
ret = render(maps[i].map, req->xmlname, maps[i].prj, req->x, req->y, req->z);
ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z);
#endif
send_response(item, ret);
break;

View File

@ -60,6 +60,7 @@ typedef struct {
int max_load_old;
int max_load_missing;
char renderd_socket_name[PATH_MAX];
char tile_dir[PATH_MAX];
} tile_server_conf;
enum tileState { tileMissing, tileOld, tileCurrent };
@ -280,17 +281,19 @@ static enum tileState tile_state(request_rec *r, struct protocol *cmd)
enum tileState state = tile_state_once(r);
#ifdef METATILEFALLBACK
if (state == tileMissing) {
ap_conf_vector_t *sconf = r->server->module_config;
tile_server_conf *scfg = ap_get_module_config(sconf, &tile_module);
// Try fallback to plain PNG
char path[PATH_MAX];
xyz_to_path(path, sizeof(path), cmd->xmlname, cmd->x, cmd->y, cmd->z);
xyz_to_path(path, sizeof(path), scfg->tile_dir, cmd->xmlname, cmd->x, cmd->y, cmd->z);
r->filename = apr_pstrdup(r->pool, path);
state = tile_state_once(r);
ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "png fallback %d/%d/%d",x,y,z);
if (state == tileMissing) {
// PNG not available either, if it gets rendered, it'll now be a .meta
xyz_to_meta(path, sizeof(path), cmd->xmlname, cmd->x, cmd->y, cmd->z);
xyz_to_meta(path, sizeof(path), scfg->tile_dir, cmd->xmlname, cmd->x, cmd->y, cmd->z);
r->filename = apr_pstrdup(r->pool, path);
}
}
@ -376,9 +379,11 @@ static int tile_storage_hook(request_rec *r)
should already be done
// Generate the tile filename
#ifdef METATILE
xyz_to_meta(abs_path, sizeof(abs_path), cmd->xmlname, cmd->x, cmd->y, cmd->z);
ap_conf_vector_t *sconf = r->server->module_config;
tile_server_conf *scfg = ap_get_module_config(sconf, &tile_module);
xyz_to_meta(abs_path, sizeof(abs_path), scfg->tile_dir, cmd->xmlname, cmd->x, cmd->y, cmd->z);
#else
xyz_to_path(abs_path, sizeof(abs_path), cmd->xmlname, cmd->x, cmd->y, cmd->z);
xyz_to_path(abs_path, sizeof(abs_path), scfg->tile_dir, cmd->xmlname, cmd->x, cmd->y, cmd->z);
#endif
ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "abs_path(%s)", abs_path);
r->filename = apr_pstrdup(r->pool, abs_path);
@ -545,9 +550,9 @@ static int tile_translate(request_rec *r)
// Generate the tile filename?
char abs_path[PATH_MAX];
#ifdef METATILE
xyz_to_meta(abs_path, sizeof(abs_path), cmd->xmlname, cmd->x, cmd->y, cmd->z);
xyz_to_meta(abs_path, sizeof(abs_path), scfg->tile_dir, cmd->xmlname, cmd->x, cmd->y, cmd->z);
#else
xyz_to_path(abs_path, sizeof(abs_path), cmd->xmlname, cmd->x, cmd->y, cmd->z);
xyz_to_path(abs_path, sizeof(abs_path), scfg->tile_dir, cmd->xmlname, cmd->x, cmd->y, cmd->z);
#endif
r->filename = apr_pstrdup(r->pool, abs_path);
@ -692,6 +697,14 @@ static const char *mod_tile_renderd_socket_name_config(cmd_parms *cmd, void *mco
return NULL;
}
static const char *mod_tile_tile_dir_config(cmd_parms *cmd, void *mconfig, const char *tile_dir_string)
{
tile_server_conf *scfg = ap_get_module_config(cmd->server->module_config, &tile_module);
strncpy(scfg->tile_dir, tile_dir_string, PATH_MAX-1);
scfg->tile_dir[PATH_MAX-1] = 0;
return NULL;
}
static void *create_tile_config(apr_pool_t *p, server_rec *s)
{
tile_server_conf * scfg = (tile_server_conf *) apr_pcalloc(p, sizeof(tile_server_conf));
@ -700,6 +713,10 @@ static void *create_tile_config(apr_pool_t *p, server_rec *s)
scfg->request_timeout = REQUEST_TIMEOUT;
scfg->max_load_old = MAX_LOAD_OLD;
scfg->max_load_missing = MAX_LOAD_MISSING;
strncpy(scfg->renderd_socket_name, RENDER_SOCKET, PATH_MAX-1);
scfg->renderd_socket_name[PATH_MAX-1] = 0;
strncpy(scfg->tile_dir, HASH_PATH, PATH_MAX-1);
scfg->tile_dir[PATH_MAX-1] = 0;
return scfg;
}
@ -716,6 +733,8 @@ static void *merge_tile_config(apr_pool_t *p, void *basev, void *overridesv)
scfg->max_load_missing = scfg_over->max_load_missing;
strncpy(scfg->renderd_socket_name, scfg_over->renderd_socket_name, PATH_MAX-1);
scfg->renderd_socket_name[PATH_MAX-1] = 0;
strncpy(scfg->tile_dir, scfg_over->tile_dir, PATH_MAX-1);
scfg->tile_dir[PATH_MAX-1] = 0;
return scfg;
}
@ -764,6 +783,13 @@ static const command_rec tile_cmds[] =
OR_OPTIONS, /* where available */
"Set name of unix domain socket for connecting to rendering daemon" /* directive description */
),
AP_INIT_TAKE1(
"ModTileTileDir", /* directive name */
mod_tile_tile_dir_config, /* config action routine */
NULL, /* argument to include in call */
OR_OPTIONS, /* where available */
"Set name of tile cache directory" /* directive description */
),
{NULL}
};

View File

@ -112,8 +112,9 @@ int process_loop(int fd, const char *mapname, int x, int y, int z)
int main(int argc, char **argv)
{
char spath[PATH_MAX] = RENDER_SOCKET;
char mapname[PATH_MAX] = "default";
char *spath = RENDER_SOCKET;
char *mapname = "default";
char *tile_dir = HASH_PATH;
int fd;
struct sockaddr_un addr;
int ret=0;
@ -136,6 +137,7 @@ int main(int argc, char **argv)
{"min-y", 1, 0, 'y'},
{"max-y", 1, 0, 'Y'},
{"socket", 1, 0, 's'},
{"tile-dir", 1, 0, 't'},
{"map", 1, 0, 'm'},
{"verbose", 0, 0, 'v'},
{"all", 0, 0, 'a'},
@ -143,7 +145,7 @@ int main(int argc, char **argv)
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, "hvaz:Z:x:X:y:Y:s:m:", long_options, &option_index);
c = getopt_long(argc, argv, "hvaz:Z:x:X:y:Y:s:m:t:", long_options, &option_index);
if (c == -1)
break;
@ -152,12 +154,13 @@ int main(int argc, char **argv)
all=1;
break;
case 's': /* -s, --socket */
strncpy(spath, optarg, PATH_MAX-1);
spath[PATH_MAX-1] = 0;
spath = strdup(optarg);
break;
case 't':
tile_dir=strdup(optarg);
break;
case 'm': /* -m, --map */
strncpy(mapname, optarg, PATH_MAX-1);
spath[PATH_MAX-1] = 0;
mapname=strdup(optarg);
break;
case 'x': /* -x, --min-x */
minX=atoi(optarg);
@ -190,11 +193,12 @@ int main(int argc, char **argv)
break;
case 'h': /* -h, --help */
fprintf(stderr, "Usage: render_list [OPTION] ...\n");
fprintf(stderr, " -z, --min-zoom=ZOOM filter input to only render tiles greater or equal to this zoom level (default 0)\n");
fprintf(stderr, " -Z, --max-zoom=ZOOM filter input to only render tiles less than or equal to this zoom level (default 18)\n");
fprintf(stderr, " -s, --socket=SOCKET unix domain socket name for contacting renderd\n");
fprintf(stderr, " -m, --map=MAP name of the map config (defaults to 'default')\n");
fprintf(stderr, " -a, --all render all tiles in given zoom level range instead of reading from STDIN\n");
fprintf(stderr, " -m, --map=MAP render tiles in this map (defaults to 'default')\n");
fprintf(stderr, " -s, --socket=SOCKET unix domain socket name for contacting renderd\n");
fprintf(stderr, " -t, --tile-dir tile cache directory (defaults to '" HASH_PATH "')\n");
fprintf(stderr, " -z, --min-zoom=ZOOM filter input to only render tiles greater or equal to this zoom level (default is 0)\n");
fprintf(stderr, " -Z, --max-zoom=ZOOM filter input to only render tiles less than or equal to this zoom level (default is 18)\n");
fprintf(stderr, "If you are using --all, you can restrict the tile range by adding these options:\n");
fprintf(stderr, " -x, --min-x=X minimum X tile coordinate\n");
fprintf(stderr, " -X, --max-x=X maximum X tile coordinate\n");
@ -301,7 +305,7 @@ int main(int argc, char **argv)
printf("got: x(%d) y(%d) z(%d)\n", x, y, z);
num_all++;
xyz_to_path(name, sizeof(name), XMLCONFIG_DEFAULT, x, y, z);
xyz_to_path(name, sizeof(name), tile_dir, XMLCONFIG_DEFAULT, x, y, z);
if ((stat(name, &s) < 0) || (planetTime > s.st_mtime)) {
// missing or old, render it

View File

@ -231,7 +231,7 @@ int main(int argc, char **argv)
for (x=xmin; x<=xmax; x++) {
for (y=ymin; y<=ymax; y++) {
struct stat s;
xyz_to_meta(name, sizeof(name), XMLCONFIG_DEFAULT, x, y, z);
xyz_to_meta(name, sizeof(name), HASH_PATH, XMLCONFIG_DEFAULT, x, y, z);
if (stat(name, &s) < 0) {
// File doesn't exist
ret = process_loop(fd, x, y, z);

22
store.c
View File

@ -33,7 +33,7 @@ int read_from_meta(const char *xmlconfig, int x, int y, int z, unsigned char *bu
struct meta_layout *m = (struct meta_layout *)header;
size_t file_offset, tile_size;
meta_offset = xyz_to_meta(path, sizeof(path), xmlconfig, x, y, z);
meta_offset = xyz_to_meta(path, sizeof(path), HASH_PATH, xmlconfig, x, y, z);
fd = open(path, O_RDONLY);
if (fd < 0)
@ -107,7 +107,7 @@ int read_from_file(const char *xmlconfig, int x, int y, int z, unsigned char *bu
int fd;
size_t pos;
xyz_to_path(path, sizeof(path), xmlconfig, x, y, z);
xyz_to_path(path, sizeof(path), HASH_PATH, xmlconfig, x, y, z);
fd = open(path, O_RDONLY);
if (fd < 0)
@ -173,7 +173,7 @@ void process_meta(const char *xmlconfig, int x, int y, int z)
for (oy=0; oy < limit; oy++) {
//fprintf(stderr, "Process %d/%d/%d\n", num, ox, oy);
int len = read_from_file(xmlconfig, x + ox, y + oy, z, buf + offset, buf_len - offset);
int mt = xyz_to_meta(meta_path, sizeof(meta_path), xmlconfig, x + ox, y + oy, z);
int mt = xyz_to_meta(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x + ox, y + oy, z);
if (len <= 0) {
#if 1
fprintf(stderr, "Problem reading sub tiles for metatile xml(%s) x(%d) y(%d) z(%d), got %d\n", xmlconfig, x, y, z, len);
@ -196,7 +196,7 @@ void process_meta(const char *xmlconfig, int x, int y, int z)
m->y = y;
m->z = z;
xyz_to_meta(meta_path, sizeof(meta_path), xmlconfig, x, y, z);
xyz_to_meta(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x, y, z);
snprintf(tmp, sizeof(tmp), "%s.tmp.%d", meta_path, getpid());
fd = open(tmp, O_WRONLY | O_TRUNC | O_CREAT, 0666);
@ -224,12 +224,12 @@ void process_meta(const char *xmlconfig, int x, int y, int z)
free(buf);
// Reset meta timestamp to match one of the original tiles
xyz_to_path(meta_path, sizeof(meta_path), xmlconfig, x, y, z);
xyz_to_path(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x, y, z);
if (stat(meta_path, &s) == 0) {
struct utimbuf b;
b.actime = s.st_atime;
b.modtime = s.st_mtime;
xyz_to_meta(meta_path, sizeof(meta_path), xmlconfig, x, y, z);
xyz_to_meta(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x, y, z);
utime(tmp, &b);
}
rename(tmp, meta_path);
@ -238,7 +238,7 @@ void process_meta(const char *xmlconfig, int x, int y, int z)
// Remove raw .png's
for (ox=0; ox < limit; ox++) {
for (oy=0; oy < limit; oy++) {
xyz_to_path(meta_path, sizeof(meta_path), xmlconfig, x + ox, y + oy, z);
xyz_to_path(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x + ox, y + oy, z);
if (unlink(meta_path)<0)
perror(meta_path);
}
@ -256,7 +256,7 @@ void process_pack(const char *name)
return;
// Launch the .meta creation for only 1 tile of the whole block
meta_offset = xyz_to_meta(meta_path, sizeof(meta_path), xmlconfig, x, y, z);
meta_offset = xyz_to_meta(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x, y, z);
//fprintf(stderr,"Requesting x(%d) y(%d) z(%d) - mo(%d)\n", x, y, z, meta_offset);
if (meta_offset == 0)
@ -269,7 +269,7 @@ static void write_tile(const char *xmlconfig, int x, int y, int z, const unsigne
char path[PATH_MAX];
size_t pos;
xyz_to_path(path, sizeof(path), xmlconfig, x, y, z);
xyz_to_path(path, sizeof(path), HASH_PATH, xmlconfig, x, y, z);
fd = open(path, O_WRONLY | O_TRUNC | O_CREAT, 0666);
if (fd < 0) {
fprintf(stderr, "Error creating file: %s\n", path);
@ -333,14 +333,14 @@ void process_unpack(const char *name)
b.modtime = s.st_mtime;
for (ox=0; ox < limit; ox++) {
for (oy=0; oy < limit; oy++) {
xyz_to_path(meta_path, sizeof(meta_path), xmlconfig, x+ox, y+oy, z);
xyz_to_path(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x+ox, y+oy, z);
utime(meta_path, &b);
}
}
}
// Remove the .meta file
xyz_to_meta(meta_path, sizeof(meta_path), xmlconfig, x, y, z);
xyz_to_meta(meta_path, sizeof(meta_path), HASH_PATH, xmlconfig, x, y, z);
if (unlink(meta_path)<0)
perror(meta_path);
}