Files
openstreetmap-mod_tile-pyth…/store_file_utils.c
Kai Krueger e8001c859f Add support for pluggable storage backends to mod_tile / renderd
So far, (meta)tiles could only be stored and retrieved from a locally mounted posix filesystem.  Although, the posix
filesystem API is it self a plugable storage layer into which one can mount many different filsystems, from temporary
filesystems to network filesystems, as mod_tile installations scale up to multi-server environments this might not be sufficient and one
may want to use other storage layers.

This patch therefore abstracts all storage calls out into a separate API that can be implemented by various storage backends to fit the needs of different
installations. 

Three storage backends are included in this commit:

- file backend: This is the equivalent of what existed before in mod_tile / renderd. This uses a posix filesystem to store (meta)tiles
- memcached: This stores tiles in a memcached store
- rados: This stores tiles in a rados / ceph cluster

The memcached and rados backends should currently still be considered as experimental

There are also other refactoring and cleanups in this commit
2013-03-23 22:19:30 +00:00

175 lines
4.8 KiB
C

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <limits.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include "protocol.h"
#include "render_config.h"
#include "store_file_utils.h"
// Build parent directories for the specified file name
// Note: the part following the trailing / is ignored
// e.g. mkdirp("/a/b/foo.png") == shell mkdir -p /a/b
int mkdirp(const char *path) {
struct stat s;
char tmp[PATH_MAX];
char *p;
strncpy(tmp, path, sizeof(tmp) - 1);
// Look for parent directory
p = strrchr(tmp, '/');
if (!p)
return 0;
*p = '\0';
if (!stat(tmp, &s))
return !S_ISDIR(s.st_mode);
*p = '/';
// Walk up the path making sure each element is a directory
p = tmp;
if (!*p)
return 0;
p++; // Ignore leading /
while (*p) {
if (*p == '/') {
*p = '\0';
if (!stat(tmp, &s)) {
if (!S_ISDIR(s.st_mode)) {
fprintf(stderr, "Error, is not a directory: %s\n", tmp);
return 1;
}
} else if (mkdir(tmp, 0777)) {
// Ignore multiple threads attempting to create the same directory
if (errno != EEXIST) {
perror(tmp);
return 1;
}
}
*p = '/';
}
p++;
}
return 0;
}
/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* to work
*/
static int check_xyz(int x, int y, int z) {
int oob, limit;
// Validate tile co-ordinates
oob = (z < 0 || z > MAX_ZOOM);
if (!oob) {
// valid x/y for tiles are 0 ... 2^zoom-1
limit = (1 << z) - 1;
oob = (x < 0 || x > limit || y < 0 || y > limit);
}
if (oob)
fprintf(stderr, "got bad co-ords: x(%d) y(%d) z(%d)\n", x, y, z);
return oob;
}
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
#ifdef DIRECTORY_HASH
// We attempt to cluster the tiles so that a 16x16 square of tiles will be in a single directory
// Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates
// 4 bits of x & y are used per byte of output
unsigned char i, hash[5];
for (i=0; i<5; i++) {
hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
x >>= 4;
y >>= 4;
}
snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.png", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
#else
snprintf(path, len, TILE_PATH "/%s/%d/%d/%d.png", xmlconfig, z, x, y);
#endif
return;
}
int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz)
{
#ifdef DIRECTORY_HASH
int i, n, hash[5], x, y, z;
for(i = 0; tilepath[i] && tilepath[i] == path[i]; ++i)
;
if(tilepath[i]) {
fprintf(stderr, "Tile path does not match settings (%s): %s\n", tilepath, path);
return 1;
}
n = sscanf(path+i, "/%40[^/]/%d/%d/%d/%d/%d/%d", xmlconfig, pz, &hash[0], &hash[1], &hash[2], &hash[3], &hash[4]);
if (n != 7) {
fprintf(stderr, "Failed to parse tile path: %s\n", path);
return 1;
} else {
x = y = 0;
for (i=0; i<5; i++) {
if (hash[i] < 0 || hash[i] > 255) {
fprintf(stderr, "Failed to parse tile path (invalid %d): %s\n", hash[i], path);
return 2;
}
x <<= 4;
y <<= 4;
x |= (hash[i] & 0xf0) >> 4;
y |= (hash[i] & 0x0f);
}
z = *pz;
*px = x;
*py = y;
return check_xyz(x, y, z);
}
#else
int n;
n = sscanf(path, TILE_PATH "/%40[^/]/%d/%d/%d", xmlconfig, pz, px, py);
if (n != 4) {
fprintf(stderr, "Failed to parse tile path: %s\n", path);
return 1;
} else {
return check_xyz(*px, *py, *pz);
}
#endif
}
#ifdef METATILE
// Returns the path to the meta-tile and the offset within the meta-tile
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
unsigned char i, hash[5], offset, mask;
// Each meta tile winds up in its own file, with several in each leaf directory
// the .meta tile name is beasd on the sub-tile at (0,0)
mask = METATILE - 1;
offset = (x & mask) * METATILE + (y & mask);
x &= ~mask;
y &= ~mask;
for (i=0; i<5; i++) {
hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
x >>= 4;
y >>= 4;
}
#ifdef DIRECTORY_HASH
snprintf(path, len, "%s/%s/%d/%u/%u/%u/%u/%u.meta", tile_dir, xmlconfig, z, hash[4], hash[3], hash[2], hash[1], hash[0]);
#else
snprintf(path, len, "%s/%s/%d/%u/%u.meta", tile_dir, xmlconfig, z, x, y);
#endif
return offset;
}
#endif