From 923337038cc099fe448151daef9ca3dcf8ded48c Mon Sep 17 00:00:00 2001 From: Jon Burgess Date: Sat, 16 Feb 2008 20:30:49 +0000 Subject: [PATCH] mod_tile: Update to a new meta tile scheme which stores all sub-tiles in a single .meta file. The PNG files are extracted from this on the fly by mod_tile. This is more efficient in disk space and inode usage --- COPYING | 340 +++++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 8 +- convert_meta.c | 164 ++++++++++++++++++++++++ daemon.c | 7 +- dir_utils.c | 23 +++- dir_utils.h | 6 + gen_tile.cpp | 3 +- mod_tile.c | 307 ++++++++++++++++++++++++++------------------ modules.mk | 2 +- protocol.h | 1 - render_old.c | 6 +- speedtest.cpp | 2 +- store.c | 308 ++++++++++++++++++++++++++++++++++++++++++++ store.h | 42 ++++++ 14 files changed, 1076 insertions(+), 143 deletions(-) create mode 100644 COPYING create mode 100644 convert_meta.c create mode 100644 store.c create mode 100644 store.h diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..60549be --- /dev/null +++ b/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) 19yy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) 19yy name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/Makefile b/Makefile index 18a9e92..f1b5de0 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,11 @@ EXTRA_CFLAGS = -I$(builddir) EXTRA_CPPFLAGS += -g -O2 -Wall EXTRA_LDFLAGS += $(shell pkg-config --libs libagg) -all: local-shared-build renderd speedtest render_list render_old +all: local-shared-build renderd speedtest render_list render_old convert_meta clean: rm -f *.o *.lo *.slo *.la .libs/* - rm -f renderd render_list speedtest render_old + rm -f renderd render_list speedtest render_old convert_meta RENDER_CPPFLAGS += -g -O2 -Wall RENDER_CPPFLAGS += -I/usr/local/include/mapnik @@ -35,7 +35,7 @@ RENDER_LDFLAGS += $(shell pkg-config --libs freetype2) #RENDER_LDFLAGS += $(shell Magick++-config --ldflags --libs) RENDER_LDFLAGS += $(shell pkg-config --libs libagg) -renderd: daemon.c gen_tile.cpp dir_utils.c protocol.h render_config.h dir_utils.h +renderd: store.c daemon.c gen_tile.cpp dir_utils.c protocol.h render_config.h dir_utils.h store.h $(CXX) -o $@ $^ $(RENDER_LDFLAGS) $(RENDER_CPPFLAGS) speedtest: render_config.h protocol.h dir_utils.c dir_utils.h @@ -44,6 +44,8 @@ render_list: render_config.h protocol.h dir_utils.c dir_utils.h render_old: render_config.h protocol.h dir_utils.c dir_utils.h +convert_meta: render_config.h protocol.h dir_utils.c dir_utils.h store.c + MYSQL_CFLAGS += -g -O2 -Wall MYSQL_CFLAGS += $(shell mysql_config --cflags) diff --git a/convert_meta.c b/convert_meta.c new file mode 100644 index 0000000..eaef9fd --- /dev/null +++ b/convert_meta.c @@ -0,0 +1,164 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "render_config.h" +#include "dir_utils.h" +#include "store.h" + +static int minZoom = 0; +static int maxZoom = 18; +static int verbose = 0; +static int num_render = 0, num_all = 0; +static struct timeval start, end; +static int unpack; + +void display_rate(struct timeval start, struct timeval end, int num) +{ + int d_s, d_us; + float sec; + + d_s = end.tv_sec - start.tv_sec; + d_us = end.tv_usec - start.tv_usec; + + sec = d_s + d_us / 1000000.0; + + printf("Converted %d tiles in %.2f seconds (%.2f tiles/s)\n", num, sec, num / sec); + fflush(NULL); +} + +static void descend(const char *search) +{ + DIR *tiles = opendir(search); + struct dirent *entry; + char path[PATH_MAX]; + + if (!tiles) { + //fprintf(stderr, "Unable to open directory: %s\n", search); + return; + } + + while ((entry = readdir(tiles))) { + struct stat b; + char *p; + + //check_load(); + + if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) + continue; + snprintf(path, sizeof(path), "%s/%s", search, entry->d_name); + if (stat(path, &b)) + continue; + if (S_ISDIR(b.st_mode)) { + descend(path); + continue; + } + p = strrchr(path, '.'); + if (p) { + if (unpack) { + if (!strcmp(p, ".meta")) + process_unpack(path); + } else { + if (!strcmp(p, ".png")) + process_pack(path); + } + } + } + closedir(tiles); +} + + +int main(int argc, char **argv) +{ + int z, c; + + while (1) { + int option_index = 0; + static struct option long_options[] = { + {"min-zoom", 1, 0, 'z'}, + {"max-zoom", 1, 0, 'Z'}, + {"unpack", 0, 0, 'u'}, + {"verbose", 0, 0, 'v'}, + {"help", 0, 0, 'h'}, + {0, 0, 0, 0} + }; + + c = getopt_long(argc, argv, "uhvz:Z:", long_options, &option_index); + if (c == -1) + break; + + switch (c) { + case 'z': + minZoom=atoi(optarg); + if (minZoom < 0 || minZoom > 18) { + fprintf(stderr, "Invalid minimum zoom selected, must be between 0 and 18\n"); + return 1; + } + break; + case 'Z': + maxZoom=atoi(optarg); + if (maxZoom < 0 || maxZoom > 18) { + fprintf(stderr, "Invalid maximum zoom selected, must be between 0 and 18\n"); + return 1; + } + break; + case 'u': + unpack=1; + break; + case 'v': + verbose=1; + break; + case 'h': + fprintf(stderr, "Convert the rendered PNGs into the more efficient .meta format\n"); + fprintf(stderr, "\t-u|--unpack\tUnpack the .meta files back to PNGs\n"); + fprintf(stderr, "\t-z|--min-zoom\tonly process tiles greater or equal this zoom level (default 0)\n"); + fprintf(stderr, "\t-Z|--max-zoom\tonly process tiles less than or equal to this zoom level (default 18)\n"); + return -1; + default: + fprintf(stderr, "unhandled char '%c'\n", c); + break; + } + } + + if (maxZoom < minZoom) { + fprintf(stderr, "Invalid zoom range, max zoom must be greater or equal to minimum zoom\n"); + return 1; + } + + fprintf(stderr, "Converting tiles\n"); + + gettimeofday(&start, NULL); + + for (z=minZoom; z<=maxZoom; z++) { + char path[PATH_MAX]; + snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d", z); + descend(path); + } + + gettimeofday(&end, NULL); + printf("\nTotal for all tiles converted\n"); + printf("Meta tiles converted: "); + display_rate(start, end, num_render); + printf("Total tiles converted: "); + display_rate(start, end, num_render * METATILE * METATILE); + printf("Total tiles handled: "); + display_rate(start, end, num_all); + + return 0; +} diff --git a/daemon.c b/daemon.c index dd78ea7..aff7932 100644 --- a/daemon.c +++ b/daemon.c @@ -176,8 +176,8 @@ enum protoCmd rx_request(const struct protocol *req, int fd) return cmdIgnore; } - fprintf(stderr, "%s fd(%d) z(%d), x(%d), y(%d), path(%s)\n", - cmdStr(req->cmd), fd, req->z, req->x, req->y, req->path); + fprintf(stderr, "%s fd(%d) z(%d), x(%d), y(%d)\n", + cmdStr(req->cmd), fd, req->z, req->x, req->y); if ((req->cmd != cmdRender) && (req->cmd != cmdDirty)) return cmdIgnore; @@ -185,9 +185,6 @@ enum protoCmd rx_request(const struct protocol *req, int fd) if (check_xyz(req->x, req->y, req->z)) return cmdNotDone; - if (mkdirp(req->path)) - return cmdNotDone; - item = (struct item *)malloc(sizeof(*item)); if (!item) { fprintf(stderr, "malloc failed\n"); diff --git a/dir_utils.c b/dir_utils.c index 44ff32d..4f18860 100644 --- a/dir_utils.c +++ b/dir_utils.c @@ -101,7 +101,7 @@ int path_to_xyz(const char *path, int *px, int *py, int *pz) { int i, n, hash[5], x, y, z; - n = sscanf(path, WWW_ROOT HASH_PATH "/%d/%d/%d/%d/%d/%d.png", pz, &hash[0], &hash[1], &hash[2], &hash[3], &hash[4]); + n = sscanf(path, WWW_ROOT HASH_PATH "/%d/%d/%d/%d/%d/%d", pz, &hash[0], &hash[1], &hash[2], &hash[3], &hash[4]); if (n != 6) { fprintf(stderr, "Failed to parse tile path: %s\n", path); return 1; @@ -123,3 +123,24 @@ int path_to_xyz(const char *path, int *px, int *py, int *pz) return check_xyz(x, y, z); } } + +// Returns the path to the meta-tile and the offset within the meta-tile +int xyz_to_meta(char *path, size_t len, int x, int y, int z) +{ + unsigned char i, hash[5], offset, mask; + + // Each meta tile winds up in its own file, with several in each leaf directory + // the .meta tile name is beasd on the sub-tile at (0,0) + mask = METATILE - 1; + offset = (x & mask) * METATILE + (y & mask); + x &= ~mask; + y &= ~mask; + + for (i=0; i<5; i++) { + hash[i] = ((x & 0x0f) << 4) | (y & 0x0f); + x >>= 4; + y >>= 4; + } + snprintf(path, len, WWW_ROOT HASH_PATH "/%d/%u/%u/%u/%u/%u.meta", z, hash[4], hash[3], hash[2], hash[1], hash[0]); + return offset; +} diff --git a/dir_utils.h b/dir_utils.h index c06dc62..1dea486 100644 --- a/dir_utils.h +++ b/dir_utils.h @@ -23,6 +23,12 @@ const char *xyz_to_path(char *path, size_t len, int x, int y, int z); int check_xyz(int x, int y, int z); int path_to_xyz(const char *path, int *px, int *py, int *pz); +/* New meta-tile storage functions */ + +/* Returns the path to the meta-tile and the offset within the meta-tile */ +int xyz_to_meta(char *path, size_t len, int x, int y, int z); + + #ifdef __cplusplus } #endif diff --git a/gen_tile.cpp b/gen_tile.cpp index 9b49c4c..6a7d429 100644 --- a/gen_tile.cpp +++ b/gen_tile.cpp @@ -18,7 +18,7 @@ #include "protocol.h" #include "render_config.h" #include "dir_utils.h" - +#include "store.h" using namespace mapnik; @@ -241,6 +241,7 @@ void *render_thread(__attribute__((unused)) void *unused) unsigned int size = MIN(METATILE, 1 << req->z); //pthread_mutex_lock(&map_lock); ret = render(m, item->mx, item->my, req->z, size); + process_meta(item->mx, item->my, req->z); //pthread_mutex_unlock(&map_lock); #else diff --git a/mod_tile.c b/mod_tile.c index 84a256e..0d660df 100644 --- a/mod_tile.c +++ b/mod_tile.c @@ -43,6 +43,8 @@ module AP_MODULE_DECLARE_DATA tile_module; #include "gen_tile.h" #include "protocol.h" #include "render_config.h" +#include "store.h" +#include "dir_utils.h" enum tileState { tileMissing, tileOld, tileCurrent }; @@ -98,14 +100,33 @@ int socket_init(request_rec *r) } -int request_tile(request_rec *r, int x, int y, int z, const char *filename, int dirtyOnly) +int request_tile(request_rec *r, int dirtyOnly) { struct protocol cmd; //struct pollfd fds[1]; static int fd = FD_INVALID; int ret = 0; int retry = 1; - + int x, y, z, n, limit, oob; + + /* URI = ...///.png[/option] */ + n = sscanf(r->uri, TILE_PATH "/%d/%d/%d", &z, &x, &y); + if (n != 3) + return 0; + + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "z(%d) x(%d) y(%d)", z, x, y); + + // Validate tile co-ordinates + oob = (z < 0 || z > MAX_ZOOM); + if (!oob) { + // valid x/y for tiles are 0 ... 2^zoom-1 + limit = (1 << z) - 1; + oob = (x < 0 || x > limit || y < 0 || y > limit); + } + + if (oob) + return 0; + if (fd == FD_INVALID) { fd = socket_init(r); @@ -124,7 +145,6 @@ int request_tile(request_rec *r, int x, int y, int z, const char *filename, int cmd.z = z; cmd.x = x; cmd.y = y; - strcpy(cmd.path, filename); //fprintf(stderr, "Requesting tile(%d,%d,%d)\n", z,x,y); do { @@ -209,12 +229,12 @@ static int getPlanetTime(request_rec *r) return planet_timestamp; } -enum tileState tile_state(request_rec *r, const char *filename) +static enum tileState tile_state_once(request_rec *r) { // FIXME: Apache already has most, if not all, this info recorded in r->fileinfo, use this instead! struct stat buf; - if (stat(filename, &buf)) + if (stat(r->filename, &buf)) return tileMissing; if (buf.st_mtime < getPlanetTime(r)) @@ -223,12 +243,37 @@ enum tileState tile_state(request_rec *r, const char *filename) return tileCurrent; } -static apr_status_t expires_filter(ap_filter_t *f, apr_bucket_brigade *b) +static enum tileState tile_state(request_rec *r) +{ + enum tileState state = tile_state_once(r); + + if (state == tileMissing) { + // Try fallback to plain .png + char path[PATH_MAX]; + int x, y, z, n; + /* URI = ...///.png[/option] */ + n = sscanf(r->uri, TILE_PATH "/%d/%d/%d", &z, &x, &y); + if (n == 3) { + xyz_to_path(path, sizeof(path), x,y,z); + r->filename = apr_pstrdup(r->pool, path); + state = tile_state_once(r); + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "png fallback %d/%d/%d",x,y,z); + + if (state == tileMissing) { + // PNG not available either, if it gets rendered, it'll now be a .meta + xyz_to_meta(path, sizeof(path), x,y,z); + r->filename = apr_pstrdup(r->pool, path); + } + } + } + return state; +} + +static void add_expiry(request_rec *r) { - request_rec *r = f->r; apr_time_t expires, holdoff, nextPlanet; apr_table_t *t = r->headers_out; - enum tileState state = tile_state(r, r->filename); + enum tileState state = tile_state(r); char *timestr; /* Append expiry headers ... */ @@ -248,96 +293,84 @@ static apr_status_t expires_filter(ap_filter_t *f, apr_bucket_brigade *b) timestr = apr_palloc(r->pool, APR_RFC822_DATE_LEN); apr_rfc822_date(timestr, expires); apr_table_setn(t, "Expires", timestr); +} + +static apr_status_t expires_filter(ap_filter_t *f, apr_bucket_brigade *b) +{ + request_rec *r = f->r; + + add_expiry(r); ap_remove_output_filter(f); return ap_pass_brigade(f->next, b); } -static int serve_tile(request_rec *r, const char *rel_path) +double get_load_avg(request_rec *r) { - // Redirect request to the tile - r->method = apr_pstrdup(r->pool, "GET"); - r->method_number = M_GET; - apr_table_unset(r->headers_in, "Content-Length"); - ap_internal_redirect_handler(rel_path, r); - return OK; -} + double loadavg[1]; + int n = getloadavg(loadavg, 1); -static int serve_blank(request_rec *r) -{ - return serve_tile(r, IMG_PATH "/blank-000000.png"); -} - -int get_load_avg(request_rec *r) -{ - FILE *loadavg = fopen("/proc/loadavg", "r"); - int avg = 1000; - - if (!loadavg) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "failed to read /proc/loadavg"); + if (n < 1) return 1000; - } - if (fscanf(loadavg, "%d", &avg) != 1) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "failed to parse /proc/loadavg"); - fclose(loadavg); - return 1000; - } - fclose(loadavg); - - return avg; + else + return loadavg[0]; } -static int tile_dirty(request_rec *r, int x, int y, int z, const char *path) +static int tile_handler_dirty(request_rec *r) { - request_tile(r, x,y,z,path, 1); - return error_message(r, "Rendering request submitted"); + if(strcmp(r->handler, "tile_dirty")) + return DECLINED; + + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "handler(%s), uri(%s), filename(%s), path_info(%s)", + // r->handler, r->uri, r->filename, r->path_info); + + request_tile(r, 1); + return error_message(r, "Tile submitted for rendering"); } - -static int get_tile(request_rec *r, int x, int y, int z, const char *abs_path, const char *rel_path) +static int tile_storage_hook(request_rec *r) { - int avg = get_load_avg(r); + int avg; enum tileState state; - if (avg > MAX_LOAD_ANY) { - // we're too busy to send anything now - return error_message(r, "error: Load above MAX_LOAD_ANY threshold %d > %d", avg, MAX_LOAD_ANY); - } + if (!r->handler || strcmp(r->handler, "tile_serve")) + return DECLINED; - state = tile_state(r, abs_path); + avg = get_load_avg(r); + state = tile_state(r); switch (state) { case tileCurrent: - return serve_tile(r, rel_path); + return OK; break; case tileOld: if (avg > MAX_LOAD_OLD) { // Too much load to render it now, mark dirty but return old tile - tile_dirty(r, x, y, z, abs_path); - return serve_tile(r, rel_path); + request_tile(r, 1); + return OK; } break; case tileMissing: if (avg > MAX_LOAD_MISSING) { - tile_dirty(r, x, y, z, abs_path); - return error_message(r, "error: File missing and load above MAX_LOAD_MISSING threshold %d > %d", avg, MAX_LOAD_MISSING); + request_tile(r, 1); + return HTTP_NOT_FOUND; } break; } - if (request_tile(r, x,y,z,abs_path, 0)) - return serve_tile(r, rel_path); + if (request_tile(r, 0)) + return OK; if (state == tileOld) - return serve_tile(r, rel_path); + return OK; - return error_message(r, "rendering failed for %s", rel_path); + return HTTP_NOT_FOUND; } -static int tile_status(request_rec *r, int x, int y, int z, const char *path) +static int tile_handler_status(request_rec *r) { // FIXME: Apache already has most, if not all, this info recorded in r->fileinfo, use this instead! struct stat buf; @@ -347,8 +380,14 @@ static int tile_status(request_rec *r, int x, int y, int z, const char *path) char AtimeStr[32]; // At least 26 according to man ctime_r char *p; - if (stat(path, &buf)) - return error_message(r, "Unable to find a tile at %s", path); + if(strcmp(r->handler, "tile_status")) + return DECLINED; + + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "handler(%s), uri(%s), filename(%s), path_info(%s)", + // r->handler, r->uri, r->filename, r->path_info); + + if (stat(r->filename, &buf)) + return error_message(r, "Unable to find a tile at %s", r->filename); now = time(NULL); old = (buf.st_mtime < getPlanetTime(r)); @@ -367,76 +406,26 @@ static int tile_status(request_rec *r, int x, int y, int z, const char *path) return error_message(r, "Tile is %s. Last rendered at %s", old ? "due to be rendered" : "clean", MtimeStr); } -static const char *xyz_to_path(char *path, size_t len, int x, int y, int z) -{ -#ifdef DIRECTORY_HASH - // Directory hashing is optimised for z18 max (2^18 tiles split into 2 levels of 2^9) - //snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d/%03d/%03d/%03d/%03d.png", z, - // x/512 x%512, y/512, y%512); - - // We attempt to cluseter the tiles so that a 16x16 square of tiles will be in a single directory - // Hash stores our 40 bit result of mixing the 20 bits of the x & y co-ordinates - // 4 bits of x & y are used per byte of output - unsigned char i, hash[5]; - - for (i=0; i<5; i++) { - hash[i] = ((x & 0x0f) << 4) | (y & 0x0f); - x >>= 4; - y >>= 4; - } - snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d/%u/%u/%u/%u/%u.png", z, hash[4], hash[3], hash[2], hash[1], hash[0]); -#else - snprintf(path, PATH_MAX, WWW_ROOT TILE_PATH "/%d/%d/%d.png", z, x, y); -#endif - return path + strlen(WWW_ROOT); -} - - -static int tile_handler(request_rec *r) +static int tile_translate(request_rec *r) { int x, y, z, n, limit; char option[11]; int oob; char abs_path[PATH_MAX]; - const char *rel_path; option[0] = '\0'; - if(strcmp(r->handler, "tile")) - return DECLINED; - - //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "handler(%s), uri(%s), filename(%s), path_info(%s)", - // r->handler, r->uri, r->filename, r->path_info); - - if (!strncmp(r->uri, HASH_PATH, strlen(HASH_PATH))) { - // Add cache expiry headers on the hashed tiles - ap_add_output_filter("MOD_TILE", NULL, r, r->connection); - return DECLINED; - } + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "translate uri(%s)", r->uri); /* URI = ...///.png[/option] */ n = sscanf(r->uri, TILE_PATH "/%d/%d/%d.png/%10s", &z, &x, &y, option); /* The original rewrite config matched anything that ended with 3 numbers */ //if (n < 3) // n = sscanf(r->uri, TILE_PATH "%*[^0-9]%d/%d/%d.png/%10s", &z, &x, &y, option); -#if 0 - if (n < 3) - n = sscanf(r->uri, TILE_PATH "//%d/%d/%d.png/%10s", &z, &x, &y, option); - if (n < 3) - n = sscanf(r->uri, TILE_PATH "/%*[^/]/%d/%d/%d.png/%10s", &z, &x, &y, option); - if (n < 3) - n = sscanf(r->uri, TILE_PATH "/%*[^/]//%d/%d/%d.png/%10s", &z, &x, &y, option); - if (n < 3) - n = sscanf(r->uri, TILE_PATH "/%*[^/]/%*[^/]/%d/%d/%d.png/%10s", &z, &x, &y, option); -#endif - if (n < 3) { - //return error_message(r, "unable to process: %s", r->path_info); - return DECLINED; - } - // Generate the tile filename - rel_path = xyz_to_path(abs_path, sizeof(abs_path), x, y, z); - //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "abs_path(%s), rel_path(%s)", abs_path, rel_path); + if (n < 3) + return DECLINED; + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "z(%d) x(%d) y(%d)", z, x, y); // Validate tile co-ordinates @@ -447,27 +436,91 @@ static int tile_handler(request_rec *r) oob = (x < 0 || x > limit || y < 0 || y > limit); } - if (n == 3) { - ap_add_output_filter("MOD_TILE", NULL, r, r->connection); - return oob ? serve_blank(r) : get_tile(r, x, y, z, abs_path, rel_path); + if (oob) + return HTTP_NOT_FOUND; + +#if 1 + // Generate the tile filename + xyz_to_meta(abs_path, sizeof(abs_path), x, y, z); + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "abs_path(%s), rel_path(%s)", abs_path, rel_path); + r->filename = apr_pstrdup(r->pool, abs_path); +#else + //r->filename = apr_psprintf(r->pool, "tile:%d/%d/%d",z,x,y); +#endif + if (n == 4) { + if (!strcmp(option, "status")) + r->handler = "tile_status"; + else if (!strcmp(option, "dirty")) + r->handler = "tile_dirty"; + else + return DECLINED; + } else + r->handler = "tile_serve"; + + return OK; +} + + + +static int tile_handler_serve(request_rec *r) +{ + int x, y, z, n, limit, oob; + char *buf; + size_t len; + const int tile_max = 1024 * 1024; + + if(strcmp(r->handler, "tile_serve")) + return DECLINED; + + /* URI = ...///.png[/option] */ + n = sscanf(r->uri, TILE_PATH "/%d/%d/%d", &z, &x, &y); + if (n != 3) + return 0; + + // Validate tile co-ordinates + oob = (z < 0 || z > MAX_ZOOM); + if (!oob) { + // valid x/y for tiles are 0 ... 2^zoom-1 + limit = (1 << z) - 1; + oob = (x < 0 || x > limit || y < 0 || y > limit); } - if (n == 4) { - if (oob) - return error_message(r, "The tile co-ordinates that you specified are invalid"); - if (!strcmp(option, "status")) - return tile_status(r, x, y, z, abs_path); - if (!strcmp(option, "dirty")) - return tile_dirty(r, x, y, z, abs_path); - return error_message(r, "Unknown option"); + if (oob) + return HTTP_NOT_FOUND; + + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "serve handler(%s), uri(%s), filename(%s), path_info(%s)", + // r->handler, r->uri, r->filename, r->path_info); + + buf = malloc(tile_max); + if (!buf) + return HTTP_INTERNAL_SERVER_ERROR; + + len = tile_read(x,y,z,buf, tile_max); + if (len > 0) { + ap_set_content_type(r, "image/png"); + ap_set_content_length(r, len); + add_expiry(r); + ap_rwrite(buf, len, r); + free(buf); + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "fulfilled via meta"); + return OK; } + free(buf); + //ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "len = %d", len); + return DECLINED; } + + static void register_hooks(__attribute__((unused)) apr_pool_t *p) { ap_register_output_filter("MOD_TILE", expires_filter, NULL, APR_HOOK_MIDDLE); - ap_hook_handler(tile_handler, NULL, NULL, APR_HOOK_FIRST); + ap_hook_handler(tile_handler_serve, NULL, NULL, APR_HOOK_MIDDLE); + ap_hook_handler(tile_handler_dirty, NULL, NULL, APR_HOOK_MIDDLE); + ap_hook_handler(tile_handler_status, NULL, NULL, APR_HOOK_MIDDLE); + ap_hook_translate_name(tile_translate, NULL, NULL, APR_HOOK_FIRST); + ap_hook_map_to_storage(tile_storage_hook, NULL, NULL, APR_HOOK_FIRST); } module AP_MODULE_DECLARE_DATA tile_module = diff --git a/modules.mk b/modules.mk index 9187e95..61553ff 100644 --- a/modules.mk +++ b/modules.mk @@ -2,7 +2,7 @@ # this is used/needed by the APACHE2 build system # -MOD_TILE = mod_tile +MOD_TILE = mod_tile dir_utils store mod_tile.la: ${MOD_TILE:=.slo} $(SH_LINK) -rpath $(libexecdir) -module -avoid-version ${MOD_TILE:=.lo} diff --git a/protocol.h b/protocol.h index de1700e..0b26911 100644 --- a/protocol.h +++ b/protocol.h @@ -27,7 +27,6 @@ struct protocol { int x; int y; int z; - char path[TILE_PATH_MAX]; // FIXME: this is a really bad idea since it allows wrties to arbitrrary stuff }; #ifdef __cplusplus diff --git a/render_old.c b/render_old.c index ad76ffc..bec2cdd 100644 --- a/render_old.c +++ b/render_old.c @@ -106,7 +106,7 @@ int process_loop(int fd, int x, int y, int z) cmd.y = y; //strcpy(cmd.path, "/tmp/foo.png"); - //printf("Sending request\n"); + //printf("Sending request\n"); ret = send(fd, &cmd, sizeof(cmd), 0); if (ret != sizeof(cmd)) { perror("send error"); @@ -196,8 +196,8 @@ static void descend(int fd, const char *search) continue; } p = strrchr(path, '.'); - if (p && !strcmp(p, ".png")) { - //printf("Found tile %s\n", path); + if (p && !strcmp(p, ".meta")) { + //printf("Found tile %s\n", path); process(fd, path); } } diff --git a/speedtest.cpp b/speedtest.cpp index ee11018..e60dcbe 100644 --- a/speedtest.cpp +++ b/speedtest.cpp @@ -219,7 +219,7 @@ int main(int argc, char **argv) for (x=xmin; x<=xmax; x++) { for (y=ymin; y<=ymax; y++) { struct stat s; - xyz_to_path(name, sizeof(name), x, y, z); + xyz_to_meta(name, sizeof(name), x, y, z); if (stat(name, &s) < 0) { // File doesn't exist ret = process_loop(fd, x, y, z); diff --git a/store.c b/store.c new file mode 100644 index 0000000..b1fb619 --- /dev/null +++ b/store.c @@ -0,0 +1,308 @@ +/* Meta-tile optimised file storage + * + * Instead of storing each individual tile as a file, + * bundle the 8x8 meta tile into a special meta-file. + * This reduces the Inode usage and more efficient + * utilisation of disk space. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "store.h" +#include "render_config.h" +#include "dir_utils.h" + +int read_from_meta(int x, int y, int z, char *buf, size_t sz) +{ + char path[PATH_MAX]; + int meta_offset, fd; + unsigned int pos; + char header[4096]; + struct meta_layout *m = (struct meta_layout *)header; + size_t file_offset, tile_size; + + meta_offset = xyz_to_meta(path, sizeof(path), x, y, z); + + fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + pos = 0; + while (pos < sizeof(header)) { + size_t len = sizeof(header) - pos; + int got = read(fd, header + pos, len); + if (got < 0) { + close(fd); + return -2; + } else if (got > 0) { + pos += got; + } else { + break; + } + } + if (pos < sizeof(struct meta_layout)) { + fprintf(stderr, "Meta file %s too small to contain header\n", path); + return -3; + } + if (memcmp(m->magic, META_MAGIC, strlen(META_MAGIC))) { + fprintf(stderr, "Meta file %s header magic mismatch\n", path); + return -4; + } +#if 1 + // Currently this code only works with fixed metatile sizes (due to xyz_to_meta above) + if (m->count != (METATILE * METATILE)) { + fprintf(stderr, "Meta file %s header bad count %d != %d\n", path, m->count, METATILE * METATILE); + return -5; + } +#else + if (m->count < 0 || m->count > 256) { + fprintf(stderr, "Meta file %s header bad count %d\n", path, m->count); + return -5; + } +#endif + file_offset = m->index[meta_offset].offset; + tile_size = m->index[meta_offset].size; + + if (lseek(fd, file_offset, SEEK_SET) < 0) { + fprintf(stderr, "Meta file %s seek error %d\n", path, m->count); + return -6; + } + if (tile_size > sz) { + fprintf(stderr, "Truncating tile %zd to fit buffer of %zd\n", tile_size, sz); + tile_size = sz; + } + pos = 0; + while (pos < tile_size) { + size_t len = tile_size - pos; + int got = read(fd, buf + pos, len); + if (got < 0) { + close(fd); + return -7; + } else if (got > 0) { + pos += got; + } else { + break; + } + } + close(fd); + return pos; +} + +int read_from_file(int x, int y, int z, char *buf, size_t sz) +{ + char path[PATH_MAX]; + int fd; + size_t pos; + + xyz_to_path(path, sizeof(path), x, y, z); + + fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + pos = 0; + while (pos < sz) { + size_t len = sz - pos; + int got = read(fd, buf + pos, len); + if (got < 0) { + close(fd); + return -2; + } else if (got > 0) { + pos += got; + } else { + break; + } + } + if (pos == sz) { + fprintf(stderr, "file %s truncated at %zd bytes\n", path, sz); + } + close(fd); + return pos; +} + +int tile_read(int x, int y, int z, char *buf, int sz) +{ + int r; + + r = read_from_meta(x, y, z, buf, sz); + if (r >= 0) + return r; + + return read_from_file(x, y, z, buf, sz); +} + + +void process_meta(int x, int y, int z) +{ + int fd; + int ox, oy, limit; + size_t offset, pos; + const int buf_len = 10 * 1024 * 1024; // To store all tiles in this .meta + char *buf; + struct meta_layout *m; + char meta_path[PATH_MAX]; + + buf = (char *)malloc(buf_len); + if (!buf) + return; + + m = (struct meta_layout *)buf; + offset = sizeof(struct meta_layout) + (sizeof(struct entry) * (METATILE * METATILE)); + memset(buf, 0, offset); + + limit = (1 << z); + limit = MIN(limit, METATILE); + + for (ox=0; ox < limit; ox++) { + for (oy=0; oy < limit; oy++) { + //fprintf(stderr, "Process %d/%d/%d\n", num, ox, oy); + int len = read_from_file(x + ox, y + oy, z, buf + offset, buf_len - offset); + int mt = xyz_to_meta(meta_path, sizeof(meta_path), x + ox, y + oy, z); + if (len <= 0) { +#if 1 + fprintf(stderr, "Problem reading sub tiles for metatile x(%d) y(%d) z(%d), got %d\n", x, y, z, len); + free(buf); + return; +#else + m->index[mt].offset = 0; + m->index[mt].size = 0; +#endif + } else { + m->index[mt].offset = offset; + m->index[mt].size = len; + offset += len; + } + } + } + m->count = METATILE * METATILE; + memcpy(m->magic, META_MAGIC, strlen(META_MAGIC)); + m->x = x; + m->y = y; + m->z = z; + + xyz_to_meta(meta_path, sizeof(meta_path), x, y, z); + fd = open(meta_path, O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd < 0) { + fprintf(stderr, "Error creating file: %s\n", meta_path); + free(buf); + return; + } + + pos = 0; + while (pos < offset) { + int len = write(fd, buf + pos, offset - pos); + if (len < 0) { + perror("Writing file"); + free(buf); + close(fd); + return; + } else if (len > 0) { + pos += len; + } else { + break; + } + } + close(fd); + free(buf); + printf("Produced .meta: %s\n", meta_path); + + // Remove raw .png's + for (ox=0; ox < limit; ox++) { + for (oy=0; oy < limit; oy++) { + xyz_to_path(meta_path, sizeof(meta_path), x + ox, y + oy, z); + if (unlink(meta_path)<0) + perror(meta_path); + } + } +} + +void process_pack(const char *name) +{ + char meta_path[PATH_MAX]; + int x, y, z; + int meta_offset; + + if (path_to_xyz(name, &x, &y, &z)) + return; + + // Launch the .meta creation for only 1 tile of the whole block + meta_offset = xyz_to_meta(meta_path, sizeof(meta_path), x, y, z); + //fprintf(stderr,"Requesting x(%d) y(%d) z(%d) - mo(%d)\n", x, y, z, meta_offset); + + if (meta_offset == 0) + process_meta(x, y, z); +} + +static void write_tile(int x, int y, int z, const char *buf, size_t sz) +{ + int fd; + char path[PATH_MAX]; + size_t pos; + + xyz_to_path(path, sizeof(path), x, y, z); + fd = open(path, O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd < 0) { + fprintf(stderr, "Error creating file: %s\n", path); + return; + } + + pos = 0; + while (pos < sz) { + int len = write(fd, buf + pos, sz - pos); + if (len < 0) { + perror("Writing file"); + close(fd); + return; + } else if (len > 0) { + pos += len; + } else { + break; + } + } + close(fd); + printf("Produced tile: %s\n", path); +} + +void process_unpack(const char *name) +{ + char meta_path[PATH_MAX]; + int x, y, z; + int ox, oy, limit; + const int buf_len = 1024 * 1024; + char *buf; + + // path_to_xyz is valid for meta tile names as well + if (path_to_xyz(name, &x, &y, &z)) + return; + + buf = (char *)malloc(buf_len); + if (!buf) + return; + + limit = (1 << z); + limit = MIN(limit, METATILE); + + for (ox=0; ox < limit; ox++) { + for (oy=0; oy < limit; oy++) { + int len = read_from_meta(x + ox, y + oy, z, buf, buf_len); + + if (len <= 0) + fprintf(stderr, "Failed to get tile x(%d) y(%d) z(%d)\n", x + ox, y + oy, z); + else + write_tile(x + ox, y + oy, z, buf, len); + } + } + // Remove the .meta file + xyz_to_meta(meta_path, sizeof(meta_path), x, y, z); + if (unlink(meta_path)<0) + perror(meta_path); +} + diff --git a/store.h b/store.h new file mode 100644 index 0000000..397d928 --- /dev/null +++ b/store.h @@ -0,0 +1,42 @@ +#ifndef STORE_H +#define STORE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +int tile_read(int x, int y, int z, char *buf, int sz); + +#define META_MAGIC "META" +//static const char meta_magic[4] = { 'M', 'E', 'T', 'A' }; + +struct entry { + int offset; + int size; +}; + +struct meta_layout { + char magic[4]; + int count; // METATILE ^ 2 + int x, y, z; // lowest x,y of this metatile, plus z + struct entry index[]; // count entries + // Followed by the tile data + // The index offsets are measured from the start of the file +}; + + +int read_from_file(int x, int y, int z, char *buf, size_t sz); +int read_from_meta(int x, int y, int z, char *buf, size_t sz); + +void process_meta(int x, int y, int z); +void process_pack(const char *name); +void process_unpack(const char *name); + + + +#ifdef __cplusplus +} +#endif +#endif