mirror of
https://github.com/dokuwiki/dokuwiki-extensionmirror.git
synced 2025-07-22 17:41:16 +00:00
initial working version
This commit is contained in:
Andreas Gohr
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
data/
|
||||
vendor/
|
||||
9
README.md
Normal file
9
README.md
Normal file
@ -0,0 +1,9 @@
|
||||
# DokuWiki Extension Mirror Script
|
||||
|
||||
This tool will download all available DokuWiki plugins/templates and extract them to `data/src`. This is useful for DokuWiki developers who need to check if a certain DokuWiki function is used by any plugins.
|
||||
|
||||
It's also helpful to figure out which extensions are no longer downloadable. An error log is placed into `./data/meta/error.log`.
|
||||
|
||||
Downloaded archives are kept in `data/meta` when extraction fails to ease debugging.
|
||||
|
||||
The tool uses the DokuWiki [plugin repository API](https://github.com/splitbrain/dokuwiki-plugin-pluginrepo/blob/master/README-API) and will only download a plugin again when it's version changes.
|
||||
22
composer.json
Normal file
22
composer.json
Normal file
@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "splitbrain/dokuwiki-extensionmirror",
|
||||
"description": "download all dokuwiki plugins",
|
||||
"type": "project",
|
||||
"license": "MIT",
|
||||
"authors": [
|
||||
{
|
||||
"name": "Andreas Gohr",
|
||||
"email": "andi@splitbrain.org"
|
||||
}
|
||||
],
|
||||
"require": {
|
||||
"splitbrain/php-cli": "^1.1",
|
||||
"ptcong/easyrequest": "^1.0",
|
||||
"splitbrain/php-archive": "^1.0"
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"splitbrain\\DokuWikiExtensionMirror\\": "src"
|
||||
}
|
||||
}
|
||||
}
|
||||
219
composer.lock
generated
Normal file
219
composer.lock
generated
Normal file
@ -0,0 +1,219 @@
|
||||
{
|
||||
"_readme": [
|
||||
"This file locks the dependencies of your project to a known state",
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "4c8de48ea783fc3bf0c1f10841c59737",
|
||||
"packages": [
|
||||
{
|
||||
"name": "psr/http-message",
|
||||
"version": "1.0.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/php-fig/http-message.git",
|
||||
"reference": "f6561bf28d520154e4b0ec72be95418abe6d9363"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/php-fig/http-message/zipball/f6561bf28d520154e4b0ec72be95418abe6d9363",
|
||||
"reference": "f6561bf28d520154e4b0ec72be95418abe6d9363",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"branch-alias": {
|
||||
"dev-master": "1.0.x-dev"
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Psr\\Http\\Message\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "PHP-FIG",
|
||||
"homepage": "http://www.php-fig.org/"
|
||||
}
|
||||
],
|
||||
"description": "Common interface for HTTP messages",
|
||||
"homepage": "https://github.com/php-fig/http-message",
|
||||
"keywords": [
|
||||
"http",
|
||||
"http-message",
|
||||
"psr",
|
||||
"psr-7",
|
||||
"request",
|
||||
"response"
|
||||
],
|
||||
"time": "2016-08-06T14:39:51+00:00"
|
||||
},
|
||||
{
|
||||
"name": "ptcong/easyrequest",
|
||||
"version": "1.0.6",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/ptcong/easyrequest.git",
|
||||
"reference": "f5c87818e961a9eed50047352d1bf546ca562f86"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/ptcong/easyrequest/zipball/f5c87818e961a9eed50047352d1bf546ca562f86",
|
||||
"reference": "f5c87818e961a9eed50047352d1bf546ca562f86",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3",
|
||||
"psr/http-message": "^1.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"EasyRequest\\": "src/"
|
||||
},
|
||||
"files": [
|
||||
"src/helpers.php"
|
||||
]
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Phan Thanh Cong",
|
||||
"email": "ptcong90@gmail.com",
|
||||
"homepage": "http://ptcong.com"
|
||||
}
|
||||
],
|
||||
"description": "A light weight PHP http client implements PSR7, use socket/curl for sending requests.",
|
||||
"keywords": [
|
||||
"Socket",
|
||||
"curl",
|
||||
"http",
|
||||
"php",
|
||||
"php http client",
|
||||
"request"
|
||||
],
|
||||
"time": "2016-12-04T14:06:46+00:00"
|
||||
},
|
||||
{
|
||||
"name": "splitbrain/php-archive",
|
||||
"version": "1.0.9",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/splitbrain/php-archive.git",
|
||||
"reference": "2a63b8cf0bfc7fdc0d987c9b7348e639e55cce76"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/splitbrain/php-archive/zipball/2a63b8cf0bfc7fdc0d987c9b7348e639e55cce76",
|
||||
"reference": "2a63b8cf0bfc7fdc0d987c9b7348e639e55cce76",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "4.5.*"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-iconv": "Used for proper filename encode handling",
|
||||
"ext-mbstring": "Can be used alternatively for handling filename encoding"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"splitbrain\\PHPArchive\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Andreas Gohr",
|
||||
"email": "andi@splitbrain.org"
|
||||
}
|
||||
],
|
||||
"description": "Pure-PHP implementation to read and write TAR and ZIP archives",
|
||||
"keywords": [
|
||||
"archive",
|
||||
"extract",
|
||||
"tar",
|
||||
"unpack",
|
||||
"unzip",
|
||||
"zip"
|
||||
],
|
||||
"time": "2017-06-11T06:11:38+00:00"
|
||||
},
|
||||
{
|
||||
"name": "splitbrain/php-cli",
|
||||
"version": "1.1.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/splitbrain/php-cli.git",
|
||||
"reference": "1d6f0bf9eccbfd79d1f4d185ef27573601185c23"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/splitbrain/php-cli/zipball/1d6f0bf9eccbfd79d1f4d185ef27573601185c23",
|
||||
"reference": "1d6f0bf9eccbfd79d1f4d185ef27573601185c23",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=5.3.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "4.5.*"
|
||||
},
|
||||
"suggest": {
|
||||
"psr/log": "Allows you to make the CLI available as PSR-3 logger"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"splitbrain\\phpcli\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Andreas Gohr",
|
||||
"email": "andi@splitbrain.org"
|
||||
}
|
||||
],
|
||||
"description": "Easy command line scripts for PHP with opt parsing and color output. No dependencies",
|
||||
"keywords": [
|
||||
"argparse",
|
||||
"cli",
|
||||
"command line",
|
||||
"console",
|
||||
"getopt",
|
||||
"optparse",
|
||||
"terminal"
|
||||
],
|
||||
"time": "2018-02-02T08:46:12+00:00"
|
||||
}
|
||||
],
|
||||
"packages-dev": [],
|
||||
"aliases": [],
|
||||
"minimum-stability": "stable",
|
||||
"stability-flags": [],
|
||||
"prefer-stable": false,
|
||||
"prefer-lowest": false,
|
||||
"platform": [],
|
||||
"platform-dev": []
|
||||
}
|
||||
7
extensionmirror.php
Executable file
7
extensionmirror.php
Executable file
@ -0,0 +1,7 @@
|
||||
#!/bin/env php
|
||||
<?php
|
||||
|
||||
require 'vendor/autoload.php';
|
||||
|
||||
$cli = new \splitbrain\DokuWikiExtensionMirror\Downloader();
|
||||
$cli->run();
|
||||
222
src/Downloader.php
Normal file
222
src/Downloader.php
Normal file
@ -0,0 +1,222 @@
|
||||
<?php
|
||||
|
||||
namespace splitbrain\DokuWikiExtensionMirror;
|
||||
|
||||
use splitbrain\PHPArchive\Tar;
|
||||
use splitbrain\PHPArchive\Zip;
|
||||
use splitbrain\phpcli\CLI;
|
||||
use splitbrain\phpcli\Exception;
|
||||
use splitbrain\phpcli\Options;
|
||||
|
||||
class Downloader extends CLI
|
||||
{
|
||||
|
||||
const API = 'http://www.dokuwiki.org/lib/plugins/pluginrepo/api.php?fmt=php&order=lastupdate';
|
||||
|
||||
protected $datadir;
|
||||
protected $logfile;
|
||||
|
||||
/**
|
||||
* Register options and arguments on the given $options object
|
||||
*
|
||||
* @param Options $options
|
||||
* @return void
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function setup(Options $options)
|
||||
{
|
||||
$options->setHelp('Download all known DokuWiki extensions');
|
||||
$options->registerOption('datadir', 'Where to store downloaded data', 'd', 'directory');
|
||||
}
|
||||
|
||||
/**
|
||||
* Dowload all the files
|
||||
*
|
||||
* @param Options $options
|
||||
* @return void
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function main(Options $options)
|
||||
{
|
||||
$this->datadir = $options->getOpt('datadir', './data');
|
||||
if (!is_dir($this->datadir)) {
|
||||
if (!mkdir($this->datadir, 0777, true)) throw new Exception('Could not create data dir');
|
||||
}
|
||||
if (!is_dir($this->datadir . '/meta')) {
|
||||
if (!mkdir($this->datadir . '/meta', 0777, true)) throw new Exception('Could not create meta dir');
|
||||
}
|
||||
if (!is_dir($this->datadir . '/src')) {
|
||||
if (!mkdir($this->datadir . '/src', 0777, true)) throw new Exception('Could not create src dir');
|
||||
}
|
||||
|
||||
$this->logfile = $this->datadir . '/meta/error.log';
|
||||
if (file_exists($this->logfile)) unlink($this->logfile);
|
||||
|
||||
$dls = $this->getDownloads();
|
||||
foreach ($dls as $dl) {
|
||||
$this->info('Fetching {p}...', ['p' => $dl['name']]);
|
||||
try {
|
||||
$this->download($dl['name'], $dl['url'], $dl['date']);
|
||||
} catch (\Exception $e) {
|
||||
$this->error($e->getMessage());
|
||||
file_put_contents(
|
||||
$this->logfile,
|
||||
$dl['name'] . "\t" . $e->getMessage() . "\n",
|
||||
FILE_APPEND
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param $name
|
||||
* @param $url
|
||||
* @param $version
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function download($name, $url, $version)
|
||||
{
|
||||
$last = $this->datadir . '/meta/' . $name . '.last';
|
||||
$target = $this->datadir . '/src/' . $name;
|
||||
$tmp = $this->datadir . '/meta/' . $name . '.tmp';
|
||||
$archive = $this->datadir . '/meta/' . $name . '.archive';
|
||||
|
||||
$this->info('Downloading {url}', ['url' => $url]);
|
||||
$request = \EasyRequest\Client::request($url, 'GET', ['follow_redirects' => true]);
|
||||
$response = $request->send();
|
||||
$body = (string)$response->getBody();
|
||||
if ($response->getStatusCode() >= 400) {
|
||||
throw new Exception('Download failed. Status ' . $response->getStatusCode());
|
||||
}
|
||||
if (strlen($body) < 100) {
|
||||
throw new Exception('Download not an archive: ' . $body);
|
||||
}
|
||||
unset($response);
|
||||
|
||||
if (substr($body, 0, 4) === "\x50\x4b\x03\x04") {
|
||||
$extractor = new Zip();
|
||||
} else {
|
||||
$extractor = new Tar();
|
||||
}
|
||||
if (file_exists($archive)) unlink($archive);
|
||||
file_put_contents($archive, $body);
|
||||
$this->info('Downloaded {b} bytes', ['b' => strlen($body)]);
|
||||
unset($body);
|
||||
|
||||
if (is_dir($tmp)) $this->delTree($tmp);
|
||||
$extractor->open($archive);
|
||||
$extractor->extract($tmp);
|
||||
$extractor->close();
|
||||
$this->info('Extracted archive');
|
||||
|
||||
$path = $this->getFilesPath($tmp);
|
||||
if (is_dir($target)) $this->delTree($target);
|
||||
rename($path, $target);
|
||||
file_put_contents($last, $version);
|
||||
|
||||
unlink($archive);
|
||||
$this->delTree($tmp);
|
||||
$this->success('Downloaded {p} version {d}', ['p'=>$name, 'd' => $version]);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get all releases that have not been downloaded, yet
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function getDownloads()
|
||||
{
|
||||
$request = \EasyRequest\Client::request(self::API);
|
||||
$response = $request->send();
|
||||
$results = unserialize($response->getBody());
|
||||
|
||||
$this->info('{cnt} extensions found', ['cnt' => count($results)]);
|
||||
|
||||
$downloads = array();
|
||||
foreach ($results as $extension) {
|
||||
@list($type, $name) = explode(':', $extension['plugin'], 2);
|
||||
if (empty($name)) {
|
||||
$name = $type;
|
||||
$type = 'plugin';
|
||||
}
|
||||
$fullname = "$type-$name";
|
||||
|
||||
if (empty($extension['downloadurl'])) {
|
||||
$this->error('No download for {ext}', ['ext' => $fullname]);
|
||||
file_put_contents(
|
||||
$this->logfile,
|
||||
$fullname . "\tno download URL\n",
|
||||
FILE_APPEND
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($this->needsDownload($fullname, $extension['lastupdate'])) {
|
||||
$downloads[] = [
|
||||
'name' => $fullname,
|
||||
'url' => $extension['downloadurl'],
|
||||
'date' => $extension['lastupdate']
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$this->info('{cnt} extensions need updating', ['cnt' => count($downloads)]);
|
||||
|
||||
return ($downloads);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the version changes since last download
|
||||
*
|
||||
* @param string $fullname
|
||||
* @param string $date
|
||||
* @return bool
|
||||
*/
|
||||
protected function needsDownload($fullname, $date)
|
||||
{
|
||||
$file = $this->datadir . '/meta/' . $fullname . '.last';
|
||||
if (!file_exists($file)) return true;
|
||||
$last = trim(file_get_contents($file));
|
||||
return ($last != $date);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Recursively delete a directory
|
||||
*
|
||||
* @link http://php.net/manual/de/function.rmdir.php#110489
|
||||
* @param string $dir
|
||||
* @return bool
|
||||
*/
|
||||
protected function delTree($dir)
|
||||
{
|
||||
if (!is_dir($dir)) return false;
|
||||
$files = array_diff(scandir($dir), array('.', '..'));
|
||||
foreach ($files as $file) {
|
||||
(is_dir("$dir/$file")) ? $this->delTree("$dir/$file") : unlink("$dir/$file");
|
||||
}
|
||||
return rmdir($dir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the first directory containing php files
|
||||
*
|
||||
* @param string $dir
|
||||
* @return string
|
||||
*/
|
||||
protected function getFilesPath($dir)
|
||||
{
|
||||
$files = glob("$dir/*.php");
|
||||
$dirs = glob("$dir/*", GLOB_ONLYDIR);
|
||||
|
||||
if (!count($files) && count($dirs) === 1) {
|
||||
return $this->getFilesPath($dirs[0]); // go one deeper
|
||||
}
|
||||
return $dir;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user