| | Gabor Hojtsy | | Mark Kronsbein | | Jan Fabry | | SQLite version Authors: | | Nilgün Belma Bugüner | | Mehdi Achour | | Maciej Sobaczewski | +----------------------------------------------------------------------+ */ error_reporting(E_ALL); set_time_limit(0); // include required files include '../include/init.inc.php'; include '../include/lib_proj_lang.inc.php'; $DOCS = GIT_DIR; // Test the languages: $LANGS = array_keys($LANGUAGES); $langc = count($LANGS); for ($i = 0; $i < $langc; $i++) { if (!is_dir($DOCS . $LANGS[$i])) { echo "Error: the \"{$LANGS[$i]}\" lang doesn't exist in dir {$DOCS}, skipping..\n"; unset($LANGS[$i]); } } if (count($LANGS) == 0) { echo "Error: No language to revcheck, exiting.\n"; exit; } $CREATE = <<path . '/' . $this->name , '/' ); } } class FileStatusEnum { const Untranslated = 'Untranslated'; const RevTagProblem = 'RevTagProblem'; const TranslatedWip = 'TranslatedWip'; const TranslatedOk = 'TranslatedOk'; const TranslatedOld = 'TranslatedOld'; const TranslatedCritial = 'TranslatedCritial'; const NotInEnTree = 'NotInEnTree'; } class TranslatorInfo { public $name; public $email; public $nick; public $vcs; public $files_uptodate; public $files_outdated; public $files_wip; public $files_sum; public $files_other; public function __construct() { $this->files_uptodate = 0; $this->files_outdated = 0; $this->files_wip = 0; $this->files_sum = 0; $this->files_other = 0; } public static function getKey( $fileStatus ) { switch ( $fileStatus ) { case FileStatusEnum::RevTagProblem: case FileStatusEnum::TranslatedOld: case FileStatusEnum::TranslatedCritial: case FileStatusEnum::NotInEnTree: return "files_outdated"; break; case FileStatusEnum::TranslatedWip: return "files_wip"; break; case FileStatusEnum::TranslatedOk: return "files_uptodate"; break; default: return "files_other"; } } } // Get a multidimensional array with tag attributes function parse_attr_string ( $tags_attrs ) { $tag_attrs_processed = array(); foreach($tags_attrs as $attrib_list) { preg_match_all("!(.+)=\\s*([\"'])\\s*(.+)\\2!U", $attrib_list, $attribs); $attrib_array = array(); foreach ($attribs[1] as $num => $attrname) { $attrib_array[trim($attrname)] = trim($attribs[3][$num]); } $tag_attrs_processed[] = $attrib_array; } return $tag_attrs_processed; } function computeTranslatorStatus( $lang, $enFiles, $trFiles ) { global $SQL_BUFF, $DOCS, $LANGUAGES; $translation_xml = $DOCS . $lang . "/translation.xml"; $charset = 'utf-8'; if (!file_exists($translation_xml)) { return []; } $txml = join("", file($translation_xml)); $txml = preg_replace("/\\s+/", " ", $txml); $intro = "No intro available for the {$LANGUAGES[$lang]} translation of the manual."; if ( preg_match("!(.+)!s", $txml, $match) ) $intro = SQLite3::escapeString(@iconv($charset, 'UTF-8//IGNORE', trim($match[1]))); $SQL_BUFF .= "INSERT INTO descriptions VALUES ('$lang', '$intro');\n"; $pattern = "!!U"; preg_match_all($pattern, $txml, $matches); $translators = parse_attr_string($matches[1]); $translatorInfos = []; $unknownInfo = new TranslatorInfo(); $unknownInfo->nick = "unknown"; $translatorInfos["unknown"] = $unknownInfo; foreach ($translators as $key => $translator) { $info = new TranslatorInfo(); $info->name = $translator["name"]; $info->email = $translator["email"]; $info->nick = $translator["nick"]; $info->vcs = isset($translator["vcs"]) ? $translator["vcs"] : ''; $translatorInfos[$info->nick] = $info; } foreach( $enFiles as $key => $enFile ) { $info_exists = false; if (array_key_exists($enFile->getKey(), $trFiles)) { $trFile = $trFiles[$enFile->getKey()]; $statusKey = TranslatorInfo::getKey($trFile->syncStatus); if (array_key_exists($trFile->maintainer, $translatorInfos)) { $translatorInfos[$trFile->maintainer]->$statusKey++; $translatorInfos[$trFile->maintainer]->files_sum++; $info_exists = true; } } if (!$info_exists) { $translatorInfos["unknown"]->$statusKey++; $translatorInfos["unknown"]->files_sum++; } } foreach ($translatorInfos as $key => $person) { if ($person->nick != "unknown" ) { $nick = SQLite3::escapeString($person->nick); $name = SQLite3::escapeString(@iconv($charset, 'UTF-8//IGNORE', $person->name)); $email = SQLite3::escapeString($person->email); $vcs = SQLite3::escapeString($person->vcs); $SQL_BUFF .= "INSERT INTO translators VALUES ('$lang', '$nick', '$name', '$email', '$vcs', $person->files_uptodate, $person->files_outdated, $person->files_wip, $person->files_sum, $person->files_other);\n"; } } } function populateFileTree( $lang ) { global $DOCS; $dir = new \DirectoryIterator( $DOCS . $lang ); if ( $dir === false ) { print "$lang is not a directory.\n"; exit; } $cwd = getcwd(); $ret = array(); chdir( $DOCS . $lang ); populateFileTreeRecurse( $lang , "." , $ret ); chdir( $cwd ); return $ret; } function populateFileTreeRecurse( $lang , $path , & $output ) { global $DOCS, $SQL_BUFF; $dir = new DirectoryIterator( $path ); if ( $dir === false ) { print "$path is not a directory.\n"; exit; } $todoPaths = []; $trimPath = ltrim( $path , "./"); foreach( $dir as $entry ) { $filename = $entry->getFilename(); if ( $filename[0] == '.' ) continue; if ( substr( $filename , 0 , 9 ) == "entities." ) continue; if ( $entry->isDir() ) { $todoPaths[] = $path . '/' . $entry->getFilename(); continue; } if ( $entry->isFile() ) { $ignoredFileNames = [ 'README.md', 'translation.xml', 'readme.first', 'license.xml', 'extensions.xml', 'versions.xml', 'book.developer.xml', 'contributors.ent', 'contributors.xml', 'README', 'DO_NOT_TRANSLATE', 'rsusi.txt', 'missing-ids.xml', ]; $ignoredDirectories = [ 'chmonly', 'output', ]; $ignoredFullPaths = [ 'appendices/reserved.constants.xml', 'appendices/extensions.xml', 'reference/datetime/timezones.xml', ]; if( in_array($trimPath, $ignoredDirectories, true) || in_array($filename, $ignoredFileNames, true) || (strpos($filename, 'entities.') === 0) || !in_array(substr($filename, -3), ['xml','ent'], true) || (substr($filename, -13) === 'PHPEditBackup') || (in_array($trimPath . '/' .$filename, $ignoredFullPaths, true)) ) continue; $file = new FileStatusInfo; $file->path = $trimPath; $file->name = $filename; $file->size = filesize( $path . '/' . $filename ); $file->syncStatus = null; if ( $lang != 'en' ) { parseRevisionTag( $entry->getPathname() , $file ); $path_en = $DOCS . 'en/' . $trimPath . '/' . $filename; if( !is_file($path_en) ) //notinen { $filesize = $file->size < 1024 ? 1 : floor( $file->size / 1024 ); $SQL_BUFF .= "INSERT INTO notinen VALUES ('$lang', '$trimPath', '$filename', $filesize);\n"; } else { $output[ $file->getKey() ] = $file; } } else { $output[ $file->getKey() ] = $file; } } } sort( $todoPaths ); foreach( $todoPaths as $path ) populateFileTreeRecurse( $lang , $path , $output ); } function parseRevisionTag( $filename , FileStatusInfo $file ) { $fp = fopen( $filename , "r" ); $contents = fread( $fp , 1024 ); fclose( $fp ); // No match before the preg $match = array (); $regex = "''U"; if (preg_match ($regex , $contents , $match )) { $file->hash = trim( $match[1] ); $file->maintainer = trim( $match[2] ); $file->completion = trim( $match[3] ); } if ( $file->hash == null or strlen( $file->hash ) != 40 or $file->maintainer == null or $file->completion == null ) $file->syncStatus = FileStatusEnum::RevTagProblem; $regex = "//U"; $match = array(); preg_match ( $regex , $contents , $match ); if ( count( $match ) == 2 ) $file->credits = str_replace( ' ' , '' , trim( $match[1] ) ); else $file->credits = ''; } function captureGitValues( & $output ) { global $DOCS; $cwd = getcwd(); chdir( $DOCS . 'en' ); $fp = popen( "git --no-pager log --name-only" , "r" ); $hash = null; $skipThisCommit = false; while ( ( $line = fgets( $fp ) ) !== false ) { if ( substr( $line , 0 , 7 ) == "commit " ) { $hash = trim( substr( $line , 7 ) ); $skipThisCommit = false; continue; } if ( strpos( $line , 'Date:' ) === 0 ) continue; if ( trim( $line ) == "" ) continue; if ( substr( $line , 0 , 4 ) == ' ' ) { if ( stristr( $line, '[skip-revcheck]' ) !== false ) { $skipThisCommit = true; } continue; } if ( strpos( $line , ': ' ) > 0 ) continue; $filename = trim( $line ); if ( isset( $output[$filename] ) ) continue; $output[$filename]['hash'] = $hash; $output[$filename]['skip'] = $skipThisCommit; } pclose( $fp ); chdir( $cwd ); } /** * Script execution **/ $time_start = microtime(true); $path = null; $id = 0; asort( $enFiles ); foreach( $enFiles as $key => $en ) { if ( $path !== $en->path ) { $id++; $path = $en->path; $path2 = $path == '' ? '/' : $path; $SQL_BUFF .= "INSERT INTO dirs VALUES ($id, '$path2');\n"; } $size = $en->size < 1024 ? 1 : floor( $en->size / 1024 ); $filename = $path . ($path == '' ? '' : '/') . $en->name; $en->hash = null; if ( isset( $gitData[ $filename ] ) ) { $en->hash = $gitData[ $filename ]['hash']; $en->skip = $gitData[ $filename ]['skip']; } else print "Warn: No hash for en/$filename\n"; $SQL_BUFF .= "INSERT INTO enfiles VALUES ($id, '$en->name', '$en->hash', $size);\n"; foreach( $LANGS as $lang ) { $trFile = isset( $trFiles[$lang][$filename] ) ? $trFiles[$lang][$filename] : null; if ( $trFile == null ) // Untranslated { $SQL_BUFF .= "INSERT INTO Untranslated VALUES ($id, '$lang', '$en->name', $size);\n"; } else if ($trFile->syncStatus == FileStatusEnum::RevTagProblem) { $SQL_BUFF .= "INSERT INTO translated VALUES ($id, '$lang', '$en->name', '$trFile->hash', $size, '$trFile->maintainer', '$trFile->completion', '$trFile->syncStatus', 0, 0);\n"; } else { $additions = $deletions = -1; if ( $en->hash == $trFile->hash ){ $trFile->syncStatus = FileStatusEnum::TranslatedOk; } elseif ( $trFile->hash != null and strlen( $trFile->hash ) == 40 ) { $trFile->syncStatus = FileStatusEnum::TranslatedOld; $cwd = getcwd(); chdir( $DOCS . 'en' ); $subject = `git diff --numstat $trFile->hash -- {$filename}`; chdir( $cwd ); if ( $subject ) { preg_match('/(\d+)\s+(\d+)/', $subject, $matches); if ($matches) [, $additions, $deletions] = $matches; } } if ( $trFile->completion != null && $trFile->completion != "ready" ) $trFile->syncStatus = FileStatusEnum::TranslatedWip; if ( $en->skip ) { $cwd = getcwd(); chdir( $DOCS . 'en' ); $hashes = explode ( "\n" , `git log -2 --format=%H -- {$filename}` ); chdir( $cwd ); if ( $hashes[1] == $trFile->hash ) $trFile->syncStatus = FileStatusEnum::TranslatedOk; } $SQL_BUFF .= "INSERT INTO translated VALUES ($id, '$lang', '$en->name', '$trFile->hash', $size, '$trFile->maintainer', '$trFile->completion', '$trFile->syncStatus', $additions, $deletions);\n"; } } } foreach( $LANGS as $lang ) { computeTranslatorStatus( $lang, $enFiles, $trFiles[$lang] ); } $db_name = SQLITE_DIR . 'rev.php.sqlite'; $tmp_db = SQLITE_DIR . 'rev.php.tmp.sqlite'; // 1 - Drop the old database and create the new one if (is_file($tmp_db)) { echo "Temporary database found: remove.\n"; if (!@unlink($tmp_db)) { echo "Error: Can't remove temporary database\n"; exit; } } // 2 - Create the new database try { $db = new SQLite3($tmp_db); /* Didn't throw exception at some point? */ if (!$db) { throw Exception("Cant open $tmp_db"); } } catch(Exception $e) { echo $e->getMessage(); echo "Could not open $tmp_db"; exit; } $db->exec($CREATE); // 3 - Fill in the description table while cleaning the langs // without revision.xml file // 4 - Recurse in the manual seeking for files and fill $SQL_BUFF // 5 - Query $SQL_BUFF and exit $db->exec('BEGIN TRANSACTION'); $db->exec($SQL_BUFF); $db->exec('COMMIT'); $db->close(); echo "Copying temporary database to final database\n"; copy($tmp_db, $db_name); @unlink($tmp_db); $time = microtime(true) - $time_start; echo "Time of generation: $time s\n"; echo "End\n";