#!/usr/bin/python3 # Copyright (c) 2000-2014 Synology Inc. All rights reserved. import sys import os from subprocess import check_call, check_output, CalledProcessError, STDOUT, Popen import argparse import glob import shutil from time import localtime, strftime, gmtime, time from collections import defaultdict # Paths ScriptDir = os.path.dirname(os.path.abspath(__file__)) BaseDir = os.path.dirname(ScriptDir) ScriptName = os.path.basename(__file__) PkgScripts = '/pkgscripts-ng' sys.path.append(ScriptDir+'/include') sys.path.append(ScriptDir+'/include/python') import BuildEnv from chroot import Chroot from parallel import doPlatformParallel, doParallel from link_project import link_projects, link_scripts, LinkProjectError from tee import Tee import config_parser from project_visitor import UpdateHook, ProjectVisitor, UpdateFailedError, ConflictError from version_file import VersionFile log_file = os.path.join(BaseDir, 'pkgcreate.log') sys.stdout = Tee(sys.stdout, log_file) sys.stderr = Tee(sys.stderr, log_file, move=False) MinSDKVersion = "6.0" BasicProjects = set() class PkgCreateError(RuntimeError): pass class SignPackageError(PkgCreateError): pass class CollectPackageError(PkgCreateError): pass class LinkPackageError(PkgCreateError): pass class BuildPackageError(PkgCreateError): pass class InstallPacageError(PkgCreateError): pass class TraverseProjectError(PkgCreateError): pass def show_msg_block(msg, title=None, error=False): if not msg: return if error: tok_s = "#" tok_e = "#" else: tok_s = "=" tok_e = "-" if title: print("\n" + tok_s * 60) print("{:^60s}".format(title)) print(tok_e * 60) print("\n".join(msg)) print() def args_parser(argv): argparser = argparse.ArgumentParser() argparser.add_argument('-p', dest='platforms', help='Specify target platforms. Default to detect available platforms under build_env/') argparser.add_argument('-e', '--env', dest='env_section', default='default', help='Specify environment section in SynoBuildConf/depends. Default is [default].') argparser.add_argument('-v', '--version', dest='env_version', help='Specify target DSM version manually.') argparser.add_argument('-x', dest='dep_level', type=int, default=1, help='Build dependant level') argparser.add_argument('-b', dest='branch', default='master', help='Specify branch of package.') argparser.add_argument('-s', dest='suffix', default="", help='Specify suffix of folder of build environment (build_env/).') argparser.add_argument('-c', dest='collect', action='store_true', help='collect package.') argparser.add_argument('-U', dest='update', action='store_false', help='Not update projects.') argparser.add_argument('-L', dest='link', action='store_false', help='Not link projects.') argparser.add_argument('-B', dest='build', action='store_false', help='Not build projects.') argparser.add_argument('-I', dest='install', action='store_false', help='Not install projects.') argparser.add_argument('-i', dest='only_install', action='store_true', help='Only install projects.') argparser.add_argument('-S', dest="sign", action='store_false', help='Do not make code sign.') argparser.add_argument('--build-opt', default="", help='Argument pass to SynoBuild') argparser.add_argument('--install-opt', default="", help='Argument pass to SynoInstall') argparser.add_argument('--print-log', action='store_true', help='Print SynoBuild/SynoInstall error log.') argparser.add_argument('--min-sdk', dest='sdk_ver', default=MinSDKVersion, help='Min sdk version, default=6.0') argparser.add_argument('package', help='Target packages') args = argparser.parse_args(argv) if not args.build: args.link = False if args.only_install: args.update = args.link = args.build = False if args.platforms: args.platforms = args.platforms.split() msg = [] for key, value in vars(args).items(): if isinstance(value, list): value = " ".join(value) else: value = str(value) msg.append("{:13s}".format(key) + ": " + value) show_msg_block(msg, "Parse argument result") return args class WorkerFactory: def __init__(self, args): self.package = Package(args.package) self.env_config = EnvConfig(args.package, args.env_section, args.env_version, args.platforms, args.dep_level, args.branch, args.suffix) self.package.chroot = self.env_config.get_chroot() def new(self, worker_class, *args, **kwargs): return worker_class(self.package, self.env_config, *args, **kwargs) class Worker: def __init__(self, package, env_config): self.package = package self.env_config = env_config self.__time_log = None def execute(self, *argv): if not self._check_executable(): return init_time = time() if hasattr(self, 'title'): print("\n" + "=" * 60) print("{:^60s}".format('Start to run "%s"' % self.title)) print("-" * 60) self._process_output(self._run(*argv)) self.__time_log = strftime('%H:%M:%S', gmtime(time()-init_time)) def _run(self): pass def _check_executable(self): return True def _process_output(self, output): pass def get_time_cost(self): time_cost = [] if hasattr(self, 'title') and self.__time_log: time_cost.append("%s: %s" % (self.__time_log, self.title)) return time_cost class EnvPrepareWorker(Worker): def __init__(self, package, env_config, update): Worker.__init__(self, package, env_config) self.update = update self.sub_workers = [] def _run(self, *argv): depends_cache = None update_hook = None for version, platforms in self.env_config.toolkit_versions.items(): print("Processing [%s]: " % version + " ".join(platforms)) dsm_ver, build_num = version.split('-') if self.update: update_hook = UpdateHook(self.env_config.branch, build_num) for worker in self.sub_workers: worker.execute(version, update_hook, depends_cache) def add_subworker(self, sub_worker): self.sub_workers.append(sub_worker) def get_time_cost(self): time_cost = [] if self.sub_workers: for sub_worker in self.sub_workers: time_cost += sub_worker.get_time_cost() return time_cost class ProjectTraverser(Worker): title = "Traverse project" def _run(self, version, update_hook, cache): dep_level = self.env_config.dep_level platforms = self.env_config.toolkit_versions[version] try: visitor = ProjectVisitor(update_hook, dep_level, platforms, depends_cache=cache) dict_projects = visitor.traverse(self.package.name) visitor.checkout_git_refs() visitor.show_proj_info() except UpdateFailedError as e: log = os.path.join(BaseDir, 'logs', 'error.update') with open(log, 'r') as fd: print(fd.read()) raise TraverseProjectError("Error log: " + log) except ConflictError as e: raise TraverseProjectError(str(e)) self.package.dict_projects = dict_projects class ProjectLinker(Worker): title = "Link Project" def _run(self, version, *argv): tasks = [] for platform in self.env_config.toolkit_versions[version]: chroot = self.env_config.get_chroot(platform) if not os.path.isdir(os.path.join(chroot, 'source')): os.makedirs(os.path.join(chroot, 'source')) link_scripts(chroot) tasks.append((set(BasicProjects) | self.package.get_build_projects(platform) | self.package.ref_projs, chroot)) try: doParallel(link_projects, tasks) except LinkProjectError as e: raise LinkPackageError(str(e)) class CodeSignWorker(Worker): title = "Generate code sign" def _run(self): return doPlatformParallel(self._code_sign, self.env_config.platforms) def check_gpg_key_exist(self): try: gpg = check_output(['gpg', '--list-keys']).decode().strip() except CalledProcessError: return False return gpg def _code_sign(self, platform): spks = self.package.spk_config.chroot_spks(self.env_config.get_chroot(platform)) if not spks: raise SignPackageError('[%s] No spk found' % platform) for spk in spks: cmd = ' php ' + PkgScripts + '/CodeSign.php --sign=/image/packages/' + os.path.basename(spk) with Chroot(self.env_config.get_chroot(platform)): if not self.check_gpg_key_exist(): raise SignPackageError("[%s] Gpg key not exist. You can add `-S' to skip package code sign or import gpg key first." % platform) print("[%s] Sign package: " % platform + cmd) try: check_call(cmd, shell=True, executable="/bin/bash") except CalledProcessError: raise SignPackageError('Failed to create signature: ' + spk) class PackageCollecter(Worker): title = "Collect package" def _run(self): spks = defaultdict(list) dest_dir = self.package.spk_config.spk_result_dir(self.env_config.suffix) if os.path.exists(dest_dir): old_dir = dest_dir + '.bad.' + strftime('%Y%m%d-%H%M', localtime()) if os.path.isdir(old_dir): shutil.rmtree(old_dir) os.rename(dest_dir, old_dir) os.makedirs(dest_dir) for platform in self.env_config.platforms: for spk in self.package.spk_config.chroot_spks(self.env_config.get_chroot(platform)): spks[os.path.basename(spk)].append(spk) hook = self.package.collect if os.path.isfile(hook): print("Run hook " + hook) hook_env = { 'SPK_SRC_DIR': self.package.spk_config.chroot_packages_dir(self.env_config.get_chroot(platform)), 'SPK_DST_DIR': dest_dir, 'SPK_VERSION': self.package.spk_config.version, 'SPK_NAME': self.package.spk_config.name } pipe = Popen(hook, shell=True, stdout=None, stderr=None, env=hook_env) pipe.communicate() if pipe.returncode != 0: raise CollectPackageError("Execute package collect script failed.") for spk, source_list in spks.items(): print("%s -> %s" % (source_list[0], dest_dir)) try: shutil.copy(source_list[0], dest_dir) except: raise CollectPackageError("Collect package failed") if len(source_list) > 1: raise CollectPackageError("Found duplicate %s: \n%s" % (spk, "\n".join(source_list))) class CommandRunner(Worker): __log__ = None __error_msg__ = None __failed_exception__ = None def _rename_log(self, suffix='.old'): if os.path.isfile(self.log): os.rename(self.log, self.log + suffix) def _run(self, *argv): self._rename_log() cmd = self._wrap_cmd(self._get_command(*argv)) try: print(" ".join(cmd)) output = check_output(" ".join(cmd), stderr=STDOUT, shell=True, executable="/bin/bash").decode() self._post_hook() except CalledProcessError as e: output = e.output.decode() print(output) raise self.__failed_exception__(self.__error_msg__) return output def _get_command(self): raise PkgCreateError("Not implement") def _post_hook(self): pass def _process_output(self, output): pass def _wrap_cmd(self, cmd): return ["set -o pipefail;"] + cmd + ["2>&1", '|', 'tee', self.log] @property def log(self): return os.path.join(BaseDir, self.__log__) # Run SynoBuild/SynoInstall in chroot class ChrootRunner(CommandRunner): def __init__(self, package, env_config, print_log=False): CommandRunner.__init__(self, package, env_config) self.print_log = print_log self.__log__ = None def _process_output(self, output): msg = [] log_file = [] for platform, failed_projs in output.items(): if not failed_projs: continue if self.print_log: self._dump_log(platform) msg.append(self.__error_msg__ + ' [%s] : %s' % (platform, " ".join(failed_projs))) log_file.append("Error log: " + self.get_platform_log(platform)) if msg: show_msg_block(msg + log_file, title=self.__error_msg__, error=True) raise self.__failed_exception__(self.__error_msg__) def _dump_log(self, platform): log = self.get_platform_log(platform) with open(log, 'r') as fd: show_msg_block(fd.read().split("\n"), title=log, error=True) def get_platform_log(self, platform): return os.path.join(self.env_config.get_chroot(platform), self.log) def run_command(self, platform, *argv): cmd = self._wrap_cmd(self._get_command(platform, *argv)) with Chroot(self.env_config.get_chroot(platform)): self._rename_log() try: print("[%s] " % platform + " ".join(cmd)) with open(os.devnull, 'wb') as null: check_call(" ".join(cmd), stdout=null, shell=True, executable='/bin/bash') except CalledProcessError: failed_projs = self.__get_failed_projects() if not failed_projs: raise self.__failed_exception__("%s failed. \n Error log: %s" % (" ".join(cmd), self.get_platform_log(platform))) return failed_projs def __get_failed_projects(self): projects = [] with open(self.log, 'r') as fd: for line in fd: if 'Error(s) occurred on project' not in line: continue projects.append(line.split('"')[1]) return projects def _run(self): return doPlatformParallel(self.run_command, self.env_config.platforms) @property def log(self): raise PkgCreateError("Not implemented") class PackageBuilder(ChrootRunner): title = "Build Package" log = "logs.build" __error_msg__ = "Failed to build package." __failed_exception__ = BuildPackageError def __init__(self, package, env_config, sdk_ver, build_opt, *argv, **kwargs): ChrootRunner.__init__(self, package, env_config, *argv, **kwargs) self.build_opt = build_opt self.sdk_ver = sdk_ver def _get_command(self, platform): build_script = os.path.join(PkgScripts, 'SynoBuild') projects = self.package.get_build_projects(platform) build_cmd = ['env', 'PackageName=' + self.package.name, build_script, '--' + platform, '-c', '--min-sdk', self.sdk_ver] if self.build_opt: build_cmd.append(self.build_opt) return build_cmd + list(projects) class PackageInstaller(ChrootRunner): title = "Install Package" log = "logs.install" __error_msg__ = "Failed to install package." __failed_exception__ = InstallPacageError def __init__(self, package, env_config, install_opt, print_log): ChrootRunner.__init__(self, package, env_config, print_log) self.install_opt = list(install_opt) def _get_command(self, platform): cmd = ['env', 'PackageName=' + self.package.name, os.path.join(PkgScripts, 'SynoInstall')] if self.install_opt: cmd += self.install_opt return cmd + [self.package.name] class Package(): def __init__(self, package): self.name = package self.package_proj = BuildEnv.Project(self.name) self.dict_projects = dict() self.__additional_build = defaultdict(list) self.__spk_config = None self.__chroot = None def get_additional_build_projs(self, platform): if platform in self.__additional_build: return set(self.__additional_build[platform]) return set() def add_additional_build_projs(self, platform, projs): self.__additional_build[platform] += projs @property def ref_projs(self): return self.dict_projects['refs'] | self.dict_projects['refTags'] def get_build_projects(self, platform): return self.dict_projects['branches'] | self.get_additional_build_projs(platform) @property def spk_config(self): if not self.__spk_config: self.__spk_config = SpkConfig(self.name, self.info, self.settings) return self.__spk_config @property def info(self): return self.package_proj.info(self.chroot) @property def settings(self): return self.package_proj.settings(self.chroot) @property def collect(self): return self.package_proj.collect(self.chroot) @property def chroot(self): return self.__chroot @chroot.setter def chroot(self, chroot): self.__chroot = chroot class SpkConfig: def __init__(self, name, info, settings): self.info = config_parser.KeyValueParser(info) try: self.settings = config_parser.PackageSettingParser(settings).get_section(name) except config_parser.ConfigNotFoundError: self.settings = None @property def name(self): if self.settings and "pkg_name" in self.settings: return self.settings["pkg_name"][0] return self.info['package'] @property def version(self): return self.info['version'] @property def build_num(self): try: return self.version.split("-")[-1] except IndexError: return self.version @property def spk_pattern(self): return self.name + '*' + self.version + '*spk' def chroot_packages_dir(self, chroot): return os.path.join(chroot, 'image', 'packages') def chroot_spks(self, chroot): return glob.glob(os.path.join(self.chroot_packages_dir(chroot), self.spk_pattern)) def spk_result_dir(self, suffix=""): if suffix: suffix = '-' + suffix return os.path.join(BaseDir, 'result_spk' + suffix, self.name + '-' + self.version) class EnvConfig(): def __init__(self, package, env_section, version, platforms, dep_level, branch, suffix): self.dict_env = getBaseEnvironment(package, env_section, version) self.suffix = suffix self.env_section = env_section self.env_version = version self.dep_level = dep_level self.branch = branch self.platforms = set(self.__get_package_platforms(platforms)) self.toolkit_versions = self.__resolve_toolkit_versions() if not self.platforms: raise PkgCreateError("No platform found!") def __get_package_platforms(self, platforms): def __get_toolkit_available_platforms(version): toolkit_config = os.path.join(ScriptDir, 'include', 'toolkit.config') major, minor = version.split('.') pattern = '$AvailablePlatform_%s_%s' % (major, minor) return check_output('source %s && echo %s' % (toolkit_config, pattern), shell=True, executable='/bin/bash').decode().split() package_platforms = set() for platform in self.dict_env: if platform == 'all': package_platforms |= set(__get_toolkit_available_platforms(self.dict_env['all'])) else: package_platforms.add(platform) if platforms: package_platforms = set(package_platforms) & set(platforms) # remove platform if dir not exist return [platform for platform in package_platforms if os.path.isdir(self.get_chroot(platform))] def __get_version(self, platform): if platform in self.dict_env: version = self.dict_env[platform] elif 'all' in self.dict_env: version = self.dict_env['all'] else: raise PkgCreateError("Package version not found") return version def get_version_map_file(self, platform): return os.path.join(self.get_chroot(platform), 'version_map') def get_chroot(self, platform=None): if not platform: platform = list(self.platforms)[0] return BuildEnv.getChrootSynoBase(platform, self.__get_version(platform), self.suffix) def __resolve_toolkit_versions(self): def __get_toolkit_version(version_file): version = VersionFile(version_file) return "%s-%s" % (version.dsm_version, version.buildnumber) versions = defaultdict(list) for platform in self.platforms: toolkit_version = __get_toolkit_version(self.__get_version_file(platform)) versions[toolkit_version].append(platform) if len(versions) > 1: msg = [] for version, platforms in versions.items(): msg.append("[%s]: %s" % (version, " ".join(platforms))) show_msg_block(msg, title="[WARNING] Multiple toolkit version found", error=True) return versions def __get_version_file(self, platform): return os.path.join(self.get_chroot(platform), 'PkgVersion') def get_chroot_synodebug(self, platform): return os.path.join(self.get_chroot(platform), 'image', 'synodebug') @property def prebuild_projects(self): return BuildEnv.getList('PreBuildProjects') or [] class PackagePacker: def __init__(self): self.__workers = [] def add_worker(self, worker): self.__workers.append(worker) def pack_package(self): for worker in self.__workers: worker.execute() def show_time_cost(self): time_cost = [] for worker in self.__workers: time_cost += worker.get_time_cost() show_msg_block(time_cost, title="Time Cost Statistic") def getBaseEnvironment(proj, env, ver=None): dict_env = {} if ver: dict_env['all'] = ver return dict_env if not env: env = 'default' depends = config_parser.DependsParser(BuildEnv.Project(proj).depends_script) dict_env = depends.get_env_section(env) return dict_env def main(argv): args = args_parser(argv) packer = PackagePacker() worker_factory = WorkerFactory(args) new_worker = worker_factory.new prepare_worker = new_worker(EnvPrepareWorker, args.update) prepare_worker.add_subworker(new_worker(ProjectTraverser)) if args.link: prepare_worker.add_subworker(new_worker(ProjectLinker)) packer.add_worker(prepare_worker) if args.build: packer.add_worker(new_worker(PackageBuilder, args.sdk_ver, args.build_opt, args.print_log)) if args.install: packer.add_worker(new_worker(PackageInstaller, install_opt=[args.install_opt, '--with-debug'], print_log=args.print_log)) packer.add_worker(new_worker(PackageInstaller, install_opt=[args.install_opt], print_log=args.print_log)) if args.collect: if args.sign: packer.add_worker(new_worker(CodeSignWorker)) packer.add_worker(new_worker(PackageCollecter)) packer.pack_package() packer.show_time_cost() if __name__ == '__main__': ret = 0 try: main(sys.argv[1:]) print("[SUCCESS] " + " ".join(sys.argv) + " finished.") except PkgCreateError as e: ret = 1 print("\n\033[91m%s:\033[0m" % type(e).__name__) print(str(e)) print("\n[ERROR] " + " ".join(sys.argv) + " failed!") sys.exit(ret)