Files
pkgscripts-ng/PkgCreate.py
kevinfang 709c456afa Synology DSM6.0.2 toolkit framework
1. pythonize EnvDeploy and PkgCreate.py
2. EnvDeploy change sourceforge directory
2016-11-11 19:04:44 +08:00

731 lines
24 KiB
Python
Executable File

#!/usr/bin/python3
# Copyright (c) 2000-2014 Synology Inc. All rights reserved.
import sys
import os
from subprocess import check_call, check_output, CalledProcessError, STDOUT, Popen
import argparse
import glob
import shutil
from time import localtime, strftime, gmtime, time
from collections import defaultdict
# Paths
ScriptDir = os.path.dirname(os.path.abspath(__file__))
BaseDir = os.path.dirname(ScriptDir)
ScriptName = os.path.basename(__file__)
PkgScripts = '/pkgscripts-ng'
sys.path.append(ScriptDir+'/include')
sys.path.append(ScriptDir+'/include/python')
import BuildEnv
from chroot import Chroot
from parallel import doPlatformParallel, doParallel
from link_project import link_projects, link_scripts, LinkProjectError
from tee import Tee
import config_parser
from project_visitor import UpdateHook, ProjectVisitor, UpdateFailedError, ConflictError
from version_file import VersionFile
log_file = os.path.join(BaseDir, 'pkgcreate.log')
sys.stdout = Tee(sys.stdout, log_file)
sys.stderr = Tee(sys.stderr, log_file, move=False)
MinSDKVersion = "6.0"
BasicProjects = set()
class PkgCreateError(RuntimeError):
pass
class SignPackageError(PkgCreateError):
pass
class CollectPackageError(PkgCreateError):
pass
class LinkPackageError(PkgCreateError):
pass
class BuildPackageError(PkgCreateError):
pass
class InstallPacageError(PkgCreateError):
pass
class TraverseProjectError(PkgCreateError):
pass
def show_msg_block(msg, title=None, error=False):
if not msg:
return
if error:
tok_s = "#"
tok_e = "#"
else:
tok_s = "="
tok_e = "-"
if title:
print("\n" + tok_s * 60)
print("{:^60s}".format(title))
print(tok_e * 60)
print("\n".join(msg))
print()
def args_parser(argv):
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', dest='platforms',
help='Specify target platforms. Default to detect available platforms under build_env/')
argparser.add_argument('-e', '--env', dest='env_section', default='default',
help='Specify environment section in SynoBuildConf/depends. Default is [default].')
argparser.add_argument('-v', '--version', dest='env_version', help='Specify target DSM version manually.')
argparser.add_argument('-x', dest='dep_level', type=int, default=1, help='Build dependant level')
argparser.add_argument('-b', dest='branch', default='master', help='Specify branch of package.')
argparser.add_argument('-s', dest='suffix', default="",
help='Specify suffix of folder of build environment (build_env/).')
argparser.add_argument('-c', dest='collect', action='store_true', help='collect package.')
argparser.add_argument('-U', dest='update', action='store_false', help='Not update projects.')
argparser.add_argument('-L', dest='link', action='store_false', help='Not link projects.')
argparser.add_argument('-B', dest='build', action='store_false', help='Not build projects.')
argparser.add_argument('-I', dest='install', action='store_false', help='Not install projects.')
argparser.add_argument('-i', dest='only_install', action='store_true', help='Only install projects.')
argparser.add_argument('-S', dest="sign", action='store_false', help='Do not make code sign.')
argparser.add_argument('--build-opt', default="", help='Argument pass to SynoBuild')
argparser.add_argument('--install-opt', default="", help='Argument pass to SynoInstall')
argparser.add_argument('--print-log', action='store_true', help='Print SynoBuild/SynoInstall error log.')
argparser.add_argument('--min-sdk', dest='sdk_ver', default=MinSDKVersion, help='Min sdk version, default=6.0')
argparser.add_argument('package', help='Target packages')
args = argparser.parse_args(argv)
if not args.build:
args.link = False
if args.only_install:
args.update = args.link = args.build = False
if args.platforms:
args.platforms = args.platforms.split()
msg = []
for key, value in vars(args).items():
if isinstance(value, list):
value = " ".join(value)
else:
value = str(value)
msg.append("{:13s}".format(key) + ": " + value)
show_msg_block(msg, "Parse argument result")
return args
class WorkerFactory:
def __init__(self, args):
self.package = Package(args.package)
self.env_config = EnvConfig(args.package, args.env_section, args.env_version, args.platforms, args.dep_level,
args.branch, args.suffix)
self.package.chroot = self.env_config.get_chroot()
def new(self, worker_class, *args, **kwargs):
return worker_class(self.package, self.env_config, *args, **kwargs)
class Worker:
def __init__(self, package, env_config):
self.package = package
self.env_config = env_config
self.__time_log = None
def execute(self, *argv):
if not self._check_executable():
return
init_time = time()
if hasattr(self, 'title'):
print("\n" + "=" * 60)
print("{:^60s}".format('Start to run "%s"' % self.title))
print("-" * 60)
self._process_output(self._run(*argv))
self.__time_log = strftime('%H:%M:%S', gmtime(time()-init_time))
def _run(self):
pass
def _check_executable(self):
return True
def _process_output(self, output):
pass
def get_time_cost(self):
time_cost = []
if hasattr(self, 'title') and self.__time_log:
time_cost.append("%s: %s" % (self.__time_log, self.title))
return time_cost
class EnvPrepareWorker(Worker):
def __init__(self, package, env_config, update):
Worker.__init__(self, package, env_config)
self.update = update
self.sub_workers = []
def _run(self, *argv):
depends_cache = None
update_hook = None
for version, platforms in self.env_config.toolkit_versions.items():
print("Processing [%s]: " % version + " ".join(platforms))
dsm_ver, build_num = version.split('-')
if self.update:
update_hook = UpdateHook(self.env_config.branch, build_num)
for worker in self.sub_workers:
worker.execute(version, update_hook, depends_cache)
def add_subworker(self, sub_worker):
self.sub_workers.append(sub_worker)
def get_time_cost(self):
time_cost = []
if self.sub_workers:
for sub_worker in self.sub_workers:
time_cost += sub_worker.get_time_cost()
return time_cost
class ProjectTraverser(Worker):
title = "Traverse project"
def _run(self, version, update_hook, cache):
dep_level = self.env_config.dep_level
platforms = self.env_config.toolkit_versions[version]
try:
visitor = ProjectVisitor(update_hook, dep_level, platforms, depends_cache=cache)
dict_projects = visitor.traverse(self.package.name)
visitor.checkout_git_refs()
visitor.show_proj_info()
except UpdateFailedError as e:
log = os.path.join(BaseDir, 'logs', 'error.update')
with open(log, 'r') as fd:
print(fd.read())
raise TraverseProjectError("Error log: " + log)
except ConflictError as e:
raise TraverseProjectError(str(e))
self.package.dict_projects = dict_projects
class ProjectLinker(Worker):
title = "Link Project"
def _run(self, version, *argv):
tasks = []
for platform in self.env_config.toolkit_versions[version]:
chroot = self.env_config.get_chroot(platform)
if not os.path.isdir(os.path.join(chroot, 'source')):
os.makedirs(os.path.join(chroot, 'source'))
link_scripts(chroot)
tasks.append((set(BasicProjects) |
self.package.get_build_projects(platform) |
self.package.ref_projs, chroot))
try:
doParallel(link_projects, tasks)
except LinkProjectError as e:
raise LinkPackageError(str(e))
class CodeSignWorker(Worker):
title = "Generate code sign"
def _run(self):
return doPlatformParallel(self._code_sign, self.env_config.platforms)
def check_gpg_key_exist(self):
try:
gpg = check_output(['gpg', '--list-keys']).decode().strip()
except CalledProcessError:
return False
return gpg
def _code_sign(self, platform):
spks = self.package.spk_config.chroot_spks(self.env_config.get_chroot(platform))
if not spks:
raise SignPackageError('[%s] No spk found' % platform)
for spk in spks:
cmd = ' php ' + PkgScripts + '/CodeSign.php --sign=/image/packages/' + os.path.basename(spk)
with Chroot(self.env_config.get_chroot(platform)):
if not self.check_gpg_key_exist():
raise SignPackageError("[%s] Gpg key not exist. You can add `-S' to skip package code sign or import gpg key first." % platform)
print("[%s] Sign package: " % platform + cmd)
try:
check_call(cmd, shell=True, executable="/bin/bash")
except CalledProcessError:
raise SignPackageError('Failed to create signature: ' + spk)
class PackageCollecter(Worker):
title = "Collect package"
def _run(self):
spks = defaultdict(list)
dest_dir = self.package.spk_config.spk_result_dir(self.env_config.suffix)
if os.path.exists(dest_dir):
old_dir = dest_dir + '.bad.' + strftime('%Y%m%d-%H%M', localtime())
if os.path.isdir(old_dir):
shutil.rmtree(old_dir)
os.rename(dest_dir, old_dir)
os.makedirs(dest_dir)
for platform in self.env_config.platforms:
for spk in self.package.spk_config.chroot_spks(self.env_config.get_chroot(platform)):
spks[os.path.basename(spk)].append(spk)
hook = self.package.collect
if os.path.isfile(hook):
print("Run hook " + hook)
hook_env = {
'SPK_SRC_DIR': self.package.spk_config.chroot_packages_dir(self.env_config.get_chroot(platform)),
'SPK_DST_DIR': dest_dir,
'SPK_VERSION': self.package.spk_config.version,
'SPK_NAME': self.package.spk_config.name
}
pipe = Popen(hook, shell=True, stdout=None, stderr=None, env=hook_env)
pipe.communicate()
if pipe.returncode != 0:
raise CollectPackageError("Execute package collect script failed.")
for spk, source_list in spks.items():
print("%s -> %s" % (source_list[0], dest_dir))
try:
shutil.copy(source_list[0], dest_dir)
except:
raise CollectPackageError("Collect package failed")
if len(source_list) > 1:
raise CollectPackageError("Found duplicate %s: \n%s" % (spk, "\n".join(source_list)))
class CommandRunner(Worker):
__log__ = None
__error_msg__ = None
__failed_exception__ = None
def _rename_log(self, suffix='.old'):
if os.path.isfile(self.log):
os.rename(self.log, self.log + suffix)
def _run(self, *argv):
self._rename_log()
cmd = self._wrap_cmd(self._get_command(*argv))
try:
print(" ".join(cmd))
output = check_output(" ".join(cmd), stderr=STDOUT, shell=True, executable="/bin/bash").decode()
self._post_hook()
except CalledProcessError as e:
output = e.output.decode()
print(output)
raise self.__failed_exception__(self.__error_msg__)
return output
def _get_command(self):
raise PkgCreateError("Not implement")
def _post_hook(self):
pass
def _process_output(self, output):
pass
def _wrap_cmd(self, cmd):
return ["set -o pipefail;"] + cmd + ["2>&1", '|', 'tee', self.log]
@property
def log(self):
return os.path.join(BaseDir, self.__log__)
# Run SynoBuild/SynoInstall in chroot
class ChrootRunner(CommandRunner):
def __init__(self, package, env_config, print_log=False):
CommandRunner.__init__(self, package, env_config)
self.print_log = print_log
self.__log__ = None
def _process_output(self, output):
msg = []
log_file = []
for platform, failed_projs in output.items():
if not failed_projs:
continue
if self.print_log:
self._dump_log(platform)
msg.append(self.__error_msg__ + ' [%s] : %s' % (platform, " ".join(failed_projs)))
log_file.append("Error log: " + self.get_platform_log(platform))
if msg:
show_msg_block(msg + log_file, title=self.__error_msg__, error=True)
raise self.__failed_exception__(self.__error_msg__)
def _dump_log(self, platform):
log = self.get_platform_log(platform)
with open(log, 'r') as fd:
show_msg_block(fd.read().split("\n"), title=log, error=True)
def get_platform_log(self, platform):
return os.path.join(self.env_config.get_chroot(platform), self.log)
def run_command(self, platform, *argv):
cmd = self._wrap_cmd(self._get_command(platform, *argv))
with Chroot(self.env_config.get_chroot(platform)):
self._rename_log()
try:
print("[%s] " % platform + " ".join(cmd))
with open(os.devnull, 'wb') as null:
check_call(" ".join(cmd), stdout=null, shell=True, executable='/bin/bash')
except CalledProcessError:
failed_projs = self.__get_failed_projects()
if not failed_projs:
raise self.__failed_exception__("%s failed. \n Error log: %s"
% (" ".join(cmd), self.get_platform_log(platform)))
return failed_projs
def __get_failed_projects(self):
projects = []
with open(self.log, 'r') as fd:
for line in fd:
if 'Error(s) occurred on project' not in line:
continue
projects.append(line.split('"')[1])
return projects
def _run(self):
return doPlatformParallel(self.run_command, self.env_config.platforms)
@property
def log(self):
raise PkgCreateError("Not implemented")
class PackageBuilder(ChrootRunner):
title = "Build Package"
log = "logs.build"
__error_msg__ = "Failed to build package."
__failed_exception__ = BuildPackageError
def __init__(self, package, env_config, sdk_ver, build_opt, *argv, **kwargs):
ChrootRunner.__init__(self, package, env_config, *argv, **kwargs)
self.build_opt = build_opt
self.sdk_ver = sdk_ver
def _get_command(self, platform):
build_script = os.path.join(PkgScripts, 'SynoBuild')
projects = self.package.get_build_projects(platform)
build_cmd = ['env', 'PackageName=' + self.package.name, build_script, '--' + platform,
'-c', '--min-sdk', self.sdk_ver]
if self.build_opt:
build_cmd.append(self.build_opt)
return build_cmd + list(projects)
class PackageInstaller(ChrootRunner):
title = "Install Package"
log = "logs.install"
__error_msg__ = "Failed to install package."
__failed_exception__ = InstallPacageError
def __init__(self, package, env_config, install_opt, print_log):
ChrootRunner.__init__(self, package, env_config, print_log)
self.install_opt = list(install_opt)
def _get_command(self, platform):
cmd = ['env', 'PackageName=' + self.package.name, os.path.join(PkgScripts, 'SynoInstall')]
if self.install_opt:
cmd += self.install_opt
return cmd + [self.package.name]
class Package():
def __init__(self, package):
self.name = package
self.package_proj = BuildEnv.Project(self.name)
self.dict_projects = dict()
self.__additional_build = defaultdict(list)
self.__spk_config = None
self.__chroot = None
def get_additional_build_projs(self, platform):
if platform in self.__additional_build:
return set(self.__additional_build[platform])
return set()
def add_additional_build_projs(self, platform, projs):
self.__additional_build[platform] += projs
@property
def ref_projs(self):
return self.dict_projects['refs'] | self.dict_projects['refTags']
def get_build_projects(self, platform):
return self.dict_projects['branches'] | self.get_additional_build_projs(platform)
@property
def spk_config(self):
if not self.__spk_config:
self.__spk_config = SpkConfig(self.name, self.info, self.settings)
return self.__spk_config
@property
def info(self):
return self.package_proj.info(self.chroot)
@property
def settings(self):
return self.package_proj.settings(self.chroot)
@property
def collect(self):
return self.package_proj.collect(self.chroot)
@property
def chroot(self):
return self.__chroot
@chroot.setter
def chroot(self, chroot):
self.__chroot = chroot
class SpkConfig:
def __init__(self, name, info, settings):
self.info = config_parser.KeyValueParser(info)
try:
self.settings = config_parser.PackageSettingParser(settings).get_section(name)
except config_parser.ConfigNotFoundError:
self.settings = None
@property
def name(self):
if self.settings and "pkg_name" in self.settings:
return self.settings["pkg_name"][0]
return self.info['package']
@property
def version(self):
return self.info['version']
@property
def build_num(self):
try:
return self.version.split("-")[-1]
except IndexError:
return self.version
@property
def spk_pattern(self):
return self.name + '*' + self.version + '*spk'
def chroot_packages_dir(self, chroot):
return os.path.join(chroot, 'image', 'packages')
def chroot_spks(self, chroot):
return glob.glob(os.path.join(self.chroot_packages_dir(chroot), self.spk_pattern))
def spk_result_dir(self, suffix=""):
if suffix:
suffix = '-' + suffix
return os.path.join(BaseDir, 'result_spk' + suffix, self.name + '-' + self.version)
class EnvConfig():
def __init__(self, package, env_section, version, platforms, dep_level, branch, suffix):
self.dict_env = getBaseEnvironment(package, env_section, version)
self.suffix = suffix
self.env_section = env_section
self.env_version = version
self.dep_level = dep_level
self.branch = branch
self.platforms = set(self.__get_package_platforms(platforms))
self.toolkit_versions = self.__resolve_toolkit_versions()
if not self.platforms:
raise PkgCreateError("No platform found!")
def __get_package_platforms(self, platforms):
def __get_toolkit_available_platforms(version):
toolkit_config = os.path.join(ScriptDir, 'include', 'toolkit.config')
major, minor = version.split('.')
pattern = '$AvailablePlatform_%s_%s' % (major, minor)
return check_output('source %s && echo %s' % (toolkit_config, pattern),
shell=True, executable='/bin/bash').decode().split()
package_platforms = set()
for platform in self.dict_env:
if platform == 'all':
package_platforms |= set(__get_toolkit_available_platforms(self.dict_env['all']))
else:
package_platforms.add(platform)
if platforms:
package_platforms = set(package_platforms) & set(platforms)
# remove platform if dir not exist
return [platform for platform in package_platforms if os.path.isdir(self.get_chroot(platform))]
def __get_version(self, platform):
if platform in self.dict_env:
version = self.dict_env[platform]
elif 'all' in self.dict_env:
version = self.dict_env['all']
else:
raise PkgCreateError("Package version not found")
return version
def get_version_map_file(self, platform):
return os.path.join(self.get_chroot(platform), 'version_map')
def get_chroot(self, platform=None):
if not platform:
platform = list(self.platforms)[0]
return BuildEnv.getChrootSynoBase(platform, self.__get_version(platform), self.suffix)
def __resolve_toolkit_versions(self):
def __get_toolkit_version(version_file):
version = VersionFile(version_file)
return "%s-%s" % (version.dsm_version, version.buildnumber)
versions = defaultdict(list)
for platform in self.platforms:
toolkit_version = __get_toolkit_version(self.__get_version_file(platform))
versions[toolkit_version].append(platform)
if len(versions) > 1:
msg = []
for version, platforms in versions.items():
msg.append("[%s]: %s" % (version, " ".join(platforms)))
show_msg_block(msg, title="[WARNING] Multiple toolkit version found", error=True)
return versions
def __get_version_file(self, platform):
return os.path.join(self.get_chroot(platform), 'PkgVersion')
def get_chroot_synodebug(self, platform):
return os.path.join(self.get_chroot(platform), 'image', 'synodebug')
@property
def prebuild_projects(self):
return BuildEnv.getList('PreBuildProjects') or []
class PackagePacker:
def __init__(self):
self.__workers = []
def add_worker(self, worker):
self.__workers.append(worker)
def pack_package(self):
for worker in self.__workers:
worker.execute()
def show_time_cost(self):
time_cost = []
for worker in self.__workers:
time_cost += worker.get_time_cost()
show_msg_block(time_cost, title="Time Cost Statistic")
def getBaseEnvironment(proj, env, ver=None):
dict_env = {}
if ver:
dict_env['all'] = ver
return dict_env
if not env:
env = 'default'
depends = config_parser.DependsParser(BuildEnv.Project(proj).depends_script)
dict_env = depends.get_env_section(env)
return dict_env
def main(argv):
args = args_parser(argv)
packer = PackagePacker()
worker_factory = WorkerFactory(args)
new_worker = worker_factory.new
prepare_worker = new_worker(EnvPrepareWorker, args.update)
prepare_worker.add_subworker(new_worker(ProjectTraverser))
if args.link:
prepare_worker.add_subworker(new_worker(ProjectLinker))
packer.add_worker(prepare_worker)
if args.build:
packer.add_worker(new_worker(PackageBuilder, args.sdk_ver, args.build_opt, args.print_log))
if args.install:
packer.add_worker(new_worker(PackageInstaller,
install_opt=[args.install_opt, '--with-debug'],
print_log=args.print_log))
packer.add_worker(new_worker(PackageInstaller,
install_opt=[args.install_opt],
print_log=args.print_log))
if args.collect:
if args.sign:
packer.add_worker(new_worker(CodeSignWorker))
packer.add_worker(new_worker(PackageCollecter))
packer.pack_package()
packer.show_time_cost()
if __name__ == '__main__':
ret = 0
try:
main(sys.argv[1:])
print("[SUCCESS] " + " ".join(sys.argv) + " finished.")
except PkgCreateError as e:
ret = 1
print("\n\033[91m%s:\033[0m" % type(e).__name__)
print(str(e))
print("\n[ERROR] " + " ".join(sys.argv) + " failed!")
sys.exit(ret)