Release for DSM7.0

include/python/pkgdeploy.py

@@ -0,0 +1,319 @@
+#! /usr/bin/env python3
+# Copyright (c) 2000-2020 Synology Inc. All rights reserved.
+
+import os
+import subprocess
+import logging
+import urllib.request
+import sys
+import shutil
+import json
+import hashlib
+from glob import glob
+from cache import cache
+import BuildEnv
+from toolkit import TarballManager
+from exec_env import ChrootEnv, EnvError
+from parallel import doPlatformParallel, doParallel
+from utils import move_old
+from version_file import VersionFile
+VersionMap = 'version_map'
+DownloadDir = os.path.join(BuildEnv.SynoBase, 'toolkit_tarballs')
+ToolkitServer = 'https://sourceforge.net/projects/dsgpl/files/toolkit'
+
+
+class EnvDeployError(RuntimeError):
+    pass
+
+
+class TarballNotFoundError(EnvDeployError):
+    pass
+
+
+class PlatformNotAvailableError(EnvDeployError):
+    pass
+
+
+class DownloadToolkitError(EnvDeployError):
+    pass
+
+
+class EnvHookError(EnvDeployError):
+    pass
+
+
+def set_log(log_name):
+    log = os.path.join(BuildEnv.SynoBase, 'envdeploy.log')
+    move_old(log)
+    logfmt = '[%(asctime)s] %(levelname)s: %(message)s'
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format=logfmt,
+        filename=log
+    )
+
+    console = logging.StreamHandler()
+    console.setLevel(logging.INFO)
+    formatter = logging.Formatter(logfmt)
+    console.setFormatter(formatter)
+    logging.getLogger().addHandler(console)
+
+
+def link_project(projects, platforms, version):
+    if type(projects) is str:
+        projects = [projects]
+
+    for proj in projects:
+        for platform in platforms:
+            BuildEnv.LinkProject(proj, platform, version)
+
+
+@cache
+def split_version(version):
+    if '-' in version:
+        return version.split('-')
+    else:
+        return version, None
+
+
+class ToolkitDownloader:
+    def __init__(self, version, tarball_manager):
+        self.dsm_ver, self.build_num = split_version(version)
+        self.tarball_manager = tarball_manager
+
+        if not os.path.isdir(DownloadDir):
+            os.makedirs(DownloadDir)
+
+    def download_base_tarball(self, quiet):
+        self._download(
+            self._join_download_url(self.tarball_manager.base_tarball_name),
+            quiet
+        )
+
+    def download_platform_tarball(self, platform, quiet):
+        self._download(self._join_download_url(
+            self.tarball_manager.get_env_tarball_name(platform)), quiet)
+        self._download(self._join_download_url(
+            self.tarball_manager.get_dev_tarball_name(platform)), quiet)
+
+    def _join_download_url(self, *patterns):
+        url = ToolkitServer
+        for pattern in ['DSM' + self.dsm_ver, self.build_num] + list(patterns):
+            if not pattern:
+                continue
+            url += '/%s' % pattern
+        return url
+
+    def _download(self, url, quiet):
+        logging.info("Download... " + url)
+        if quiet or not sys.stdout.isatty():
+            reporthook = None
+        else:
+            reporthook = self.dl_progress
+
+        try:
+            urllib.request.urlretrieve(url, os.path.join(
+                DownloadDir, url.split("/")[-1]), reporthook=reporthook)
+        except urllib.error.HTTPError as e:
+            raise DownloadToolkitError("Failed to download toolkit: " + url + ", reason: " + str(e))
+
+    def dl_progress(self, count, dl_size, total_size):
+        percent = int(count * dl_size * 50 / total_size)
+        sys.stdout.write("[%-50s] %d%%" %
+                         ('=' * (percent - 1) + ">", 2 * percent))
+        sys.stdout.write("\b" * 102)
+        sys.stdout.flush()
+
+
+class ToolkitEnv(object):
+    def __init__(self, version, platforms, suffix=""):
+        self.version = version
+        self.platforms = platforms
+        self.dsm_ver, self.build_num = split_version(version)
+        self.suffix = suffix
+
+    def download(self, quiet=False):
+        raise NotImplementedError("download() not implemented")
+
+    def deploy(self):
+        raise NotImplementedError("deploy() not implemented")
+
+    def clean(self):
+        raise NotImplementedError("clean() not implemented")
+
+    def get_chroot(self, platform):
+        return BuildEnv.getChrootSynoBase(platform, self.dsm_ver, self.suffix)
+
+    def __remove_chroot(self, chroot):
+        for f in os.listdir(chroot):
+            if 'ccaches' in f:
+                continue
+            file_path = os.path.join(chroot, f)
+            subprocess.check_call(['rm', '-rf', file_path])
+
+    def __umount_proc(self, chroot):
+        proc = os.path.join(chroot, 'proc')
+        if os.path.ismount(proc):
+            subprocess.check_call(['umount', proc])
+
+    # clear and mkdir chroot
+    def clear_chroot(self, platform):
+        chroot = self.get_chroot(platform)
+        if not os.path.isdir(chroot):
+            return
+
+        logging.info("Clear %s..." % chroot)
+        self.__umount_proc(chroot)
+        self.__remove_chroot(chroot)
+
+    def link_pkgscripts(self, env):
+        if not env.islink('pkgscripts'):
+            env.link('pkgscripts-ng', 'pkgscripts')
+
+    def get_sysroot_include(self, platform):
+        all_sysroot = list()
+        for arch in [32, 64]:
+            variable = "ToolChainInclude" + str(arch)
+            sysroot = BuildEnv.getPlatformVariable(platform, variable)
+            if sysroot:
+                all_sysroot.append(sysroot)
+        return all_sysroot
+
+    def get_env_build_num(self, platform):
+        raise NotImplementedError("get_env_build_num() not implemented")
+
+    def create_chroot(self, platform):
+        os.makedirs(self.get_chroot(platform), exist_ok=True)
+
+
+class ChrootToolkit(ToolkitEnv):
+    def __init__(self, version, platforms, suffix="", tarball_root=DownloadDir):
+        super().__init__(version, platforms, suffix)
+        self.tarball_manager = TarballManager(self.dsm_ver, tarball_root)
+        self.__downloader = ToolkitDownloader(
+            self.version, self.tarball_manager)
+
+    def download(self, quiet):
+        self.__downloader.download_base_tarball(quiet)
+        for platform in self.platforms:
+            self.__downloader.download_platform_tarball(platform, quiet)
+
+    def deploy(self):
+        envs = {}
+        self.__check_tarball_exists()
+        for platform in self.platforms:
+            self.create_chroot(platform)
+            self.deploy_base_env(platform)
+            self.deploy_env(platform)
+            self.deploy_dev(platform)
+            self.adjust_chroot(platform)
+
+            envs[platform] = ChrootEnv(self.get_chroot(platform))
+            self.link_pkgscripts(envs[platform])
+        return envs
+
+    def clean(self):
+        doPlatformParallel(self.clear_chroot, self.platforms)
+
+    def __check_tarball_exists(self):
+        files = [self.tarball_manager.base_tarball_path]
+        for platform in self.platforms:
+            files.append(self.tarball_manager.get_dev_tarball_path(platform))
+            files.append(self.tarball_manager.get_env_tarball_path(platform))
+
+        for f in files:
+            if not os.path.isfile(f):
+                raise TarballNotFoundError("Needed file not found! " + f)
+
+    @property
+    def has_pixz(self):
+        try:
+            with open(os.devnull, 'wb') as null:
+                subprocess.check_call(
+                    ['which', 'pixz'], stdout=null, stderr=null)
+        except subprocess.CalledProcessError:
+            return False
+        return True
+
+    def __extract__(self, tarball, dest_dir):
+        cmd = ['tar']
+        if self.has_pixz:
+            cmd.append('-Ipixz')
+        cmd += ['-xhf', tarball, '-C', dest_dir]
+        logging.info(" ".join(cmd))
+        subprocess.check_call(cmd)
+
+    def deploy_base_env(self, platform):
+        self.__extract__(
+            self.tarball_manager.base_tarball_path,
+            self.get_chroot(platform)
+        )
+
+    def deploy_env(self, platform):
+        self.__extract__(
+            self.tarball_manager.get_env_tarball_path(platform),
+            self.get_chroot(platform)
+        )
+
+    def deploy_dev(self, platform):
+        self.__extract__(
+            self.tarball_manager.get_dev_tarball_path(platform),
+            self.get_chroot(platform)
+        )
+
+    def mkdir_source(self, chroot):
+        source_dir = os.path.join(chroot, 'source')
+        if not os.path.isdir(source_dir):
+            os.makedirs(source_dir)
+
+    def copy_user_env_config(self, chroot):
+        configs = ['/etc/hosts', '/root/.gitconfig',
+                   '/root/.ssh', '/etc/resolv.conf']
+
+        for config in configs:
+            dest = chroot + config
+            if os.path.isdir(config):
+                if os.path.isdir(dest):
+                    shutil.rmtree(dest)
+                shutil.copytree(config, dest)
+            elif os.path.isfile(config):
+                shutil.copy(config, dest)
+
+    def adjust_chroot(self, platform):
+        chroot = self.get_chroot(platform)
+        self.mkdir_source(chroot)
+        self.copy_user_env_config(chroot)
+
+    def get_env_build_num(self, platform):
+        version = VersionFile(os.path.join(
+            self.get_chroot(platform), 'PkgVersion'))
+        return version.buildnumber
+
+def get_all_platforms(dsm_ver, build_num):
+    pattern = 'AvailablePlatform_%s_%s' % (
+        dsm_ver.split('.')[0], dsm_ver.split('.')[1])
+
+    if build_num:
+        for line in config:
+            if pattern in line:
+                platforms = line.split('=')[1].strip('"').split()
+    else:
+        platforms = BuildEnv.getIncludeVariable(
+            'toolkit.config', pattern).split()
+
+    return platforms
+
+
+def filter_platforms(version, platforms):
+    dsm_ver, build_num = split_version(version)
+    all_platforms = get_all_platforms(dsm_ver, build_num)
+
+    if not platforms:
+        return all_platforms
+
+    redundant_platforms = set(platforms) - set(all_platforms)
+    if redundant_platforms:
+        raise PlatformNotAvailableError(
+            "[%s] is not available platform." % " ".join(redundant_platforms))
+
+    return platforms