Files
pkgscripts-ng/ProjectDepends.py
2020-09-18 09:05:10 +00:00

423 lines
13 KiB
Python
Executable File

#!/usr/bin/python3
# Copyright (c) 2000-2020 Synology Inc. All rights reserved.
import sys
import string
import os
import glob
import argparse
import pickle
import pprint
from collections import defaultdict
import re
from include import pythonutils
from include.python import BuildEnv
from config_parser import DependsParser, ProjectDependsParser
ScriptDir = os.path.dirname(os.path.abspath(sys.argv[0]))
# XXX: unified the variable with pythonutils.py
# config file, to get basic project parameters
section_depends = "project dependency"
section_variables = "variables"
config_path = ScriptDir + "/include/project.depends"
libc_project = "uclibc0929"
# Target sections in SynoBuildConf for packages
target_sections = ["BuildDependent", "BuildDependent-Tag"]
target_sections64 = ["BuildDependent64", "BuildDependent-Tag64"]
dynamic_variable = "dynamic variable list"
class DependencyError(Exception):
def __init__(self, stack, proj):
self.stack = stack
self.project = proj
print("Error! Circluar dependency found!!")
def dumpCircluarDepList(self):
blFound = False
for proj in self.stack:
if proj == self.project:
blFound = True
if blFound:
print(proj + " -> ")
print(self.project)
class ProjectNode:
def __init__(self, proj):
# FIXME name
self.proj = proj
self.depends = []
self.rev_depends = []
self.levels = defaultdict(set)
self.addProject(1, [self.proj])
self.success = False
self.visited = False
def updateDependLevel(self, child_node):
for level, projs in child_node.levels.items():
self.addProject(level + 1, projs)
def addProject(self, level, projs):
self.levels[level].update(projs)
def addDependProj(self, child_node):
self.depends.append(child_node)
child_node.rev_depends.append(self)
self.updateDependLevel(child_node)
def getProjectOrder(self):
orders = []
for level, projs in sorted(self.levels.items(), key=lambda x: x[0], reverse=True):
for proj in projs:
if proj not in orders:
orders.append(proj)
return orders
def update_status(self, success):
self.visited = True
self.success = success
def getProjectDepends(self, level):
order = self.getProjectOrder()
if level == 0:
return order
else:
return sorted(self.getLevelsProjects(1, level), key=lambda x: order.index(x))
def getLevelsProjects(self, low, high):
output = set()
for i in range(low, high + 1):
output.update(self.levels[i])
return output
def __repr__(self):
return self.proj
class DepGraph:
# all these attributes are not necessary, just for prototype reference
def __init__(self, dictDepends, direct):
self.root = ProjectNode('DependRoot')
self.root.update_status(True)
self.direct = direct
self.stack = []
self.dictDepends = dictDepends
self.reverseDep = defaultdict(list)
self.created_projects = {}
def getProjectNode(self, proj):
if proj not in self.created_projects:
raise RuntimeError("%s not created." % proj)
return self.created_projects[proj]
def createDepGraph(self, head, listProjs):
for proj in listProjs:
if proj in self.stack:
raise DependencyError(self.stack, proj)
self.stack.append(proj)
if proj in self.created_projects:
proj_node = self.created_projects[proj]
else:
proj_node = self.created_projects[proj] = ProjectNode(proj)
depProj = self.getDepProjList(proj)
if len(depProj) > 0:
self.createDepGraph(proj_node, depProj)
head.addDependProj(proj_node)
self.stack.pop()
def dumpGraph(self, head):
print(head.proj)
pprint.pprint(head.levels)
for p in head.depends:
self.dumpGraph(p)
def traverseDepends(self, listProjs, level):
try:
self.createDepGraph(self.root, listProjs)
# self.dumpGraph(self.root)
except DependencyError as e:
e.dumpCircluarDepList()
sys.exit(1)
output = []
t_level = level if level == 0 else level + 1
output = self.root.getProjectDepends(t_level)
output.remove(self.root.proj)
if self.direct == 'backwardDependency':
output.reverse()
return output
def getReverseDep(self, proj):
reverseList = []
if proj in self.reverseDep:
return self.reverseDep[proj]
for key in self.dictDepends.keys():
if proj in self.dictDepends[key]:
reverseList.append(key)
self.reverseDep[proj] = reverseList
return reverseList
def getTraverseDep(self, proj):
if proj in self.dictDepends:
return self.dictDepends[proj]
return []
def getDepProjList(self, proj):
# -r : Expand project dependency list reversely.
if self.direct == 'backwardDependency':
return self.getReverseDep(proj)
# -x: Traverse all dependant projects
return self.getTraverseDep(proj)
class DepNode:
def __init__(self, proj):
self.nodeName = proj
self.projDependOn = []
self.projDependedBy = []
def dumpNode(self):
print('Project depened on : ' + self.projDependOn)
print('Project depended by : ' + self.projDependedBy)
def dumpProjectDepends(project, dict):
if project in dict:
print()
print("check this line in file '" + config_path + "'")
print(project + ": " + string.join(dict[project]))
# replace project.depends Variable section in project dependency section
# i.e. ${KernelPacks} to synobios
def replaceVariableSection(config, dictDepends):
for var, value in config.variables.items():
# change key
if var in dictDepends:
dictDepends[var] = dictDepends.pop(var)
# change value
for proj in dictDepends:
dictDepends[proj] = [_.replace(var, value[0]) for _ in dictDepends[proj]]
def isKernelHeaderProj(newProj):
pattern_header = re.compile(r"^linux-.*-virtual-headers$")
if pattern_header.match(newProj):
return True
return False
def normalizeProjects(projects, config, kernels):
out_projects = set()
blAddKernelHeader = None
allKernels = config.all_kernels
for proj in projects:
newProj = proj.rstrip("/")
if isKernelHeaderProj(newProj):
newProj = ""
blAddKernelHeader = True
if newProj in allKernels:
out_projects.update(kernels)
continue
elif newProj == libc_project: # always skip libc
newProj = ""
out_projects.add(newProj)
return blAddKernelHeader, list(out_projects)
def findPlatformDependsProj(platformDependsSection, platforms):
listReplaceProj = set()
projects = []
# if there is no platform specified, check for all platforms
if len(platforms) <= 0:
platforms = platformDependsSection.keys()
# get all kernel projects of all specific platforms
for p in platforms:
if p in platformDependsSection:
projects = platformDependsSection[p]
else: # not in platform depends section
try:
projects = platformDependsSection['default']
except KeyError:
print("No such platform : " + p)
sys.exit(1)
for proj in projects:
listReplaceProj.add(proj)
return listReplaceProj
def replaceVariableInDictSection(dictDepends, variable, projsToReplace):
for proj in dictDepends:
listDepProj = dictDepends[proj]
if variable in listDepProj:
for proj in projsToReplace:
listDepProj.insert(listDepProj.index(variable), proj)
listDepProj.remove(variable)
# Replace ${XXXX} in dependency list by [${XXXX}] section
def replacePlatformDependsVariable(dictDepends, platforms, config):
def replaceVariableWithSection(variable, section_name):
try:
dictSection = config.get_section_dict(section_name)
except KeyError:
print("No such section:" + variable)
listPlatformDependsProj = findPlatformDependsProj(dictSection, platforms)
replaceVariableInDictSection(dictDepends, variable, listPlatformDependsProj)
if not config.dynamic_variables:
# for old project depends compatibility
replaceVariableWithSection("${KernelProjs}", pythonutils.SECTION_KERNEL_OLD)
return
for variable in config.dynamic_variables:
replaceVariableWithSection(variable, variable)
def ParseArgs():
parser = argparse.ArgumentParser()
parser.add_argument('-d', dest='display', action='store_true', help='Display (deprecated)')
parser.add_argument('-p', dest='platform', type=str, help='Platform')
parser.add_argument('-r', dest='r_level', type=int, default=-1, help='Reverse depenedency traverse level')
parser.add_argument('-x', dest='level', type=int, default=-1, help="Traverse level")
parser.add_argument('--header', dest='dump_header', default=False, action='store_true', help="Output kernel header")
parser.add_argument('--dump', dest='dump_pickle', help="Dump project depends graph into pickle file.")
group = parser.add_argument_group('Cache')
group.add_argument('--version', type=int, default=0)
group.add_argument('--type', choices=['tag', 'dep', 'both'], default='both')
parser.add_argument('listProj', nargs='*', type=str, help='Replace project list')
return parser.parse_args()
def loadConfigFiles(config):
dictDepends = config.project_depends
confList = glob.glob(ScriptDir + "/../source/*/SynoBuildConf/depends*")
confList = [conf for conf in confList if re.search(r'^depends(-virtual-[-.\w]+)*$', os.path.basename(conf))]
for confPath in confList:
project = confPath.split('/')[-3]
filename = confPath.split('/')[-1]
if BuildEnv.isVirtualProject(filename):
project = BuildEnv.deVirtual(project) + \
BuildEnv.VIRTUAL_PROJECT_SEPARATOR + \
BuildEnv.getVirtualName(filename)
if os.path.isfile(confPath):
depends = DependsParser(confPath)
if project not in dictDepends:
dictDepends[project] = []
dictDepends[project] = list(set(dictDepends[project] + depends.build_dep))
dictDepends[project] = list(set(dictDepends[project] + depends.build_tag))
return dictDepends
# main procedure
if __name__ == "__main__":
# get command line arguments
dictDepends = {}
listProjs = []
listOut = []
blAddKernelHeader = False
direct = 'forwardDependency'
platforms = []
level = -1
dictArgs = ParseArgs()
if dictArgs.platform:
platforms = dictArgs.platform.strip().split(" ")
if dictArgs.level >= 0 and dictArgs.r_level >= 0:
raise RuntimeError("Error! x and r can not use simultaneously")
if dictArgs.level >= 0:
level = dictArgs.level
elif dictArgs.r_level >= 0:
level = dictArgs.r_level
direct = 'backwardDependency'
# Reorder, we need to traverse all depend to sort the input projects.
if dictArgs.level == -1 and dictArgs.r_level == -1:
level = 0
config = ProjectDependsParser(config_path)
dictDepends = loadConfigFiles(config)
listProjs = dictArgs.listProj
kernels = config.get_platform_kernels(platforms)
if listProjs:
dictDepGraph = {}
blAddKernelHeader, normalizedProjList = normalizeProjects(listProjs, config, kernels)
replaceVariableSection(config, dictDepends)
replacePlatformDependsVariable(dictDepends, platforms, config)
depGraph = DepGraph(dictDepends, direct)
listOut = depGraph.traverseDepends(normalizedProjList, level)
if dictArgs.dump_pickle:
sys.setrecursionlimit(3000)
with open(dictArgs.dump_pickle, 'wb') as fd:
pickle.dump(depGraph, fd)
sys.exit(0)
# reorder need filter while args not contain 'x' and 'r'
if dictArgs.level == -1 and dictArgs.r_level == -1:
listReorder = []
for proj in listOut:
if proj in normalizedProjList:
listReorder.append(proj)
listOut = listReorder
if blAddKernelHeader or dictArgs.level >= 0 or dictArgs.dump_header:
listKernelHeader = []
for kernel in kernels:
listKernelHeader.append(kernel + '-virtual-headers')
listOut = listKernelHeader + listOut
strOut = " ".join(listOut)
if len(strOut) > 0:
print(strOut)
elif len(platforms) > 0:
# has platform specified, print kernel version
if len(kernels) == 0:
sys.stderr.write('Error: No matching kernel found!\n')
sys.exit(1)
if blAddKernelHeader or dictArgs.dump_header:
listKernelHeader = []
for kernel in kernels:
listKernelHeader.append(kernel + '-virtual-headers')
print(" ".join(listKernelHeader))
else:
print(" ".join(kernels))
else: # len(listProjs) <= 0
sys.exit(1)