cmbldup.py

from __future__ import print_function

import argparse
import logging
import os
import re
import subprocess
import sys

import pyhhi.build.common.util as util
import pyhhi.build.common.ver as ver
import pyhhi.build.vcsutil as vcsutil

from pyhhi.build.common.system import SystemInfo
from pyhhi.build.common.error import InvalidInputParameterError


class CMakeBuildUpdateAppParams(object):

    def __init__(self):
        self.top_dir = None
        self.update_prepare = True
        self.cmakebuild_repo = 'git@vigitlab.fe.hhi.de:git/CMakeBuildCore.git'
        # A CMakeBuild tag of None means latest tag.
        self.cmakebuild_tag = None


class CMakeBuildUpdateApp(object):

    def __init__(self):
        self._logger = logging.getLogger(__name__)
        self._sys_info = SystemInfo()
        self._prog_name = os.path.basename(sys.argv[0])
        self._prog_name_full = os.path.normpath(os.path.abspath(sys.argv[0]))
        self._list_summary = True
        self._summary_lines = []
        self._git_helper = vcsutil.GitHelper()
        self._git_executable = self._git_helper.get_git_executable()

        # List of python modules this script depends on.
        self._py_mod_list = ['pyhhi.build.app.cmbldup',
                             'pyhhi.build.vcsutil',
                             'pyhhi.build.common.cmbldver',
                             'pyhhi.build.common.error',
                             'pyhhi.build.common.system',
                             'pyhhi.build.common.util',
                             'pyhhi.build.common.ver']

        self._usage = """
%(prog)s [options] [top_dir]

%(prog)s updates a CMakeBuild Git subtree embedded inside a Git standard workspace.  
The update procedure requires a clean Git working directory. Unfinished work must be 
saved somehow; e.g. use git stash .. or anything else.

The optional positional argument allows you to update an existing Git workspace out-of-tree.
"""

        self._epilog = """
"""

    def __call__(self, argv=None):
        if argv is None:
            self.main(sys.argv[1:])
        else:
            self.main(argv)

    def main(self, argv):
        params = self._parse_command_line(argv)
        self.cmakebuild_update(params)

    def cmakebuild_update(self, params):

        initial_work_dir = os.getcwd()
        # (Re)create GitHelper in case the log level has been changed.
        self._git_helper = vcsutil.GitHelper()
        params = self._check_params(params)
        if params.top_dir is None:
            params.top_dir = util.get_top_dir()
        self._summary_lines = []
        if params.update_prepare:
            print("{}: preparing CMakeBuild update ...".format(self._prog_name))

            cmakebuild_version_current = self._get_current_cmakebuild_version(params.top_dir)
            if cmakebuild_version_current is None:
                print("{}: no existing CMakeBuild version found.".format(params.top_dir))

            if not self._is_workspace_clean(params.top_dir):
                return

            if params.cmakebuild_tag is None:
                cmakebuild_tag = self._git_helper.get_latest_cmakebuild_tag(params.cmakebuild_repo)

                if cmakebuild_version_current and (ver.version_compare(cmakebuild_version_current, ver.version_tuple_from_str(cmakebuild_tag)) == 0):
                    print("{}: CMakeBuild is up to date, nothing to be done.".format(params.top_dir))
                    return
            else:
                # Validate CMakeBuild tag
                cmakebuild_tags = self._git_helper.get_remote_tags(params.cmakebuild_repo)
                if params.cmakebuild_tag not in cmakebuild_tags:
                    raise InvalidInputParameterError("requested tag {0} does not exists in {1}.".format(params.cmakebuild_tag, params.cmakebuild_repo))
                cmakebuild_tag = params.cmakebuild_tag
                if cmakebuild_version_current and (ver.version_compare(ver.version_tuple_from_str(cmakebuild_tag), cmakebuild_version_current) <= 0):
                    print("{}: CMakeBuild is up to date, nothing to be done.".format(params.top_dir))
                    return

            install_dir = os.path.join(params.top_dir, 'build', 'cmakebuild_update')
            self._install_self(install_dir)

            update_script = os.path.join(install_dir, self._prog_name)
            # Prepare execv argument vector to call self with different arguments using the current python executable.
            if self._sys_info.is_windows():
                child_args = [self._sys_info.get_short_path(self._sys_info.get_python_executable())]
            else:
                child_args = [self._sys_info.get_python_executable()]
            child_args.extend([update_script, '--update'])
            if cmakebuild_tag:
                child_args.extend(['-t', cmakebuild_tag])
            if params.cmakebuild_repo:
                child_args.extend(['--cmakebuild-repo', params.cmakebuild_repo])

            # Add currrent log option to child_args[] to propagate the current log option to the child process.
            log_level = self._logger.getEffectiveLevel()
            log_level_str = None
            if log_level == logging.DEBUG:
                log_level_str = 'debug'
            elif log_level == logging.INFO:
                log_level_str = 'info'
            elif log_level == logging.WARNING:
                log_level_str = 'warning'
            elif log_level == logging.ERROR:
                log_level_str = 'error'
            elif log_level == logging.CRITICAL:
                log_level_str = 'critical'
            if log_level_str is not None:
                child_args.append('--log={}'.format(log_level_str))

            child_args.append(params.top_dir)
            os.chdir(params.top_dir)
            if self._sys_info.is_windows():
                # Unfortunately, there are issues with os.execv() on Windows and therefore a subprocess call is used instead.
                util.subproc_check_call_flushed(child_args)
            else:
                # execv() is preferred as the child is likely to remove python files just being executed
                # by the current python process.
                os.execv(child_args[0], child_args)
        else:
            try:
                print("{}: updating CMakeBuild in {} to {}".format(self._prog_name, params.top_dir, params.cmakebuild_tag))
                os.chdir(params.top_dir)
                if not self._is_workspace_clean():
                    return

                self._append_item_to_summary("Git workspace:", params.top_dir)
                if self._list_summary:
                    cmakebuild_version_current = self._get_current_cmakebuild_version(params.top_dir)
                    if cmakebuild_version_current is not None:
                        self._append_item_to_summary("Current CMakeBuild version:", "{}-{:d}".format(ver.version_tuple_to_str(cmakebuild_version_current[:3]), cmakebuild_version_current[-1]))
                    self._append_item_to_summary("New CMakeBuild version:", params.cmakebuild_tag)

                # Inventory of CMakeBuild to remember existing directories to be removed later on.
                (git_cmakebuild_dirs, cmakebuild_dirs) = self._get_git_cmakebuild_dirs(params.top_dir)
                if params.cmakebuild_tag in git_cmakebuild_dirs:
                    git_cmakebuild_dirs.remove(params.cmakebuild_tag)
                if params.cmakebuild_tag in cmakebuild_dirs:
                    cmakebuild_dirs.remove(params.cmakebuild_tag)
                # print("found existing Git CMakeBuild subdirs:", git_cmakebuild_dirs)
                # print("found existing CMakeBuild subdirs:", cmakebuild_dirs)

                # Add new CMakeBuild subtree
                self._add_cmakebuild_to_git_repo(params.cmakebuild_tag, params.cmakebuild_repo)

                # Update .svnimportprops in case is does exist.
                svnimportprops_file = os.path.join(params.top_dir, '.svnimportprops')
                svnimportprops_modified = self._update_svnimportprops(svnimportprops_file, params.cmakebuild_tag)

                for dname in git_cmakebuild_dirs:
                    git_argv = [self._git_executable, 'rm', '-r', os.path.join('CMakeBuild', dname)]
                    util.subproc_check_call_flushed(git_argv)
                if svnimportprops_modified or git_cmakebuild_dirs:
                    # Need a git commit if '.svnimportprops' has been updated or git rm -r has been launched.
                    git_comment = []
                    if git_cmakebuild_dirs:
                        git_comment.append("rm -r CMakeBuild/{}".format(','.join(git_cmakebuild_dirs)))
                    if svnimportprops_modified:
                        git_comment.append("modifying {}".format('.svnimportprops'))
                    git_argv = [self._git_executable, 'commit', '-am', '{}: {}'.format(self._prog_name, ', '.join(git_comment))]
                    util.subproc_check_call_flushed(git_argv)

                # Check again to get rid of any python cache files
                cmakebuild_dirs.extend(git_cmakebuild_dirs)
                for dname in cmakebuild_dirs:
                    dname_path = os.path.join(params.top_dir, 'CMakeBuild', dname)
                    if os.path.exists(dname_path):
                        util.rmtree(dname_path)

                print("{}: finished updating CMakeBuild in {} to {}".format(self._prog_name, params.top_dir, params.cmakebuild_tag))

                if self._list_summary:
                    print("\n{0:^80}".format("Summary"))
                    print("{0:^80}".format("======="))
                    for line in self._summary_lines:
                        print(line)
            finally:
                pass
        os.chdir(initial_work_dir)

    def _parse_command_line(self, argv):

        params = CMakeBuildUpdateAppParams()

        parser = argparse.ArgumentParser(usage=self._usage, epilog=self._epilog, formatter_class=argparse.RawDescriptionHelpFormatter)

        parser.add_argument("top_dir", action="store", nargs='?', default=None,
                            help="specifies a Git workspace to work on. By default the scripts installation path will be used to deduce the workspace.")

        util.app_args_add_log_level(parser)

        parser.add_argument("-t", "--tag", action="store", dest="cmakebuild_tag", metavar="TAG",
                            help="specifies a CMakeBuild tag to be used. If omitted the latest release tag will be used.")

        parser.add_argument("--cmakebuild-repo", action="store", dest="cmakebuild_repo", metavar="REPO", default=params.cmakebuild_repo,
                            help="specifies an alternative CMakeBuild repository overriding %(default)s." )

        parser.add_argument("--update", action="store_true", dest="update", default=False,
                            help="execute cmakebuild update, internal option reserved for the os.execv() machinery.")

        args = parser.parse_args(argv)

        # configure the python logger
        util.app_configure_logging(args.log_level)

        if args.top_dir:
            params.top_dir = os.path.normpath(os.path.abspath(args.top_dir))
        if args.cmakebuild_tag:
            params.cmakebuild_tag = args.cmakebuild_tag
        if args.cmakebuild_repo:
            params.cmakebuild_repo = args.cmakebuild_repo
        if args.update:
            params.update_prepare = False

        return params

    def _check_params(self, params):
        params.cmakebuild_repo = self._check_git_url(params.cmakebuild_repo)
        if params.top_dir:
            if not os.path.exists(params.top_dir):
                raise InvalidInputParameterError("Git workspace {} does not exist.".format(params.top_dir))
            if not os.path.exists(os.path.join(params.top_dir, '.git')):
                raise InvalidInputParameterError("Workspace {} does not seem to be a Git workspace.".format(params.top_dir))
        return params

    def _check_git_url(self, git_url):
        vcs_url = vcsutil.VcsUrl(git_url)
        proto = vcs_url.get_protocol()
        if proto in ['ssh', 'https']:
            # Normalize a remote Git URL to end with .git even if specified without the extension.
            res_path = vcs_url.get_resource_path()
            if not res_path.endswith('.git'):
                vcs_url.set_resource_path("{}.git".format(res_path))
        else:
            raise InvalidInputParameterError("Git URL {0} is not supported yet.".format(git_url))
        # Normalize all URLs.
        git_url = str(vcs_url)
        self._logger.debug("returning url={0}".format(git_url))
        return git_url

    def _install_self(self, install_dir):
        script_installer = self._create_script_installer()
        # print("py_mod_list:", self._py_mod_list)
        script_installer.install_script(install_dir, self._prog_name_full, self._py_mod_list)

    def _create_script_installer(self):
        import pyhhi.build.common.bldtools
        script_installer = pyhhi.build.common.bldtools.BuildScriptInstaller(verbose=True)
        return script_installer

    def _get_current_cmakebuild_version(self, top_dir):
        assert os.path.exists(top_dir)
        cmakebuild_top_dir = os.path.join(top_dir, 'CMakeBuild')
        if not os.path.exists(cmakebuild_top_dir):
            return None
        re_version_dir = re.compile(r'^\d+[0-9.]+-\d+$')
        version_list = []
        for fname in os.listdir(cmakebuild_top_dir):
            if os.path.isdir(os.path.join(cmakebuild_top_dir, fname)):
                if re_version_dir.match(fname) and os.path.exists(os.path.join(cmakebuild_top_dir, fname, 'CMakeBuild', 'bin', 'cmake.py')):
                    version_list.append(ver.version_tuple_from_str(fname))
        if not version_list:
            return None
        if len(version_list) > 1:
            raise InvalidInputParameterError("Workspace {} contains multiple CMakeBuild versions, please update manually.".format(top_dir))
        self._logger.debug("found CMakeBuild {} in {}".format(ver.version_tuple_to_str(version_list[0]), top_dir))
        return version_list[0]

    def _get_git_cmakebuild_dirs(self, top_dir):
        assert os.path.exists(top_dir)
        cmakebuild_top_dir = os.path.join(top_dir, 'CMakeBuild')
        if not os.path.exists(cmakebuild_top_dir):
            return []

        re_version_dir = re.compile(r'^\d+[0-9.]+-\d+$')
        git_cmakebuild_version_dirs = []
        cmakebuild_version_dirs = []
        for fname in os.listdir(cmakebuild_top_dir):
            if os.path.isdir(os.path.join(cmakebuild_top_dir, fname)):
                if re_version_dir.match(fname):
                    git_argv = [self._git_executable, 'ls-files', os.path.join('CMakeBuild', fname)]
                    retv = subprocess.check_output(git_argv, universal_newlines=True)
                    retv = retv.lstrip().rstrip()
                    if retv != '':
                        git_cmakebuild_version_dirs.append(fname)
                    else:
                        cmakebuild_version_dirs.append(fname)
        return git_cmakebuild_version_dirs, cmakebuild_version_dirs

    def _is_workspace_clean(self, top_dir=None):
        if top_dir:
            initial_work_dir = os.getcwd()
            os.chdir(top_dir)
        else:
            initial_work_dir = None
            top_dir = os.getcwd()
        git_argv = [self._git_executable, 'status', '-b', '--porcelain']
        retv = subprocess.check_output(git_argv, universal_newlines=True)
        retv = retv.lstrip().rstrip()
        status_lines = retv.splitlines()
        if status_lines[0].startswith('## HEAD (no branch)'):
            raise InvalidInputParameterError("Git workspace {} has detached HEAD, not upgradable.".format(top_dir))
        if len(status_lines) > 1:
            raise InvalidInputParameterError("Git workspace {} is not clean, please check.".format(top_dir))
        if initial_work_dir:
            os.chdir(initial_work_dir)
        return True

    def _add_cmakebuild_to_git_repo(self, cmakebuild_tag, cmakebuild_repo):

        self._logger.info("using CMakeBuild tag {} as subtree.".format(cmakebuild_tag))
        #
        # Extra annoying step to convince git subtree the working dir is clean
        #
        git_argv = [self._git_executable, 'checkout']
        self._git_helper.trace_git_command(git_argv)
        util.subproc_check_call_flushed(git_argv)

        subtree_prefix = "CMakeBuild/{0}".format(cmakebuild_tag)
        git_argv = [self._git_executable, 'subtree', 'add', '-P', subtree_prefix, '--squash', '-m', "{0}: creates subtree -P {1} tag={2}".format(self._prog_name, subtree_prefix, cmakebuild_tag), cmakebuild_repo, cmakebuild_tag]
        util.subproc_check_call_flushed(git_argv)

    def _update_svnimportprops(self, svnimportprops_file, cmakebuild_tag, svnimportprops_file_new=None):
        svnimportprops_modified = False
        if not os.path.exists(svnimportprops_file):
            return svnimportprops_modified
        svnprop_file_parser = vcsutil.SvnPropFileParser()
        svnprops = svnprop_file_parser.parse_property_file(svnimportprops_file)
        if not svnprops.has_property('svn:externals'):
            svnimportprops_modified = True
            svnprops.add_property('svn:externals', ["/svn/svn_CMakeBuild/tags/{}/CMakeBuild CMakeBuild".format(cmakebuild_tag)])
        else:
            svnprop_externals = svnprops.get_property('svn:externals')
            re_external_cmakebuild = re.compile(r'^/svn/svn_CMakeBuild/tags/([^/]+)/CMakeBuild\s+')
            svnprop_externals_new = []
            for line in svnprop_externals:
                if (line == '') or line.isspace():
                    continue
                self._logger.debug("processing line {}".format(line))
                re_match = re_external_cmakebuild.match(line)
                if re_match:
                    cmakebuild_tag_existing = re_match.group(1)
                    self._logger.debug("{}: found CMakeBuild tag {}".format(svnimportprops_file, cmakebuild_tag_existing))
                    if cmakebuild_tag_existing != cmakebuild_tag:
                        svnimportprops_modified = True
                        svnprop_externals_new.append("/svn/svn_CMakeBuild/tags/{}/CMakeBuild CMakeBuild".format(cmakebuild_tag))
                    else:
                        svnprop_externals_new.append(line)
                else:
                    svnprop_externals_new.append(line)
            if svnimportprops_modified:
                svnprops.update_property('svn:externals', svnprop_externals_new)
        if svnimportprops_modified:
            self._logger.debug("{}: updating changed svn:externals".format(svnimportprops_file))
            if svnimportprops_file_new is None:
                svnimportprops_file_new = svnimportprops_file
            with open(svnimportprops_file_new, 'w') as f:
                svnprops_str = str(svnprops)
                f.write(svnprops_str)
        return svnimportprops_modified

    def _append_item_to_summary(self, item, value):
        self._summary_lines.append("{0:<30} {1}".format(item, value))