prepare.py 23.3 KB
Newer Older
1
#!/usr/bin/env python
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

############################################################################
# prepare.py
# Copyright (C) 2015  Belledonne Communications, Grenoble France
#
############################################################################
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
21
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
22 23 24 25
#
############################################################################

import argparse
26
import copy
27
import imp
28 29
import os
import platform
30
import re
31 32 33 34
import shutil
import stat
import subprocess
import sys
35 36
import tempfile
from distutils.spawn import find_executable
37
from distutils.version import LooseVersion
38 39 40
from logging import error, warning, info, INFO, basicConfig
from subprocess import Popen, PIPE

41

42

43
class Target:
44 45 46 47 48 49 50 51 52 53

    def __init__(self, name, work_dir='WORK'):
        self.name = name
        self.output = 'OUTPUT'
        self.generator = None
        self.platform_name = None
        self.config_file = None
        self.toolchain_file = None
        self.required_build_platforms = None
        self.additional_args = []
54
        self.packaging_args = None
55 56 57 58
        self.work_dir = work_dir + '/' + self.name
        self.abs_work_dir = os.getcwd() + '/' + self.work_dir
        self.cmake_dir = self.work_dir + '/cmake'
        self.abs_cmake_dir = os.getcwd() + '/' + self.cmake_dir
59
        self.external_source_path = None
60
        self.alternate_external_source_path = None
61
        self.lazy_install_message = True
62 63 64 65 66 67 68 69 70 71

    def output_dir(self):
        output_dir = self.output
        if not os.path.isabs(self.output):
            top_dir = os.getcwd()
            output_dir = os.path.join(top_dir, self.output)
        if platform.system() == 'Windows':
            output_dir = output_dir.replace('\\', '/')
        return output_dir

72
    def cmake_command(self, build_type, args, additional_args, verbose=True):
73
        current_path = os.path.dirname(os.path.realpath(__file__))
74 75
        if args.generator is not None:
            self.generator = args.generator # The user defined generator takes the precedence over the default target one
76 77 78 79 80 81 82
        cmd = ['cmake', current_path]
        if self.generator is not None:
            cmd += ['-G', self.generator]
        if self.platform_name is not None:
            cmd += ['-A', self.platform_name]
        cmd += ['-DCMAKE_BUILD_TYPE=' + build_type]
        cmd += ['-DCMAKE_PREFIX_PATH=' + self.output_dir(), '-DCMAKE_INSTALL_PREFIX=' + self.output_dir()]
83
        cmd += ['-DCMAKE_NO_SYSTEM_FROM_IMPORTED=YES']
84
        cmd += ['-DLINPHONE_BUILDER_WORK_DIR=' + self.abs_work_dir]
85 86 87
        if args.ccache:
            cmd += ['-DCMAKE_C_COMPILER_LAUNCHER=ccache']
            cmd += ['-DCMAKE_CXX_COMPILER_LAUNCHER=ccache']
88 89
        if self.toolchain_file is not None:
            cmd += ['-DCMAKE_TOOLCHAIN_FILE=' + self.toolchain_file]
90 91
        if self.lazy_install_message:
            cmd += ['-DCMAKE_INSTALL_MESSAGE=LAZY']
92 93
        if self.config_file is not None:
            cmd += ['-DLINPHONE_BUILDER_CONFIG_FILE=' + self.config_file]
94 95 96 97
        if self.external_source_path is not None:
            if platform.system() == 'Windows':
                self.external_source_path = self.external_source_path.replace('\\', '/')
            cmd += ['-DLINPHONE_BUILDER_EXTERNAL_SOURCE_PATH=' + self.external_source_path]
98 99 100 101
        if self.alternate_external_source_path is not None:
            if platform.system() == 'Windows':
                self.alternate_external_source_path = self.alternate_external_source_path.replace('\\', '/')
            cmd += ['-DLINPHONE_BUILDER_ALTERNATE_EXTERNAL_SOURCE_PATH=' + self.alternate_external_source_path]
102 103 104 105 106
        if args.group:
            cmd += ['-DLINPHONE_BUILDER_GROUP_EXTERNAL_SOURCE_PATH_BUILDERS=YES']
        if args.debug_verbose:
            cmd += ['-DENABLE_DEBUG_LOGS=YES']
        if args.list_cmake_variables:
107
            cmd += ['-L']
108
        if 'package' in vars(args) and args.package:
109 110 111
            cmd += ["-DENABLE_PACKAGING=YES"]
            if self.packaging_args is not None:
                cmd += self.packaging_args
112 113
        if 'package_source' in vars(args) and args.package_source:
            cmd += ["-DENABLE_SOURCE_PACKAGING=YES"]
114 115 116 117 118 119 120 121 122 123
        for arg in self.additional_args:
            cmd += [arg]
        for arg in additional_args:
            cmd += [arg]
        cmd_str = ''
        for w in cmd:
            if ' ' in w:
                cmd_str += ' \"' + w + '\"'
            else:
                cmd_str += ' ' + w
124 125
        if verbose:
            print(cmd_str)
126 127 128 129 130
        return cmd

    def clean(self):
        if os.path.isdir(self.abs_work_dir):
            shutil.rmtree(self.abs_work_dir, ignore_errors=False, onerror=self.handle_remove_read_only)
131
        # special hack for vpx: we have switched from inside sources build to outside, so we must clean the folder properly
132 133
        vpx_dir = os.path.join(self.external_source_path, "externals", "libvpx")
        if os.path.isfile(os.path.join(vpx_dir, "Makefile")):
134
            info("Cleaning vpx source directory since we are now building it from outside directory...")
135
            Popen("git clean -xfd".split(" "), cwd=vpx_dir).wait()
136 137 138 139 140 141 142 143 144 145 146 147 148

    def veryclean(self):
        self.clean()
        if os.path.isdir(self.output_dir()):
            shutil.rmtree(self.output_dir(), ignore_errors=False, onerror=self.handle_remove_read_only)

    def handle_remove_read_only(self, func, path, exc):
        if not os.access(path, os.W_OK):
            os.chmod(path, stat.S_IWUSR)
            func(path)
        else:
            raise

149

150

151
class TargetListAction(argparse.Action):
152

153 154 155
    def __init__(self, option_strings, targets, dest=None, nargs=0, default=None, required=False, type=None, metavar=None, help=None):
        self.targets = targets
        super(TargetListAction, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, default=default, required=required, metavar=metavar, type=type, help=help)
156

157 158
    def __call__(self, parser, namespace, values, option_string=None):
        if values:
159 160
            filtered_values = []
            additional_cmake_options = []
161
            for value in values:
162 163 164
                if value.startswith('-D'):
                    additional_cmake_options += [value]
                elif value not in self.targets:
165 166
                    message = ("invalid platform: {0!r} (choose from {1})".format(value, ', '.join([repr(target) for target in self.targets])))
                    raise argparse.ArgumentError(self, message)
167 168 169 170 171 172
                else:
                    filtered_values += [value]
            if filtered_values:
                setattr(namespace, self.dest, filtered_values)
            if additional_cmake_options:
                setattr(namespace, 'additional_cmake_options', additional_cmake_options)
173

174 175 176
class ToggleAction(argparse.Action):
    def __call__(self, parser, ns, values, option):
        setattr(ns, self.dest, option[2:4] != 'no')
177

178
class Preparator:
179

180
    def __init__(self, targets={}, default_targets=[], virtual_targets={}):
181 182
        basicConfig(format="%(levelname)s: %(message)s", level=INFO)
        self.targets = targets
183
        self.virtual_targets = virtual_targets
184
        self.additional_args = []
185
        self.missing_python_dependencies = []
186
        self.missing_dependencies = {}
187
        self.wrong_cmake_version = False
188
        self.release_with_debug_info = True
189 190
        self.veryclean = False
        self.show_gpl_disclaimer = False
191
        self.min_cmake_version = None
192 193 194 195 196 197 198 199

        self.argparser = argparse.ArgumentParser(description="Prepare build of Linphone and its dependencies.")
        self.argparser.add_argument('-c', '--clean', help="Clean a previous build instead of preparing a build.", action='store_true')
        if platform.system() != 'Windows':
            self.argparser.add_argument('-cc', '--ccache', help="Use ccache to speed up the build process.", action='store_true')
        self.argparser.add_argument('-d', '--debug', help="Prepare a debug build, eg. add debug symbols and use no optimizations.", action='store_true')
        self.argparser.add_argument('-dv', '--debug-verbose', help="Activate ms_debug logs.", action='store_true')
        self.argparser.add_argument('-f', '--force', help="Force preparation, even if working directory already exist.", action='store_true')
200
        self.argparser.add_argument('-G', '--generator', help="CMake build system generator (default: let CMake choose, use cmake -h to get the complete list).", default=None, dest='generator')
201 202 203
        self.argparser.add_argument('-g', '--group', help="Group Linphone related builders.", action='store_true')
        self.argparser.add_argument('-L', '--list-cmake-variables', help="List non-advanced CMake cache variables.", action='store_true', dest='list_cmake_variables')
        self.argparser.add_argument('-lf', '--list-features', help="List optional features and their default values.", action='store_true', dest='list_features')
204 205
        self.argparser.add_argument('--tunnel', '--no-tunnel', help="Enable/Disable Tunnel.", action=ToggleAction, nargs=0)

206 207 208
        if not default_targets:
            default_targets = list(self.targets.keys())
        if len(self.targets) > 1:
209 210 211
            self.argparser.add_argument('target', nargs='*', action=TargetListAction, default=default_targets, targets=self.available_targets(),
                help="The target(s) to build for (default is '{0}'). Space separated targets in list: {1}.".format(' '.join(default_targets), ', '.join(self.available_targets())))

212 213 214 215 216 217 218
        self.argv = sys.argv[1:]
        self.load_user_config()
        self.load_project_config()

    def load_config(self, config):
        if os.path.isfile(config):
            argv = open(config).read().split()
219
            info("Loaded '{}' configuration: {}".format(config, argv))
220 221 222 223 224 225 226 227
            self.argv = argv + self.argv

    def load_project_config(self):
        self.load_config(os.path.join(os.getcwd(), "prepare.conf"))

    def load_user_config(self):
        self.load_config(os.path.join(os.getcwd(), "prepare.conf.user"))

228 229
    def available_targets(self):
        targets = [target for target in self.targets.keys()]
230
        targets += [target for target in self.virtual_targets.keys()]
231
        return targets
232 233

    def parse_args(self):
234
        self.args, self.user_additional_args = self.argparser.parse_known_args(self.argv)
235 236
        if platform.system() == 'Windows':
            self.args.ccache = False
237 238
        if hasattr(self.args, 'additional_cmake_options'):
            self.user_additional_args += self.args.additional_cmake_options
239 240 241 242 243 244 245
        new_targets = []
        for target_name in self.args.target:
            if target_name in self.virtual_targets.keys():
                new_targets += self.virtual_targets[target_name]
            else:
                new_targets += [target_name]
        self.args.target = list(set(new_targets))
246

247 248 249 250 251 252 253 254
    def check_python_module_is_present(self, modname):
        try:
            imp.find_module(modname)
            return True
        except ImportError:
            self.missing_python_dependencies += [modname]
            return False

255
    def check_is_installed(self, binary, prog, warn=True):
256 257
        if not find_executable(binary):
            if warn:
258 259
                self.missing_dependencies[binary] = prog
                #error("Could not find {}. Please install {}.".format(binary, prog))
260 261 262
            return False
        return True

263 264 265 266 267 268 269 270 271 272 273 274
    def check_cmake_version(self):
        cmake = find_executable('cmake')
        p = Popen([cmake, '--version'], shell=False, stdout=PIPE, universal_newlines=True)
        p.wait()
        if p.returncode != 0:
            self.wrong_cmake_version = True
        else:
            cmake_version = p.stdout.readlines()[0].split()[-1]
            if LooseVersion(cmake_version) < LooseVersion(self.min_cmake_version):
                self.wrong_cmake_version = True
        return not self.wrong_cmake_version

275
    def check_environment(self, submodule_directory_to_check=None):
276 277 278 279 280 281 282
        ret = 0

        # at least FFmpeg requires no whitespace in sources path...
        if " " in os.path.dirname(os.path.realpath(__file__)):
            error("Invalid location: path should not contain any spaces.")
            ret = 1

283
        ret |= not self.check_is_installed('cmake', 'cmake')
284 285
        if not ret and self.min_cmake_version is not None:
            ret |= not self.check_cmake_version()
286

287
        if submodule_directory_to_check is None:
288 289 290
            submodule_directory_to_check1 = "submodules/linphone/include"
            submodule_directory_to_check2 = "linphone-sdk/linphone/include"
        if not os.path.isdir(submodule_directory_to_check1) and not os.path.isdir(submodule_directory_to_check2):
291 292 293 294 295
            error("Missing some git submodules. Did you run:\n\tgit submodule update --init --recursive")
            ret = 1

        return ret

296
    def show_environment_errors(self):
297
        self.show_wrong_cmake_version()
298 299 300
        self.show_missing_dependencies()
        self.show_missing_python_dependencies()

301 302 303 304
    def show_wrong_cmake_version(self):
        if self.wrong_cmake_version:
            error("You need at leat CMake version {}.".format(self.min_cmake_version))

305 306 307 308
    def show_missing_dependencies(self):
        if self.missing_dependencies:
            error("The following binaries are missing: {}. Please install them.".format(' '.join(self.missing_dependencies.keys())))

309 310 311 312 313
    def show_missing_python_dependencies(self):
        if self.missing_python_dependencies:
            error("The following python modules are missing: {}. Please install them using:\n\tpip install {}".format(
                ' '.join(self.missing_python_dependencies), ' '.join(self.missing_python_dependencies)))

314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
    def gpl_disclaimer(self):
        if not self.show_gpl_disclaimer:
            return
        cmakecache = os.path.join(self.first_target().abs_work_dir, 'cmake', 'CMakeCache.txt')
        gpl_third_parties_enabled = "ENABLE_GPL_THIRD_PARTIES:BOOL=YES" in open(
           cmakecache).read() or "ENABLE_GPL_THIRD_PARTIES:BOOL=ON" in open(cmakecache).read()

        if gpl_third_parties_enabled:
            warning("\n***************************************************************************"
                    "\n***************************************************************************"
                    "\n***** CAUTION, this liblinphone SDK is built using 3rd party GPL code *****"
                    "\n***** Even if you acquired a proprietary license from Belledonne      *****"
                    "\n***** Communications, this SDK is GPL and GPL only.                   *****"
                    "\n***** To disable 3rd party gpl code, please use:                      *****"
                    "\n***** $ ./prepare.py -DENABLE_GPL_THIRD_PARTIES=NO                    *****"
                    "\n***************************************************************************"
                    "\n***************************************************************************")
        else:
            warning("\n***************************************************************************"
                    "\n***************************************************************************"
                    "\n***** Linphone SDK without 3rd party GPL software                     *****"
                    "\n***** If you acquired a proprietary license from Belledonne           *****"
                    "\n***** Communications, this SDK can be used to create                  *****"
                    "\n***** a proprietary linphone-based application.                       *****"
                    "\n***************************************************************************"
                    "\n***************************************************************************")

341 342 343
    def list_feature_target(self):
        return None

344 345 346 347 348 349
    def first_target(self):
        return self.targets[self.args.target[0]]

    def generator(self):
        return self.first_target().generator

350 351 352 353
    def get_additional_args(self):
        # Append user_additional_args to additional_args so that the user's option take the priority
        return self.additional_args + self.user_additional_args

354 355
    def list_features_with_args(self, args, additional_args):
        tmpdir = tempfile.mkdtemp(prefix="linphone-prepare")
356 357 358
        tmptarget = self.list_feature_target()
        if tmptarget is None:
            tmptarget = self.first_target()
359 360 361 362
        tmptarget.abs_cmake_dir = tmpdir
        option_regex = re.compile("ENABLE_(.*):(.*)=(.*)")
        options = {}
        ended = True
363 364 365 366 367 368
        if args.debug:
            build_type = 'Debug'
        elif self.release_with_debug_info:
            build_type = 'RelWithDebInfo'
        else:
            build_type = 'Release'
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420

        p = Popen(tmptarget.cmake_command(build_type, args, additional_args, verbose=False), cwd=tmpdir, shell=False, stdout=PIPE, universal_newlines=True)
        p.wait()
        if p.returncode != 0:
            sys.exit(-1)
        for line in p.stdout.readlines():
            match = option_regex.match(line)
            if match is not None:
                (name, typeof, value) = match.groups()
                options["ENABLE_{}".format(name)] = value
                ended &= (value == 'ON')
        shutil.rmtree(tmpdir)
        return (options, ended)

    def list_features(self, args, additional_args):
        args.list_cmake_variables = True
        additional_args_copy = additional_args
        options = {}
        info("Searching for available features...")
        # We have to iterate multiple times to activate ALL options, so that options depending
        # of others are also listed (cmake_dependent_option macro will not output options if
        # prerequisite is not met)
        while True:
            (options, ended) = self.list_features_with_args(args, additional_args_copy)
            if ended or (len(options) == len(additional_args_copy)):
                break
            else:
                additional_args_copy = []
                # Activate ALL available options
                for k in options.keys():
                    additional_args_copy.append("-D{}=ON".format(k))

        # Now that we got the list of ALL available options, we must correct default values
        # Step 1: all options are turned off by default
        for x in options.keys():
            options[x] = 'OFF'
        # Step 2: except options enabled when running with default args
        (options_tmp, ended) = self.list_features_with_args(args, additional_args)
        final_dict = dict(options, **options_tmp)

        notice_features = "Here are available features:"
        for k, v in final_dict.items():
            notice_features += "\n\t{}={}".format(k, v)
        info(notice_features)
        info("To enable some feature, please use -DENABLE_SOMEOPTION=ON (example: -DENABLE_OPUS=ON)")
        info("Similarly, to disable some feature, please use -DENABLE_SOMEOPTION=OFF (example: -DENABLE_OPUS=OFF)")

    def target_clean(self, target):
        if self.veryclean:
            target.veryclean()
        else:
            target.clean()
421

422 423 424 425 426 427 428
    def target_prepare(self, target):
        if target.required_build_platforms is not None:
            if not platform.system() in target.required_build_platforms:
                print("Cannot build target '{target}' on '{bad_build_platform}' build platform. Build it on one of {good_build_platforms}.".format(
                    target=target.name, bad_build_platform=platform.system(), good_build_platforms=', '.join(target.required_build_platforms)))
                return 52

429 430
        if type(self.args.debug) is str:
            build_type = self.args.debug
431 432 433 434
        elif self.args.debug:
            build_type = 'Debug'
        elif self.release_with_debug_info:
            build_type = 'RelWithDebInfo'
435
        else:
436
            build_type = 'Release'
437 438

        if not os.path.isdir(target.abs_cmake_dir):
439 440
            os.makedirs(target.abs_cmake_dir)

441 442 443
        self.prepare_tunnel()

        p = Popen(target.cmake_command(build_type, self.args, self.get_additional_args()), cwd=target.abs_cmake_dir, shell=False)
444 445 446
        p.communicate()

        if target.generator is None:
447 448 449 450 451 452 453
            # No generator has been specified, find the one CMake has used
            cmakecache = os.path.join(target.abs_work_dir, 'cmake', 'CMakeCache.txt')
            generator_regex = re.compile("CMAKE_GENERATOR:(.*)=(.*)", flags=re.MULTILINE)
            content = open(cmakecache).read()
            match = generator_regex.search(content)
            if match:
                target.generator = match.group(2)
454 455 456 457

        return p.returncode

    def clean(self):
458 459
        for target_name, target in self.targets.items():
            self.target_clean(target)
460 461 462
        return 0

    def prepare(self):
463
        ret = 0
464 465 466 467 468 469 470 471 472

        if self.args.list_features:
            self.list_features(self.args, self.get_additional_args())
            sys.exit(0)

        if self.args.force is False:
            for target_name, target in self.targets.items():
                if os.path.isdir(target.abs_cmake_dir):
                    print("Working directory {} already exists. Please remove it (option -c) before re-executing prepare.py "
473 474 475 476 477 478 479 480 481
                          "to avoid conflicts between executions, or force execution (option -f) if you are aware of consequences.".format(target.cmake_dir))
                    ret = 51
                    break

        if ret == 0:
            for target_name in self.args.target:
                ret = self.target_prepare(self.targets[target_name])
                if ret != 0:
                    break
482 483 484
        if ret != 0:
            if ret == 51:
                if os.path.isfile('Makefile'):
485
                    Popen("make help-prepare-options".split(" ")).wait()
486 487 488 489
                ret = 0
            return ret
        # Only generated makefile if we are using Ninja or Makefile
        if self.generator().endswith('Ninja'):
490
            if not self.check_is_installed("ninja", "ninja"):
491 492 493 494 495 496
                return 1
            self.generate_makefile('ninja -C')
            info("You can now run 'make' to build.")
        elif self.generator().endswith("Unix Makefiles"):
            self.generate_makefile('$(MAKE) -C')
            info("You can now run 'make' to build.")
497
        elif self.generator().endswith("Xcode"):
498
            self.generate_makefile('xcodebuild -project', 'Project.xcodeproj')
499
            info("You can now run 'make' to build.")
500 501
        elif self.generator().startswith("Visual Studio"):
            self.generate_vs_solution()
502 503
        else:
            warning("Not generating meta-makefile for generator {}.".format(self.generator()))
504
        self.gpl_disclaimer()
505
        return ret
506 507 508 509 510 511 512

    def run(self):
        if self.args.clean:
            return self.clean()
        else:
            return self.prepare()

513 514 515
    def generate_makefile(self, generator, project_file=''):
        pass

516 517 518
    def generate_vs_solution(self):
        pass

519 520 521
    def prepare_tunnel(self):
        if self.args.tunnel:
            self.additional_args += ["-DENABLE_TUNNEL=YES"]