prepare.py 23.5 KB
Newer Older
1
#!/usr/bin/env python
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

############################################################################
# prepare.py
# Copyright (C) 2015  Belledonne Communications, Grenoble France
#
############################################################################
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
21
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
22 23 24 25
#
############################################################################

import argparse
26
import copy
27
import imp
28 29
import os
import platform
30
import re
31 32 33 34
import shutil
import stat
import subprocess
import sys
35 36
import tempfile
from distutils.spawn import find_executable
37
from distutils.version import LooseVersion
38 39 40
from logging import error, warning, info, INFO, basicConfig
from subprocess import Popen, PIPE

41

42

43
class Target:
44 45 46 47 48 49 50 51 52 53

    def __init__(self, name, work_dir='WORK'):
        self.name = name
        self.output = 'OUTPUT'
        self.generator = None
        self.platform_name = None
        self.config_file = None
        self.toolchain_file = None
        self.required_build_platforms = None
        self.additional_args = []
54
        self.packaging_args = None
55 56 57 58
        self.work_dir = work_dir + '/' + self.name
        self.abs_work_dir = os.getcwd() + '/' + self.work_dir
        self.cmake_dir = self.work_dir + '/cmake'
        self.abs_cmake_dir = os.getcwd() + '/' + self.cmake_dir
59 60
        self.external_source_path = None
        self.lazy_install_message = True
61 62 63 64 65 66 67 68 69 70

    def output_dir(self):
        output_dir = self.output
        if not os.path.isabs(self.output):
            top_dir = os.getcwd()
            output_dir = os.path.join(top_dir, self.output)
        if platform.system() == 'Windows':
            output_dir = output_dir.replace('\\', '/')
        return output_dir

71
    def cmake_command(self, build_type, args, additional_args, verbose=True):
72
        current_path = os.path.dirname(os.path.realpath(__file__))
73 74
        if args.generator is not None:
            self.generator = args.generator # The user defined generator takes the precedence over the default target one
75 76 77 78 79 80 81
        cmd = ['cmake', current_path]
        if self.generator is not None:
            cmd += ['-G', self.generator]
        if self.platform_name is not None:
            cmd += ['-A', self.platform_name]
        cmd += ['-DCMAKE_BUILD_TYPE=' + build_type]
        cmd += ['-DCMAKE_PREFIX_PATH=' + self.output_dir(), '-DCMAKE_INSTALL_PREFIX=' + self.output_dir()]
82
        cmd += ['-DCMAKE_NO_SYSTEM_FROM_IMPORTED=YES']
83
        cmd += ['-DLINPHONE_BUILDER_WORK_DIR=' + self.abs_work_dir]
84 85 86
        if args.ccache:
            cmd += ['-DCMAKE_C_COMPILER_LAUNCHER=ccache']
            cmd += ['-DCMAKE_CXX_COMPILER_LAUNCHER=ccache']
87 88
        if self.toolchain_file is not None:
            cmd += ['-DCMAKE_TOOLCHAIN_FILE=' + self.toolchain_file]
89 90
        if self.lazy_install_message:
            cmd += ['-DCMAKE_INSTALL_MESSAGE=LAZY']
91 92
        if self.config_file is not None:
            cmd += ['-DLINPHONE_BUILDER_CONFIG_FILE=' + self.config_file]
93 94 95 96 97 98 99 100 101
        if self.external_source_path is not None:
            if platform.system() == 'Windows':
                self.external_source_path = self.external_source_path.replace('\\', '/')
            cmd += ['-DLINPHONE_BUILDER_EXTERNAL_SOURCE_PATH=' + self.external_source_path]
        if args.group:
            cmd += ['-DLINPHONE_BUILDER_GROUP_EXTERNAL_SOURCE_PATH_BUILDERS=YES']
        if args.debug_verbose:
            cmd += ['-DENABLE_DEBUG_LOGS=YES']
        if args.list_cmake_variables:
102
            cmd += ['-L']
103
        if 'package' in vars(args) and args.package:
104 105 106
            cmd += ["-DENABLE_PACKAGING=YES"]
            if self.packaging_args is not None:
                cmd += self.packaging_args
107 108
        if 'package_source' in vars(args) and args.package_source:
            cmd += ["-DENABLE_SOURCE_PACKAGING=YES"]
109 110 111 112 113 114 115 116 117 118
        for arg in self.additional_args:
            cmd += [arg]
        for arg in additional_args:
            cmd += [arg]
        cmd_str = ''
        for w in cmd:
            if ' ' in w:
                cmd_str += ' \"' + w + '\"'
            else:
                cmd_str += ' ' + w
119 120
        if verbose:
            print(cmd_str)
121 122 123 124 125
        return cmd

    def clean(self):
        if os.path.isdir(self.abs_work_dir):
            shutil.rmtree(self.abs_work_dir, ignore_errors=False, onerror=self.handle_remove_read_only)
126
        # special hack for vpx: we have switched from inside sources build to outside, so we must clean the folder properly
127 128
        vpx_dir = os.path.join(self.external_source_path, "externals", "libvpx")
        if os.path.isfile(os.path.join(vpx_dir, "Makefile")):
129
            info("Cleaning vpx source directory since we are now building it from outside directory...")
130
            Popen("git clean -xfd".split(" "), cwd=vpx_dir).wait()
131 132 133 134 135 136 137 138 139 140 141 142 143

    def veryclean(self):
        self.clean()
        if os.path.isdir(self.output_dir()):
            shutil.rmtree(self.output_dir(), ignore_errors=False, onerror=self.handle_remove_read_only)

    def handle_remove_read_only(self, func, path, exc):
        if not os.access(path, os.W_OK):
            os.chmod(path, stat.S_IWUSR)
            func(path)
        else:
            raise

144

145

146
class TargetListAction(argparse.Action):
147

148 149 150
    def __init__(self, option_strings, targets, dest=None, nargs=0, default=None, required=False, type=None, metavar=None, help=None):
        self.targets = targets
        super(TargetListAction, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, default=default, required=required, metavar=metavar, type=type, help=help)
151

152 153
    def __call__(self, parser, namespace, values, option_string=None):
        if values:
154 155
            filtered_values = []
            additional_cmake_options = []
156
            for value in values:
157 158 159
                if value.startswith('-D'):
                    additional_cmake_options += [value]
                elif value not in self.targets:
160 161
                    message = ("invalid platform: {0!r} (choose from {1})".format(value, ', '.join([repr(target) for target in self.targets])))
                    raise argparse.ArgumentError(self, message)
162 163 164 165 166 167
                else:
                    filtered_values += [value]
            if filtered_values:
                setattr(namespace, self.dest, filtered_values)
            if additional_cmake_options:
                setattr(namespace, 'additional_cmake_options', additional_cmake_options)
168

169 170 171
class ToggleAction(argparse.Action):
    def __call__(self, parser, ns, values, option):
        setattr(ns, self.dest, option[2:4] != 'no')
172

173
class Preparator:
174

175
    def __init__(self, targets={}, default_targets=[], virtual_targets={}):
176 177
        basicConfig(format="%(levelname)s: %(message)s", level=INFO)
        self.targets = targets
178
        self.virtual_targets = virtual_targets
179
        self.additional_args = []
180
        self.missing_python_dependencies = []
181
        self.missing_dependencies = {}
182
        self.wrong_cmake_version = False
183
        self.release_with_debug_info = False
184 185
        self.veryclean = False
        self.show_gpl_disclaimer = False
186
        self.min_cmake_version = None
187 188 189 190 191 192 193 194

        self.argparser = argparse.ArgumentParser(description="Prepare build of Linphone and its dependencies.")
        self.argparser.add_argument('-c', '--clean', help="Clean a previous build instead of preparing a build.", action='store_true')
        if platform.system() != 'Windows':
            self.argparser.add_argument('-cc', '--ccache', help="Use ccache to speed up the build process.", action='store_true')
        self.argparser.add_argument('-d', '--debug', help="Prepare a debug build, eg. add debug symbols and use no optimizations.", action='store_true')
        self.argparser.add_argument('-dv', '--debug-verbose', help="Activate ms_debug logs.", action='store_true')
        self.argparser.add_argument('-f', '--force', help="Force preparation, even if working directory already exist.", action='store_true')
195
        self.argparser.add_argument('-G', '--generator', help="CMake build system generator (default: let CMake choose, use cmake -h to get the complete list).", default=None, dest='generator')
196 197 198
        self.argparser.add_argument('-g', '--group', help="Group Linphone related builders.", action='store_true')
        self.argparser.add_argument('-L', '--list-cmake-variables', help="List non-advanced CMake cache variables.", action='store_true', dest='list_cmake_variables')
        self.argparser.add_argument('-lf', '--list-features', help="List optional features and their default values.", action='store_true', dest='list_features')
199 200
        self.argparser.add_argument('--tunnel', '--no-tunnel', help="Enable/Disable Tunnel.", action=ToggleAction, nargs=0)

201 202 203
        if not default_targets:
            default_targets = list(self.targets.keys())
        if len(self.targets) > 1:
204 205 206
            self.argparser.add_argument('target', nargs='*', action=TargetListAction, default=default_targets, targets=self.available_targets(),
                help="The target(s) to build for (default is '{0}'). Space separated targets in list: {1}.".format(' '.join(default_targets), ', '.join(self.available_targets())))

207 208 209 210 211 212 213
        self.argv = sys.argv[1:]
        self.load_user_config()
        self.load_project_config()

    def load_config(self, config):
        if os.path.isfile(config):
            argv = open(config).read().split()
214
            info("Loaded '{}' configuration: {}".format(config, argv))
215 216 217 218 219 220 221 222
            self.argv = argv + self.argv

    def load_project_config(self):
        self.load_config(os.path.join(os.getcwd(), "prepare.conf"))

    def load_user_config(self):
        self.load_config(os.path.join(os.getcwd(), "prepare.conf.user"))

223 224
    def available_targets(self):
        targets = [target for target in self.targets.keys()]
225
        targets += [target for target in self.virtual_targets.keys()]
226
        return targets
227 228

    def parse_args(self):
229
        self.args, self.user_additional_args = self.argparser.parse_known_args(self.argv)
230 231
        if platform.system() == 'Windows':
            self.args.ccache = False
232 233
        if hasattr(self.args, 'additional_cmake_options'):
            self.user_additional_args += self.args.additional_cmake_options
234 235 236 237 238 239 240
        new_targets = []
        for target_name in self.args.target:
            if target_name in self.virtual_targets.keys():
                new_targets += self.virtual_targets[target_name]
            else:
                new_targets += [target_name]
        self.args.target = list(set(new_targets))
241

242 243 244 245 246 247 248 249
    def check_python_module_is_present(self, modname):
        try:
            imp.find_module(modname)
            return True
        except ImportError:
            self.missing_python_dependencies += [modname]
            return False

250
    def check_is_installed(self, binary, prog, warn=True):
251 252
        if not find_executable(binary):
            if warn:
253 254
                self.missing_dependencies[binary] = prog
                #error("Could not find {}. Please install {}.".format(binary, prog))
255 256 257
            return False
        return True

258 259 260 261 262 263 264 265 266 267 268 269
    def check_cmake_version(self):
        cmake = find_executable('cmake')
        p = Popen([cmake, '--version'], shell=False, stdout=PIPE, universal_newlines=True)
        p.wait()
        if p.returncode != 0:
            self.wrong_cmake_version = True
        else:
            cmake_version = p.stdout.readlines()[0].split()[-1]
            if LooseVersion(cmake_version) < LooseVersion(self.min_cmake_version):
                self.wrong_cmake_version = True
        return not self.wrong_cmake_version

270
    def check_environment(self, submodule_directory_to_check=None):
271 272 273 274 275 276 277
        ret = 0

        # at least FFmpeg requires no whitespace in sources path...
        if " " in os.path.dirname(os.path.realpath(__file__)):
            error("Invalid location: path should not contain any spaces.")
            ret = 1

278
        ret |= not self.check_is_installed('cmake', 'cmake')
279 280
        if not ret and self.min_cmake_version is not None:
            ret |= not self.check_cmake_version()
281

282
        if submodule_directory_to_check is None:
283 284 285
            submodule_directory_to_check1 = "submodules/linphone/include"
            submodule_directory_to_check2 = "linphone-sdk/linphone/include"
        if not os.path.isdir(submodule_directory_to_check1) and not os.path.isdir(submodule_directory_to_check2):
286 287 288 289 290
            error("Missing some git submodules. Did you run:\n\tgit submodule update --init --recursive")
            ret = 1

        return ret

291
    def show_environment_errors(self):
292
        self.show_wrong_cmake_version()
293 294 295
        self.show_missing_dependencies()
        self.show_missing_python_dependencies()

296 297 298 299
    def show_wrong_cmake_version(self):
        if self.wrong_cmake_version:
            error("You need at leat CMake version {}.".format(self.min_cmake_version))

300 301 302 303
    def show_missing_dependencies(self):
        if self.missing_dependencies:
            error("The following binaries are missing: {}. Please install them.".format(' '.join(self.missing_dependencies.keys())))

304 305 306 307 308
    def show_missing_python_dependencies(self):
        if self.missing_python_dependencies:
            error("The following python modules are missing: {}. Please install them using:\n\tpip install {}".format(
                ' '.join(self.missing_python_dependencies), ' '.join(self.missing_python_dependencies)))

309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
    def gpl_disclaimer(self):
        if not self.show_gpl_disclaimer:
            return
        cmakecache = os.path.join(self.first_target().abs_work_dir, 'cmake', 'CMakeCache.txt')
        gpl_third_parties_enabled = "ENABLE_GPL_THIRD_PARTIES:BOOL=YES" in open(
           cmakecache).read() or "ENABLE_GPL_THIRD_PARTIES:BOOL=ON" in open(cmakecache).read()

        if gpl_third_parties_enabled:
            warning("\n***************************************************************************"
                    "\n***************************************************************************"
                    "\n***** CAUTION, this liblinphone SDK is built using 3rd party GPL code *****"
                    "\n***** Even if you acquired a proprietary license from Belledonne      *****"
                    "\n***** Communications, this SDK is GPL and GPL only.                   *****"
                    "\n***** To disable 3rd party gpl code, please use:                      *****"
                    "\n***** $ ./prepare.py -DENABLE_GPL_THIRD_PARTIES=NO                    *****"
                    "\n***************************************************************************"
                    "\n***************************************************************************")
        else:
            warning("\n***************************************************************************"
                    "\n***************************************************************************"
                    "\n***** Linphone SDK without 3rd party GPL software                     *****"
                    "\n***** If you acquired a proprietary license from Belledonne           *****"
                    "\n***** Communications, this SDK can be used to create                  *****"
                    "\n***** a proprietary linphone-based application.                       *****"
                    "\n***************************************************************************"
                    "\n***************************************************************************")

336 337 338
    def list_feature_target(self):
        return None

339 340 341 342 343 344
    def first_target(self):
        return self.targets[self.args.target[0]]

    def generator(self):
        return self.first_target().generator

345 346 347 348
    def get_additional_args(self):
        # Append user_additional_args to additional_args so that the user's option take the priority
        return self.additional_args + self.user_additional_args

349 350
    def list_features_with_args(self, args, additional_args):
        tmpdir = tempfile.mkdtemp(prefix="linphone-prepare")
351 352 353
        tmptarget = self.list_feature_target()
        if tmptarget is None:
            tmptarget = self.first_target()
354 355 356 357
        tmptarget.abs_cmake_dir = tmpdir
        option_regex = re.compile("ENABLE_(.*):(.*)=(.*)")
        options = {}
        ended = True
358 359 360 361 362 363
        if args.debug:
            build_type = 'Debug'
        elif self.release_with_debug_info:
            build_type = 'RelWithDebInfo'
        else:
            build_type = 'Release'
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415

        p = Popen(tmptarget.cmake_command(build_type, args, additional_args, verbose=False), cwd=tmpdir, shell=False, stdout=PIPE, universal_newlines=True)
        p.wait()
        if p.returncode != 0:
            sys.exit(-1)
        for line in p.stdout.readlines():
            match = option_regex.match(line)
            if match is not None:
                (name, typeof, value) = match.groups()
                options["ENABLE_{}".format(name)] = value
                ended &= (value == 'ON')
        shutil.rmtree(tmpdir)
        return (options, ended)

    def list_features(self, args, additional_args):
        args.list_cmake_variables = True
        additional_args_copy = additional_args
        options = {}
        info("Searching for available features...")
        # We have to iterate multiple times to activate ALL options, so that options depending
        # of others are also listed (cmake_dependent_option macro will not output options if
        # prerequisite is not met)
        while True:
            (options, ended) = self.list_features_with_args(args, additional_args_copy)
            if ended or (len(options) == len(additional_args_copy)):
                break
            else:
                additional_args_copy = []
                # Activate ALL available options
                for k in options.keys():
                    additional_args_copy.append("-D{}=ON".format(k))

        # Now that we got the list of ALL available options, we must correct default values
        # Step 1: all options are turned off by default
        for x in options.keys():
            options[x] = 'OFF'
        # Step 2: except options enabled when running with default args
        (options_tmp, ended) = self.list_features_with_args(args, additional_args)
        final_dict = dict(options, **options_tmp)

        notice_features = "Here are available features:"
        for k, v in final_dict.items():
            notice_features += "\n\t{}={}".format(k, v)
        info(notice_features)
        info("To enable some feature, please use -DENABLE_SOMEOPTION=ON (example: -DENABLE_OPUS=ON)")
        info("Similarly, to disable some feature, please use -DENABLE_SOMEOPTION=OFF (example: -DENABLE_OPUS=OFF)")

    def target_clean(self, target):
        if self.veryclean:
            target.veryclean()
        else:
            target.clean()
416

417 418 419 420 421 422 423
    def target_prepare(self, target):
        if target.required_build_platforms is not None:
            if not platform.system() in target.required_build_platforms:
                print("Cannot build target '{target}' on '{bad_build_platform}' build platform. Build it on one of {good_build_platforms}.".format(
                    target=target.name, bad_build_platform=platform.system(), good_build_platforms=', '.join(target.required_build_platforms)))
                return 52

424 425
        if type(self.args.debug) is str:
            build_type = self.args.debug
426 427 428 429
        elif self.args.debug:
            build_type = 'Debug'
        elif self.release_with_debug_info:
            build_type = 'RelWithDebInfo'
430
        else:
431
            build_type = 'Release'
432 433

        if not os.path.isdir(target.abs_cmake_dir):
434 435
            os.makedirs(target.abs_cmake_dir)

436 437 438
        self.prepare_tunnel()

        p = Popen(target.cmake_command(build_type, self.args, self.get_additional_args()), cwd=target.abs_cmake_dir, shell=False)
439 440 441
        p.communicate()

        if target.generator is None:
442 443 444 445 446 447 448
            # No generator has been specified, find the one CMake has used
            cmakecache = os.path.join(target.abs_work_dir, 'cmake', 'CMakeCache.txt')
            generator_regex = re.compile("CMAKE_GENERATOR:(.*)=(.*)", flags=re.MULTILINE)
            content = open(cmakecache).read()
            match = generator_regex.search(content)
            if match:
                target.generator = match.group(2)
449 450 451 452

        return p.returncode

    def clean(self):
453 454
        for target_name, target in self.targets.items():
            self.target_clean(target)
455 456 457
        return 0

    def prepare(self):
458
        ret = 0
459 460 461 462 463 464 465 466 467

        if self.args.list_features:
            self.list_features(self.args, self.get_additional_args())
            sys.exit(0)

        if self.args.force is False:
            for target_name, target in self.targets.items():
                if os.path.isdir(target.abs_cmake_dir):
                    print("Working directory {} already exists. Please remove it (option -c) before re-executing prepare.py "
468 469 470 471 472 473 474 475 476
                          "to avoid conflicts between executions, or force execution (option -f) if you are aware of consequences.".format(target.cmake_dir))
                    ret = 51
                    break

        if ret == 0:
            for target_name in self.args.target:
                ret = self.target_prepare(self.targets[target_name])
                if ret != 0:
                    break
477 478 479
        if ret != 0:
            if ret == 51:
                if os.path.isfile('Makefile'):
480
                    Popen("make help-prepare-options".split(" ")).wait()
481 482 483 484
                ret = 0
            return ret
        # Only generated makefile if we are using Ninja or Makefile
        if self.generator().endswith('Ninja'):
485
            if not self.check_is_installed("ninja", "ninja"):
486 487 488 489 490 491
                return 1
            self.generate_makefile('ninja -C')
            info("You can now run 'make' to build.")
        elif self.generator().endswith("Unix Makefiles"):
            self.generate_makefile('$(MAKE) -C')
            info("You can now run 'make' to build.")
492
        elif self.generator().endswith("Xcode"):
493
            self.generate_makefile('xcodebuild -project', 'Project.xcodeproj')
494
            info("You can now run 'make' to build.")
495 496
        elif self.generator().startswith("Visual Studio"):
            self.generate_vs_solution()
497 498
        else:
            warning("Not generating meta-makefile for generator {}.".format(self.generator()))
499
        self.gpl_disclaimer()
500
        return ret
501 502 503 504 505 506 507

    def run(self):
        if self.args.clean:
            return self.clean()
        else:
            return self.prepare()

508 509 510
    def generate_makefile(self, generator, project_file=''):
        pass

511 512 513
    def generate_vs_solution(self):
        pass

514 515 516 517 518 519 520 521 522 523 524
    def prepare_tunnel(self):
        if self.args.tunnel:
            if not os.path.isdir("submodules/tunnel"):
                info("Tunnel wanted but not found yet, trying to clone it...")
                p = Popen("git clone gitosis@git.linphone.org:tunnel.git submodules/tunnel".split(" "))
                p.wait()
                if p.returncode != 0:
                    error("Could not clone tunnel. Please see http://www.belledonne-communications.com/voiptunnel.html")
                    return 1
            info("Tunnel enabled.")
            self.additional_args += ["-DENABLE_TUNNEL=YES"]