diff --git a/.gitignore b/.gitignore index 3e142a47..6fba8b13 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ dist git-access-testing github-access-testing *~ +*.idea +.tox +*nosetests.xml diff --git a/docs/reference/buildsystem.md b/docs/reference/buildsystem.md index e728c9f2..81a2c5eb 100644 --- a/docs/reference/buildsystem.md +++ b/docs/reference/buildsystem.md @@ -273,3 +273,19 @@ subdirectory. Within this directory build products are further divided by the name of the [target](tutorial/targets.html) being built. This makes it safe to switch between building for different targets without cleaning. + +## # Exporting Builds + +Yotta can export the generated file structure to be later built elsewhere using: + +``` +yotta build -g -x some/output/path +``` + +In this case `-g` is used to stop the local compile from happening immediately, +and `-x` copies the build files to the specified directory. + +To build and link your project you would then have to run the cmake commands +against that output directory. The generated files use relative paths, +making them portable; for example to send to an independent build server. + diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 3ac8dd1b..7f7086d7 100755 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -80,6 +80,8 @@ Options: This option is deprecated because it is now the default, unless `--debug-build` is specified. + * **`--export`, `-x`**: export mode. If flag is set, generates a buildable directory structure at the specified path. + * **`--cmake-generator`, `-G`**: specify the CMake Generator. CMake can generate project files for various editors and IDEs. * **`name ...`**: one or more modules may be specified, in which case only these diff --git a/yotta/build.py b/yotta/build.py index a9373118..deb9a771 100644 --- a/yotta/build.py +++ b/yotta/build.py @@ -6,6 +6,7 @@ # standard library modules, , , import os import logging +import shutil # validate, , validate things, internal @@ -16,6 +17,11 @@ from yotta.lib import cmakegen # Target, , represents an installed target, internal from yotta.lib import target +# settings, , load and save settings, internal +from yotta.lib import settings +# paths +from yotta.lib import paths +from yotta.lib import fsutils # install, , install subcommand, internal from yotta import install # --config option, , , internal @@ -27,6 +33,12 @@ def addOptions(parser, add_build_targets=True): action='store_true', default=False, help='Only generate CMakeLists, don\'t run CMake or build' ) + parser.add_argument('-x', '--export', dest='export', + default=False, const=True, nargs='?', + help=( + 'Export mode. If flag is set, generates a buildable directory structure at the specified path.' + ) + ) parser.add_argument('-r', '--release-build', dest='release_build', action='store_true', default=True) parser.add_argument('-d', '--debug-build', dest='release_build', action='store_false', default=True) parser.add_argument('-r0', '--release-no-debug-info-build', dest='release_no_debug_info_build', action='store_true', default=False) @@ -57,16 +69,36 @@ def installAndBuild(args, following_args): If status: is nonzero there was some sort of error. Other properties are optional, and may not be set if that step was not attempted. ''' - build_status = generate_status = install_status = 0 - - if not hasattr(args, 'build_targets'): - vars(args)['build_targets'] = [] + build_status = 0 + generate_status = 0 + error = None + args_dict = vars(args) + export_mode_or_path = args_dict.get('export') + + if export_mode_or_path: + logging.info('this build will be configured for exporting') + new_modules = settings.getProperty('build', 'vendor_modules_directory') or 'modules' + new_targets = settings.getProperty('build', 'vendor_targets_directory') or 'targets' + new_build_modules = settings.getProperty('build', 'built_modules_directory') or 'modules' + # to save downloading the cached files, copy the old cache + try: + if not os.path.exists(new_modules): + shutil.copytree(paths.Modules_Folder, new_modules) + shutil.copytree(paths.Targets_Folder, new_targets) + except Exception as e: + logging.error(e) + logging.error('failed to use existing cache; rebuilding') + + paths.Modules_Folder = new_modules + paths.Targets_Folder = new_targets + paths.BUILT_MODULES_DIR = new_build_modules + + args_dict.setdefault('build_targets', []) if 'test' in args.build_targets: logging.error('Cannot build "test". Use "yotta test" to run tests.') return {'status':1} - cwd = os.getcwd() c = validate.currentDirectoryModule() if not c: return {'status':1} @@ -83,26 +115,27 @@ def installAndBuild(args, following_args): # run the install command before building, we need to add some options the # install command expects to be present to do this: - vars(args)['component'] = None - vars(args)['act_globally'] = False + args_dict['component'] = None + args_dict['act_globally'] = False if not hasattr(args, 'install_test_deps'): if 'all_tests' in args.build_targets: - vars(args)['install_test_deps'] = 'all' + args_dict['install_test_deps'] = 'all' elif not len(args.build_targets): - vars(args)['install_test_deps'] = 'own' + args_dict['install_test_deps'] = 'own' else: # If the named build targets include tests from other modules, we # need to install the deps for those modules. To do this we need to # be able to tell which module a library belongs to, which is not # straightforward (especially if there is custom cmake involved). # That's why this is 'all', and not 'none'. - vars(args)['install_test_deps'] = 'all' + args_dict['install_test_deps'] = 'all' # install may exit non-zero for non-fatal errors (such as incompatible # version specs), which it will display install_status = install.execCommand(args, []) - builddir = os.path.join(cwd, 'build', target.getName()) + builddir = os.path.join(os.getcwd(), paths.DEFAULT_BUILD_DIR, target.getName()) if not export_mode_or_path else \ + os.path.join(os.getcwd(), paths.DEFAULT_EXPORT_BUILD_RELPATH) all_deps = c.getDependenciesRecursive( target = target, @@ -126,7 +159,7 @@ def installAndBuild(args, following_args): logging.debug("config done, merged config: %s", config['merged_config_json']) script_environment = { - 'YOTTA_MERGED_CONFIG_FILE': config['merged_config_json'] + 'YOTTA_MERGED_CONFIG_FILE': str(config['merged_config_json']) } # run pre-generate scripts for all components: runScriptWithModules(c, all_deps.values(), 'preGenerate', script_environment) @@ -140,7 +173,42 @@ def installAndBuild(args, following_args): # run pre-build scripts for all components: runScriptWithModules(c, all_deps.values(), 'preBuild', script_environment) - if (not hasattr(args, 'generate_only')) or (not args.generate_only): + if export_mode_or_path and export_mode_or_path is not True: + # includes relative to cwd / project top level + includes = { + 'source', + paths.DEFAULT_EXPORT_BUILD_RELPATH, + paths.Modules_Folder, + paths.Targets_Folder, + 'yotta-cloud-client', # not sure about whitelisting... where is this coming from? + } + + def ignore_on_copy(current_dir, local_paths): + # only at the project root + if current_dir == '.': + # exclude = !include + return [p for p in local_paths if p not in includes] + return [] + export_to = os.path.abspath(export_mode_or_path) + logging.info('exporting unbuilt project source and dependencies to %s', export_to) + check = os.path.join(export_to, 'source') # this might be insufficient, or indeed misleading. + if os.path.exists(export_to) and os.listdir(export_to) and not os.path.exists(check): + # as the export can be any path we do a sanity check before calling 'rmRf' + raise Exception( + 'Aborting export: directory is not empty and does not look like a previous export' + ' (expecting: %s)' + % (check) + ) + fsutils.rmRf(export_to) + shutil.copytree(src='.', dst=export_to, ignore=ignore_on_copy) + fsutils.rmRf(builddir) + fsutils.rmRf(paths.Modules_Folder) + fsutils.rmRf(paths.Targets_Folder) + + if args_dict.get('generate_only'): + logging.info('skipping build step') + else: + # build in the current directory error = target.build( builddir, c, args, release_build=args.release_build, build_args=following_args, targets=args.build_targets, diff --git a/yotta/clean.py b/yotta/clean.py index 6d3ca2a5..3c16ca2b 100644 --- a/yotta/clean.py +++ b/yotta/clean.py @@ -21,4 +21,3 @@ def execCommand(args, following_args): return 1 fsutils.rmRf(os.path.join(c.path, 'build')) - diff --git a/yotta/debug.py b/yotta/debug.py index a84d617c..1d7ad7a6 100644 --- a/yotta/debug.py +++ b/yotta/debug.py @@ -9,6 +9,9 @@ # validate, , validate things, internal from yotta.lib import validate +# settings, , load and save settings, internal +# paths +from yotta.lib import paths # --config option, , , internal from yotta import options @@ -33,7 +36,7 @@ def execCommand(args, following_args): logging.error(error) return 1 - builddir = os.path.join(cwd, 'build', target.getName()) + builddir = os.path.join(os.getcwd(), paths.DEFAULT_BUILD_DIR, target.getName()) if args.program is None: if c.isApplication(): diff --git a/yotta/init.py b/yotta/init.py index 3d9f78dd..784d734e 100644 --- a/yotta/init.py +++ b/yotta/init.py @@ -11,6 +11,7 @@ # Component, , represents an installed component, internal from yotta.lib import component +from yotta.lib import paths # version, , represent versions and specifications, internal from yotta.lib import version # validate, , validate various things, internal @@ -67,7 +68,7 @@ def yesNo(string): yesNo.__allowed_message = ' Please reply "Yes", or "No".' def isBannedName(name): - return name in ('test', 'source', 'include', 'yotta_modules', 'yotta_targets') + return name in ('test', 'source', 'include', paths.Modules_Folder, paths.Targets_Folder) def notBannedName(s): if isBannedName(s): diff --git a/yotta/lib/cmakegen.py b/yotta/lib/cmakegen.py index 0f750d44..0ca3c05a 100644 --- a/yotta/lib/cmakegen.py +++ b/yotta/lib/cmakegen.py @@ -6,6 +6,7 @@ # standard library modules, , , import os import logging +import functools import re import itertools from collections import defaultdict @@ -17,11 +18,18 @@ # fsutils, , misc filesystem utils, internal from yotta.lib import fsutils +from yotta.lib import paths + Template_Dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates') logger = logging.getLogger('cmakegen') -Ignore_Subdirs = set(('build','yotta_modules', 'yotta_targets', 'CMake')) +Ignore_Subdirs = { + 'build', + 'CMake', + paths.Modules_Folder, + paths.Targets_Folder, +} jinja_environment = Environment(loader=FileSystemLoader(Template_Dir), trim_blocks=True, lstrip_blocks=True) @@ -57,6 +65,7 @@ def __init__(self, directory, target): self.config_json_file = None self.build_info_include_file = None self.build_uuid = None + self.relative = None def _writeFile(self, path, contents): dirname = os.path.dirname(path) @@ -67,25 +76,26 @@ def configure(self, component, all_dependencies): ''' Ensure all config-time files have been generated. Return a dictionary of generated items. ''' - r = {} - - builddir = self.buildroot - # only dependencies which are actually valid can contribute to the # config data (which includes the versions of all dependencies in its # build info) if the dependencies aren't available we can't tell what # version they are. Anything missing here should always be a test # dependency that isn't going to be used, otherwise the yotta build # command will fail before we get here + + # a single-input filter function to convert paths to be relative to the buildroot + self.relative = functools.partial(os.path.relpath, start=self.buildroot) + jinja_environment.filters['relative'] = self.relative + available_dependencies = OrderedDict((k, v) for k, v in all_dependencies.items() if v) self.set_toplevel_definitions = '' if self.build_info_include_file is None: - self.build_info_include_file, build_info_definitions = self.getBuildInfo(component.path, builddir) + self.build_info_include_file, build_info_definitions = self.getBuildInfo(component.path, self.buildroot) self.set_toplevel_definitions += build_info_definitions if self.config_include_file is None: - self.config_include_file, config_definitions, self.config_json_file = self._getConfigData(available_dependencies, component, builddir, self.build_info_include_file) + self.config_include_file, config_definitions, self.config_json_file = self._getConfigData(available_dependencies, component, self.buildroot, self.build_info_include_file) self.set_toplevel_definitions += config_definitions self.configured = True @@ -107,12 +117,13 @@ def generateRecursive(self, component, all_components, builddir=None, modbuilddi for error in gen.generateRecursive(...): print(error) ''' - assert(self.configured) + if not self.configured: + yield 'Not configured' if builddir is None: builddir = self.buildroot if modbuilddir is None: - modbuilddir = os.path.join(builddir, 'ym') + modbuilddir = os.path.join(builddir, paths.BUILT_MODULES_DIR) if processed_components is None: processed_components = dict() @@ -340,9 +351,9 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade # make the path to the build-info header available both to CMake and # in the preprocessor: - full_build_info_header_path = replaceBackslashes(os.path.abspath(build_info_header_path)) - logger.debug('build info header include path: "%s"', full_build_info_header_path) - definitions.append(('YOTTA_BUILD_INFO_HEADER', '"'+full_build_info_header_path+'"')) + full_build_info_header_path = '"${CMAKE_BINARY_DIR}/%s"' % replaceBackslashes(self.relative(build_info_header_path)) + logger.debug('build info header include path: %s', full_build_info_header_path) + definitions.append(('YOTTA_BUILD_INFO_HEADER', full_build_info_header_path)) for target in self.target.getSimilarTo_Deprecated(): if '*' not in target: @@ -383,10 +394,10 @@ def _getConfigData(self, all_dependencies, component, builddir, build_info_heade # out for gcc-compatible compilers only: config_include_file = os.path.join(builddir, 'yotta_config.h') config_json_file = os.path.join(builddir, 'yotta_config.json') - set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"%s\")\n' % replaceBackslashes(os.path.abspath(config_json_file)) + set_definitions += 'set(YOTTA_CONFIG_MERGED_JSON_FILE \"${CMAKE_BINARY_DIR}/%s\")\n' % replaceBackslashes(self.relative(config_json_file)) self._writeFile( - config_include_file, + config_include_file + '.cmake_template', '#ifndef __YOTTA_CONFIG_H__\n'+ '#define __YOTTA_CONFIG_H__\n'+ add_defs_header+ @@ -468,33 +479,29 @@ def generate( another component. ''' - include_root_dirs = '' + include_root_dirs = [] if application is not None and component is not application: - include_root_dirs += 'include_directories("%s")\n' % replaceBackslashes(application.path) + include_root_dirs.append(application.path) - include_sys_dirs = '' - include_other_dirs = '' + include_sys_dirs = [] + include_other_dirs = [] for name, c in itertools.chain(((component.getName(), component),), all_dependencies.items()): if c is not component and c.isTestDependency(): continue - include_root_dirs += 'include_directories("%s")\n' % replaceBackslashes(c.path) + include_root_dirs.append(c.path) dep_sys_include_dirs = c.getExtraSysIncludes() for d in dep_sys_include_dirs: - include_sys_dirs += 'include_directories(SYSTEM "%s")\n' % replaceBackslashes(os.path.join(c.path, d)) + include_sys_dirs.append(os.path.join(c.path, d)) dep_extra_include_dirs = c.getExtraIncludes() for d in dep_extra_include_dirs: - include_other_dirs += 'include_directories("%s")\n' % replaceBackslashes(os.path.join(c.path, d)) + include_other_dirs.append(os.path.join(c.path, d)) - add_depend_subdirs = '' + add_depend_subdirs = [] for name, c in active_dependencies.items(): - depend_subdir = replaceBackslashes(os.path.join(modbuilddir, name)) - relpath = replaceBackslashes(os.path.relpath(depend_subdir, self.buildroot)) - add_depend_subdirs += \ - 'add_subdirectory(\n' \ - ' "%s"\n' \ - ' "${CMAKE_BINARY_DIR}/%s"\n' \ - ')\n' \ - % (depend_subdir, relpath) + # a project-relative path to the top level dep + source_dir = os.path.join(modbuilddir, name) + binary_dir = source_dir + add_depend_subdirs.append((source_dir, binary_dir)) delegate_to_existing = None delegate_build_dir = None @@ -603,25 +610,22 @@ def generate( # generate the top-level toolchain file: template = jinja_environment.get_template('toolchain.cmake') file_contents = template.render({ #pylint: disable=no-member - # toolchain files are provided in hierarchy - # order, but the template needs them in reverse - # order (base-first): + # toolchain files are provided in hierarchy + # order, but the template needs them in reverse + # order (base-first): "toolchain_files": self.target.getToolchainFiles() }) self._writeFile(toolchain_file_path, file_contents) # generate the top-level CMakeLists.txt template = jinja_environment.get_template('base_CMakeLists.txt') - - relpath = os.path.relpath(builddir, self.buildroot) - file_contents = template.render({ #pylint: disable=no-member "toplevel": toplevel, "target_name": self.target.getName(), "set_definitions": self.set_toplevel_definitions, "toolchain_file": toolchain_file_path, "component": component, - "relpath": relpath, + "build_dir": builddir, "include_root_dirs": include_root_dirs, "include_sys_dirs": include_sys_dirs, "include_other_dirs": include_other_dirs, diff --git a/yotta/lib/component.py b/yotta/lib/component.py index 8357165c..7bfd59da 100644 --- a/yotta/lib/component.py +++ b/yotta/lib/component.py @@ -19,6 +19,7 @@ from yotta.lib import fsutils # Pack, , common parts of Components/Targets, internal from yotta.lib import pack +from yotta.lib import paths # !!! FIXME: should components lock their description file while they exist? # If not there are race conditions where the description file is modified by @@ -27,8 +28,6 @@ # Constants -Modules_Folder = 'yotta_modules' -Targets_Folder = 'yotta_targets' Component_Description_File = 'module.json' Component_Description_File_Fallback = 'package.json' Component_Definitions_File = 'defines.json' @@ -532,10 +531,10 @@ def getDependenciesRecursive(self, return components def modulesPath(self): - return os.path.join(self.path, Modules_Folder) + return os.path.join(self.path, paths.Modules_Folder) def targetsPath(self): - return os.path.join(self.path, Targets_Folder) + return os.path.join(self.path, paths.Targets_Folder) def satisfyDependenciesRecursive( self, diff --git a/yotta/lib/folders.py b/yotta/lib/folders.py index ea26876f..bc58be4c 100644 --- a/yotta/lib/folders.py +++ b/yotta/lib/folders.py @@ -10,6 +10,8 @@ # standard library modules, , , import os import sys +from yotta.lib import paths + def prefix(): if 'YOTTA_PREFIX' in os.environ: @@ -30,15 +32,15 @@ def userSettingsDirectory(): def globalInstallDirectory(): if os.name == 'nt': - return os.path.join(prefix(), 'Lib', 'yotta_modules') + return os.path.join(prefix(), 'Lib', paths.Modules_Folder) else: - return os.path.join(prefix(), 'lib', 'yotta_modules') + return os.path.join(prefix(), 'lib', paths.Modules_Folder) def globalTargetInstallDirectory(): if os.name == 'nt': - return os.path.join(prefix(), 'Lib', 'yotta_targets') + return os.path.join(prefix(), 'Lib', paths.Targets_Folder) else: - return os.path.join(prefix(), 'lib', 'yotta_targets') + return os.path.join(prefix(), 'lib', paths.Targets_Folder) def cacheDirectory(): return os.path.join(userSettingsDirectory(), 'cache') diff --git a/yotta/lib/paths.py b/yotta/lib/paths.py new file mode 100644 index 00000000..a61a4184 --- /dev/null +++ b/yotta/lib/paths.py @@ -0,0 +1,11 @@ +# Copyright 2014-2017 ARM Limited +# +# Licensed under the Apache License, Version 2.0 +# See LICENSE file for details. + +# this module to provide tools for path manipulation and caching, avoiding circular imports +DEFAULT_EXPORT_BUILD_RELPATH = 'export' +DEFAULT_BUILD_DIR = 'build' +BUILT_MODULES_DIR = 'ym' +Modules_Folder = 'yotta_modules' +Targets_Folder = 'yotta_targets' diff --git a/yotta/lib/target.py b/yotta/lib/target.py index 7a6b8249..a1c8cb08 100644 --- a/yotta/lib/target.py +++ b/yotta/lib/target.py @@ -470,7 +470,7 @@ def exec_helper(self, cmd, builddir): else: return '%s is not installed' % (cmd[0]) else: - return 'command %s failed' % (cmd) + return 'command execution %s failed with %s' % (cmd, e) if child.returncode: return 'command %s failed' % (cmd) diff --git a/yotta/lib/templates/base_CMakeLists.txt b/yotta/lib/templates/base_CMakeLists.txt index 6a53a41d..8bfb9c68 100644 --- a/yotta/lib/templates/base_CMakeLists.txt +++ b/yotta/lib/templates/base_CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! {% if toplevel %} cmake_minimum_required(VERSION 2.8.11) @@ -17,7 +17,10 @@ add_custom_target(all_tests) cmake_policy(SET CMP0017 OLD) # toolchain file for {{ target_name }} -set(CMAKE_TOOLCHAIN_FILE "{{ toolchain_file | replaceBackslashes }}") +set(CMAKE_TOOLCHAIN_FILE "{{ toolchain_file | relative | replaceBackslashes }}") + +# template replacement +configure_file( yotta_config.h.cmake_template yotta_config.h ) # provide function for post-processing executables function (yotta_postprocess_target target_type_ target_name_) @@ -54,30 +57,43 @@ if(NOT DEFINED YOTTA_FORCE_INCLUDE_FLAG) set(YOTTA_FORCE_INCLUDE_FLAG "-include") endif() endif() -add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"{{ config_include_file | replaceBackslashes }}\"") +add_definitions("${YOTTA_FORCE_INCLUDE_FLAG} \"${CMAKE_BINARY_DIR}/{{ config_include_file | relative | replaceBackslashes }}\"") {% endif %} # include root directories of all components we depend on (directly and # indirectly, including ourself) -{{ include_root_dirs }} +{% for dir in include_root_dirs %} +include_directories("${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes }}") +{% endfor %} +{% if add_depend_subdirs %} # recurse into dependencies that aren't built elsewhere -{{ add_depend_subdirs }} +{% for source_dir, binary_dir in add_depend_subdirs %} +add_subdirectory( + "${CMAKE_BINARY_DIR}/{{ source_dir | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ binary_dir | relative | replaceBackslashes }}" +) +{% endfor %} +{% endif %} {% if include_sys_dirs %} # Some components (I'm looking at you, libc), need to export system header # files with no prefix, these directories are listed in the component # description files: -{{ include_sys_dirs }} -{% endif %} +{% for dir in include_sys_dirs %} +include_directories(SYSTEM "${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes }}") +{% endfor %} +{% endif %} {% if include_other_dirs %} # And others (typically CMSIS implementations) need to export non-system header # files. Please don't use this facility. Please. It's much, much better to fix # implementations that import these headers to import them using the full path. -{{ include_other_dirs }} -{% endif %} +{% for dir in include_other_dirs %} +include_directories("${CMAKE_BINARY_DIR}/{{ dir | relative | replaceBackslashes }}") +{% endfor %} +{% endif %} # modules with custom CMake build systems may append to the # YOTTA_GLOBAL_INCLUDE_DIRS property to add compile-time-determined include # directories: @@ -100,8 +116,8 @@ set(YOTTA_MODULE_NAME {{ component.getName() }}) {% if delegate_to %} # delegate to an existing CMakeLists.txt: add_subdirectory( - "{{ delegate_to | replaceBackslashes }}" - "{{ delegate_build_dir | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ delegate_to | relative | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ delegate_build_dir | relative | replaceBackslashes }}" ) {% else %} # recurse into subdirectories for this component, using the two-argument @@ -109,8 +125,8 @@ add_subdirectory( # tree, not the working directory {% for srcdir, workingdir in add_own_subdirs %} add_subdirectory( - "{{ srcdir | replaceBackslashes }}" - "${CMAKE_BINARY_DIR}/{{ relpath | replaceBackslashes }}/{{ workingdir | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ srcdir | relative | replaceBackslashes }}" + "${CMAKE_CURRENT_BINARY_DIR}/{{ workingdir | replaceBackslashes }}" ) {% endfor %} {% endif %} @@ -127,6 +143,6 @@ target_compile_definitions({{ component.getName() }} PRIVATE "-DYOTTA_MODULE_NAM {% if cmake_includes %} # include .cmake files provided by the target: {% for f in cmake_includes %} -include("{{ f | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ f | relative | replaceBackslashes }}") {% endfor %} {% endif %} diff --git a/yotta/lib/templates/dummy_CMakeLists.txt b/yotta/lib/templates/dummy_CMakeLists.txt index 1c703fd3..06f98e53 100644 --- a/yotta/lib/templates/dummy_CMakeLists.txt +++ b/yotta/lib/templates/dummy_CMakeLists.txt @@ -1,4 +1,4 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! add_library({{ libname }} {{ cfile_name | replaceBackslashes }}) @@ -7,5 +7,5 @@ target_link_libraries({{ libname }} ) {% for include in cmake_files %} -include("{{ include | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ include | relative | replaceBackslashes }}") {% endfor %} diff --git a/yotta/lib/templates/subdir_CMakeLists.txt b/yotta/lib/templates/subdir_CMakeLists.txt index 9bde83a1..ebfed0c8 100644 --- a/yotta/lib/templates/subdir_CMakeLists.txt +++ b/yotta/lib/templates/subdir_CMakeLists.txt @@ -1,8 +1,8 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! cmake_minimum_required(VERSION 2.8.11) -include_directories("{{ source_directory | replaceBackslashes }}") +include_directories("${CMAKE_BINARY_DIR}/{{ source_directory | relative | replaceBackslashes }}") {% if 's' in languages %} enable_language(ASM) @@ -12,18 +12,18 @@ enable_language(ASM) set(YOTTA_AUTO_{{ object_name.upper() }}_{{ lang | upper }}_FILES {% for file_name, language in source_files %} {% if language in lang %} - "{{ file_name | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ file_name | relative | replaceBackslashes }}" {% endif %} {% endfor %} ) # force dependency on the config header for {{ lang }} files, which CMake otherwise wouldn't track: -set_property(SOURCE ${YOTTA_AUTO_{{ object_name.upper() }}_{{ lang | upper }}_FILES} PROPERTY OBJECT_DEPENDS "{{ config_include_file | replaceBackslashes }}") +set_property(SOURCE ${YOTTA_AUTO_{{ object_name.upper() }}_{{ lang | upper }}_FILES} PROPERTY OBJECT_DEPENDS "${CMAKE_BINARY_DIR}/{{ config_include_file | relative | replaceBackslashes }}") {% endfor %} {% if resource_files %} set(YOTTA_AUTO_{{ object_name.upper() }}_RESOURCE_FILES {% for file_name in resource_files %} - "{{ file_name | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ file_name | relative | replaceBackslashes }}" {% endfor %} ) {% endif %} @@ -74,5 +74,5 @@ target_link_libraries({{ object_name }} ) {% for include in cmake_files %} -include("{{ include | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ include | relative | replaceBackslashes }}") {% endfor %} diff --git a/yotta/lib/templates/test_CMakeLists.txt b/yotta/lib/templates/test_CMakeLists.txt index 44732c7f..89776526 100644 --- a/yotta/lib/templates/test_CMakeLists.txt +++ b/yotta/lib/templates/test_CMakeLists.txt @@ -1,32 +1,31 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! -include_directories("{{ source_directory | replaceBackslashes }}") +include_directories("${CMAKE_BINARY_DIR}/{{ source_directory | relative | replaceBackslashes }}") # add include path definitions needed only for tests (from testDependencies): {% for component in test_dependencies %} - include_directories("{{ component.path | replaceBackslashes }}") + include_directories("${CMAKE_BINARY_DIR}/{{ component.path | relative | replaceBackslashes }}") {% endfor %} {% for component in test_dependencies %} {% for d in component.getExtraSysIncludes() %} - include_directories(SYSTEM "{{ pathJoin(component.path, d) | replaceBackslashes }}") + include_directories(SYSTEM "${CMAKE_BINARY_DIR}/{{ pathJoin(component.path, d) | relative | replaceBackslashes }}") {% endfor %} {% endfor %} {% for component in test_dependencies %} {% for d in component.getExtraIncludes() %} - include_directories("{{ pathJoin(component.path, d) | replaceBackslashes }}") + include_directories("${CMAKE_BINARY_DIR}/{{ pathJoin(component.path, d) | relative | replaceBackslashes }}") {% endfor %} {% endfor %} # define the tests themselves: - {% for file_names, object_name, languages in tests %} add_executable({{ object_name }} {{ 'EXCLUDE_FROM_ALL' if exclude_from_all else '' }} {% for filename in file_names %} - "{{ filename | replaceBackslashes }}" + "${CMAKE_BINARY_DIR}/{{ filename | relative | replaceBackslashes }}" {% endfor %} ) {% if 'objc' in languages %} @@ -45,5 +44,5 @@ add_dependencies(all_tests {{ object_name }}) {% endfor %} {% for include in cmake_files %} -include("{{ include | replaceBackslashes }}") +include("${CMAKE_BINARY_DIR}/{{ include | relative | replaceBackslashes }}") {% endfor %} diff --git a/yotta/lib/templates/toolchain.cmake b/yotta/lib/templates/toolchain.cmake index 8e9817cb..64a2757e 100644 --- a/yotta/lib/templates/toolchain.cmake +++ b/yotta/lib/templates/toolchain.cmake @@ -1,11 +1,15 @@ -# NOTE: This file is generated by yotta: changes will be overwritten! +# NOTE: This file is generated from {{ self }}: changes will be overwritten! if(YOTTA_META_TOOLCHAIN_FILE_INCLUDED) return() endif() set(YOTTA_META_TOOLCHAIN_FILE_INCLUDED 1) +# this is a poor attempt to resolve build issues where CMAKE generates its own temp directories +# --> within those directories, project-relative paths will not work ... unless we commit this atrocity: +string(REPLACE "/CMakeFiles/CMakeTmp" "" CMAKE_BINDIR_NO_NESTING "${CMAKE_BINARY_DIR}") + {% for toolchain_file in toolchain_files %} -include("{{ toolchain_file | replaceBackslashes }}") +include("${CMAKE_BINDIR_NO_NESTING}/{{ toolchain_file | relative | replaceBackslashes }}") {% endfor %} diff --git a/yotta/link.py b/yotta/link.py index d13263df..60332266 100644 --- a/yotta/link.py +++ b/yotta/link.py @@ -3,6 +3,9 @@ # Licensed under the Apache License, Version 2.0 # See LICENSE file for details. +from yotta.lib import paths + + def addOptions(parser): parser.add_argument('module_or_path', default=None, nargs='?', help='Link a globally installed (or globally linked) module into '+ @@ -67,9 +70,9 @@ def execCommand(args, following_args): logging.error("%s is neither a valid module name, nor a path to an existing module.", args.module_or_path) logging.error(err) return 1 - fsutils.mkDirP(os.path.join(os.getcwd(), 'yotta_modules')) + fsutils.mkDirP(os.path.join(os.getcwd(), paths.Modules_Folder)) src = os.path.join(folders.globalInstallDirectory(), link_module_name) - dst = os.path.join(os.getcwd(), 'yotta_modules', link_module_name) + dst = os.path.join(os.getcwd(), paths.Modules_Folder, link_module_name) # if the component is already installed, rm it fsutils.rmRf(dst) else: diff --git a/yotta/link_target.py b/yotta/link_target.py index b704b1a6..cdc24c70 100644 --- a/yotta/link_target.py +++ b/yotta/link_target.py @@ -3,6 +3,9 @@ # Licensed under the Apache License, Version 2.0 # See LICENSE file for details. +from yotta.lib import paths + + def addOptions(parser): parser.add_argument('target_or_path', default=None, nargs='?', help='Link a globally installed (or globally linked) target into '+ @@ -73,9 +76,9 @@ def execCommand(args, following_args): else: logging.error(err) return 1 - fsutils.mkDirP(os.path.join(os.getcwd(), 'yotta_targets')) + fsutils.mkDirP(os.path.join(os.getcwd(), paths.Targets_Folder)) src = os.path.join(folders.globalTargetInstallDirectory(), link_target_name) - dst = os.path.join(os.getcwd(), 'yotta_targets', link_target_name) + dst = os.path.join(os.getcwd(), paths.Targets_Folder, link_target_name) # if the target is already installed, rm it fsutils.rmRf(dst) else: diff --git a/yotta/main.py b/yotta/main.py index 9bb56fed..5a14540d 100644 --- a/yotta/main.py +++ b/yotta/main.py @@ -246,7 +246,9 @@ def onParserAdded(parser): logging.warning('interrupted') status = -1 except Exception as e: + import traceback logging.error(e) + logging.debug(traceback.format_exc()) status = -1 sys.exit(status or 0) diff --git a/yotta/options/config.py b/yotta/options/config.py index c8e3ab28..3d79b86c 100644 --- a/yotta/options/config.py +++ b/yotta/options/config.py @@ -26,7 +26,7 @@ def __call__(self, parser, namespace, values, option_string=None): def addTo(parser): parser.add_argument( - '--config', default=None, dest='config', help= + '-c', '--config', default=None, dest='config', help= "Specify the path to a JSON configuration file to extend the build "+ "configuration provided by the target. This is most useful for "+ "ensuring test coverage of the ways that different targets will "+ diff --git a/yotta/start.py b/yotta/start.py index a12e1d6d..c7ae50f7 100644 --- a/yotta/start.py +++ b/yotta/start.py @@ -9,6 +9,7 @@ # validate, , validate things, internal from yotta.lib import validate +# settings, , load and save settings, internal # --config option, , , internal from yotta import options diff --git a/yotta/test/cli/cli.py b/yotta/test/cli/cli.py index 541cedb1..0fd632bc 100644 --- a/yotta/test/cli/cli.py +++ b/yotta/test/cli/cli.py @@ -10,7 +10,7 @@ import os def run(arguments, cwd='.'): - yottadir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..') + yottadir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) runyotta = [ sys.executable, '-c', @@ -24,11 +24,9 @@ def run(arguments, cwd='.'): stdin = subprocess.PIPE ) out, err = child.communicate() + # no command should ever produce a traceback: if 'traceback' in (out.decode('utf-8')+err.decode('utf-8')).lower(): print(out+err) assert(False) return out.decode('utf-8'), err.decode('utf-8'), child.returncode - - - diff --git a/yotta/test/cli/test_build.py b/yotta/test/cli/test_build.py index fe80c8d2..2cc15459 100644 --- a/yotta/test/cli/test_build.py +++ b/yotta/test/cli/test_build.py @@ -5,15 +5,18 @@ # See LICENSE file for details. # standard library modules, , , +import os import unittest -import subprocess import copy import re import datetime +import subprocess + # internal modules: from yotta.test.cli import cli from yotta.test.cli import util +from yotta.lib import paths Test_Complex = { 'module.json': '''{ @@ -48,17 +51,6 @@ } ''', -'source/a.c': ''' -#include "a/a.h" -#include "b/b.h" -#include "c/c.h" -#include "d/d.h" - -int a(){ - return 1 + b() + c() + d(); // 35 -} -''', - 'a/a.h':''' #ifndef __A_H__ #define __A_H__ @@ -663,7 +655,6 @@ def test_Defines_Application(self): self.assertIn("1234 yotta", output) util.rmRf(test_dir) - @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") def test_Defines_Library(self): test_dir = util.writeTestFiles(Test_Defines_Library) @@ -687,6 +678,87 @@ def test_Toplevel_Library(self): self.assertIn("42", output) util.rmRf(test_dir) + @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") + def test_exportIsClean(self): + """ + exported directory doesn't contain old references to yotta + :return: + """ + test_dir = util.writeTestFiles(util.Test_Complex_Lib, True) + export_dir = util.writeTestFiles({}, True) + + # export a generate-only output to an alternate directory + stdout = self.runCheckCommand(['--target', util.nativeTarget(), 'build', '-x', export_dir, '-g'], test_dir) + self.assertFalse(os.path.exists(os.path.join(export_dir, paths.Modules_Folder))) + self.assertFalse(os.path.exists(os.path.join(export_dir, paths.Targets_Folder))) + self.assertFalse(os.path.exists(os.path.join( + export_dir, paths.DEFAULT_EXPORT_BUILD_RELPATH, paths.BUILT_MODULES_DIR + ))) + + # duplicate the new strings (for brittleness?) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'modules'))) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'targets'))) + self.assertTrue(os.path.exists(os.path.join( + export_dir, paths.DEFAULT_EXPORT_BUILD_RELPATH, 'modules' + ))) + + @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") + def test_exportSpaceInPath(self): + """ + export should dump all the CMake files into a specified directory + the user should then be able to do their builds in that directory + :return: + """ + test_dir = util.writeTestFiles(util.Test_Trivial_Exe, True) + export_dir = util.writeTestFiles({}, True) + + # export a generate-only output to an alternate directory + stdout = self.runCheckCommand(['--target', util.nativeTarget(), 'build', '-x', export_dir, '-g'], test_dir) + + # search for any reference to the original directory + # /tmp/tmpUo7pB0 spaces in path --> tmpUo7pB0 + try: + # grep; skip sockets, recursive, phrase, directory + output = subprocess.check_output( + ['grep', '-D', 'skip', '-r', "'%s'" % os.path.basename(test_dir).split(' ', 1)[0], '.'], + cwd=export_dir + ).decode() + except subprocess.CalledProcessError as e: + # grep returns 1 for 'nothing found' and 2 for 'error' + # https://www.gnu.org/software/grep/manual/grep.html#Exit-Status + self.assertEqual(e.returncode, 1) + else: + raise Exception('should not have found anything') + + # build the project + built_dir = os.path.join(export_dir, paths.DEFAULT_EXPORT_BUILD_RELPATH) + try: + output = subprocess.check_output( + ['cmake', '-G', 'Ninja'], + cwd=built_dir + ).decode() + except subprocess.CalledProcessError as e: + print(e.output) + raise + + # link the project + output = subprocess.check_output( + ['ninja'], + cwd=built_dir + ).decode() + + # run the project + output = subprocess.check_output( + ['./test-trivial-exe'], + cwd=os.path.join(built_dir, 'source') + ).decode() + + # it works! + self.assertIn('trivial-exe-running', output) + + util.rmRf(test_dir) + util.rmRf(export_dir) + def runCheckCommand(self, args, test_dir): stdout, stderr, statuscode = cli.run(args, cwd=test_dir) if statuscode != 0: diff --git a/yotta/test/cli/test_debug.py b/yotta/test/cli/test_debug.py index 2902393a..87c301cd 100644 --- a/yotta/test/cli/test_debug.py +++ b/yotta/test/cli/test_debug.py @@ -45,13 +45,14 @@ def _nopDebugTargetDescription(name): class TestCLIDebug(unittest.TestCase): @unittest.skipIf(not util.canBuildNatively(), "can't build natively on windows yet") def test_noop_debug(self): + target_name = 'debug-test-target' test_dir = util.writeTestFiles(util.Test_Trivial_Exe, True) - target_dir = os.path.realpath(os.path.join(test_dir, 'yotta_targets', 'debug-test-target')) - build_dir = os.path.realpath(os.path.join(test_dir, 'build', 'debug-test-target')) + target_dir = os.path.realpath(os.path.join(test_dir, 'yotta_targets', target_name)) + build_dir = os.path.realpath(os.path.join(test_dir, 'build', target_name)) - util.writeTestFiles(_nopDebugTargetDescription('debug-test-target'), test_dir=target_dir) - output = util.runCheckCommand(['--target', 'debug-test-target', 'build'], test_dir) - output = util.runCheckCommand(['--target', 'debug-test-target', 'debug'], test_dir) + util.writeTestFiles(_nopDebugTargetDescription(target_name), test_dir=target_dir) + output = util.runCheckCommand(['--target', target_name, 'build'], test_dir) + output = util.runCheckCommand(['--target', target_name, 'debug'], test_dir) json_output = output[:output.index(JSON_MARKER)] result = json.loads(json_output) diff --git a/yotta/test/cli/util.py b/yotta/test/cli/util.py index ec5056e0..d058f28f 100644 --- a/yotta/test/cli/util.py +++ b/yotta/test/cli/util.py @@ -278,12 +278,10 @@ def writeTestFiles(files, add_space_in_path=False, test_dir=None): ''' write a dictionary of filename:contents into a new temporary directory ''' if test_dir is None: - test_dir = tempfile.mkdtemp() - if add_space_in_path: - test_dir = test_dir + ' spaces in path' + test_dir = tempfile.mkdtemp(suffix=' spaces in path' if add_space_in_path else '') for path, contents in files.items(): - path_dir, file_name = os.path.split(path) + path_dir, file_name = os.path.split(path) path_dir = os.path.join(test_dir, path_dir) fsutils.mkDirP(path_dir) with open(os.path.join(path_dir, file_name), 'w') as f: diff --git a/yotta/test/test_test_subcommand.py b/yotta/test/test_test_subcommand.py index 397b737d..07cbfa34 100644 --- a/yotta/test/test_test_subcommand.py +++ b/yotta/test/test_test_subcommand.py @@ -8,18 +8,33 @@ # standard library modules, , , import unittest - # module to test: from yotta import test_subcommand +from yotta.lib import paths + class TestTestSubcommandModule(unittest.TestCase): def test_moduleFromDirname(self): - self.assertTrue(test_subcommand.moduleFromDirname('ym/b/ym/c/d', {'b':'b', 'c':'c'}, 'a') == 'c') - self.assertTrue(test_subcommand.moduleFromDirname('ym/b/q/c/d', {'b':'b', 'c':'c'}, 'a') == 'b') - self.assertTrue(test_subcommand.moduleFromDirname('z/b/q/c/d', {'b':'b', 'c':'c'}, 'a') == 'a') - self.assertTrue(test_subcommand.moduleFromDirname('ym/e/d', {'b':'b', 'c':'c'}, 'a') == 'a') - self.assertTrue(test_subcommand.moduleFromDirname('ym/e/d', {'b':'b', 'c':'c', 'e':'e'}, 'a') == 'e') - - # see also yotta/test/cli/test.py for cli-driven testing - - + # Fstrings would be handy here + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/b/{m}/c/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'c' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/b/q/c/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'b' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + 'z/b/q/c/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'a' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/e/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c'}, 'a'), + 'a' + ) + self.assertEqual(test_subcommand.moduleFromDirname( + '{m}/e/d'.format(m=paths.BUILT_MODULES_DIR), {'b': 'b', 'c': 'c', 'e': 'e'}, 'a'), + 'e' + ) + + # see also yotta/test/cli/test.py for cli-driven testing diff --git a/yotta/test_subcommand.py b/yotta/test_subcommand.py index a6b4bd8f..c1874fe8 100644 --- a/yotta/test_subcommand.py +++ b/yotta/test_subcommand.py @@ -18,6 +18,7 @@ from yotta import build # --config option, , , internal from yotta import options +from yotta.lib import paths def addOptions(parser): @@ -52,7 +53,7 @@ def findCTests(builddir, recurse_yotta_modules=False): add_test_re = re.compile('add_test\\(([^" ]*)\s*"(.*)"\\)', flags=re.IGNORECASE) for root, dirs, files in os.walk(builddir, topdown=True): if not recurse_yotta_modules: - dirs = [d for d in dirs if d != 'ym'] + dirs = [d for d in dirs if d != paths.BUILT_MODULES_DIR] if 'CTestTestfile.cmake' in files: with open(os.path.join(root, 'CTestTestfile.cmake'), 'r') as ctestf: dir_tests = [] @@ -82,7 +83,7 @@ def moduleFromDirname(build_subdir, all_modules, toplevel_module): modtop = True submod = False else: - if part == 'ym' and modtop: + if part == paths.BUILT_MODULES_DIR and modtop: submod = True else: submod = False