diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dcd511153ca1dd4a62da33fc86346682e4a6a19e..d2b2b6e7ce8720fbc0b22b1a81fbc5a282998d90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,60 +4,27 @@ variables: GIT_SUBMODULE_STRATEGY: none stages: - - build - - update_dependencies_cache - - test - - coverage + - big_stage - deploy ############################################################################### -# Build stage +# Do most tasks ############################################################################### -build: - stage: build +build_and_test: + stage: big_stage script: - - ./ci/build.bash - artifacts: - paths: - - /builds/batsim/batsched/build - -############################################################################### -# Dependencies cache stage -############################################################################### -update_dependencies_cache: - stage: update_dependencies_cache - script: - - ./ci/update-dependencies-cache.bash - -############################################################################### -# Test stage -############################################################################### -test_pinned_batsim: - stage: test - script: - - nix-shell ./ci -A test_pinned --command 'bash ./ci/run-tests.bash' - dependencies: - - build - artifacts: - paths: - - /builds/batsim/batsched/build - -test_dev_batsim: - stage: test - script: - - nix-shell ./ci -A test_dev --command 'bash ./ci/run-tests.bash' - dependencies: - - build - -############################################################################### -# Coverage stage -############################################################################### -coverage: - stage: coverage - script: - - nix-shell ./ci -A test_deps_pinned --command 'bash ./ci/analyze-coverage.bash' - dependencies: - - test_pinned_batsim + # Build batsched + - nix-shell --pure ./release.nix -A batsched --command ${CI_PROJECT_DIR}/ci/list-store-paths-for-cachix.bash | cachix push batsim + - nix-build ./release.nix -A batsched + # Test against pinned batsim + - nix-shell --pure ./release.nix -A integration_tests --command ${CI_PROJECT_DIR}/ci/list-store-paths-for-cachix.bash | cachix push batsim + - nix-build ./release.nix -A integration_tests && cp -rL result ./integration_tests + # Test against up-to-date batsim + - nix-shell --pure ./release.nix -A integration_tests_batlatest --command ${CI_PROJECT_DIR}/ci/list-store-paths-for-cachix.bash | cachix push batsim + - nix-build ./release.nix -A integration_tests_batlatest && cp -rL result ./integration_tests_batlatest + # Fail job if tests failed + - if [[ "$(cat ./integration_tests/pytest_returncode)" -ne 0 ]] ; then echo "pytest returned non-zero (against pinned batsim), aborting" ; exit 1 ; fi + - if [[ "$(cat ./integration_tests_batlatest/pytest_returncode)" -ne 0 ]] ; then echo "pytest returned non-zero (against latest batsim), aborting" ; exit 1 ; fi artifacts: paths: - /builds/batsim/batsched/cover @@ -68,6 +35,8 @@ coverage: deploy_coverage: stage: deploy script: + - echo "Disabled for now." + - exit 1 # Pushes Batsim's code doc (doxygen) onto the gforge website. # SSH setup (do NOT run these commands on your machine) - eval $(ssh-agent -s) @@ -77,13 +46,7 @@ deploy_coverage: # Finally push the code documentation on the gforge website - rsync -rlgoDz --delete cover/html/ mpoquet@scm.gforge.inria.fr:/home/groups/batsim/htdocs/batsched/coverage dependencies: - - coverage - only: - - master - -deploy_batsched_dev_cachix: - stage: deploy - script: - - ./ci/update-batsched_dev-cache.bash + - build_and_test only: - master + allow_failure: true diff --git a/ci/list-store-paths-for-cachix.bash b/ci/list-store-paths-for-cachix.bash new file mode 100755 index 0000000000000000000000000000000000000000..399c132a7ce7aad1e89a139de2d70c37edbf0d84 --- /dev/null +++ b/ci/list-store-paths-for-cachix.bash @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +function list_store_paths_for_cachix { + var_path=$1 + echo $var_path | tr ':' '\n' | sed -E -n 'sW(/nix/store/.*)/.*W\1Wp' +} + +list_store_paths_for_cachix ${CMAKE_INCLUDE_PATH} +list_store_paths_for_cachix ${CMAKE_LIBRARY_PATH} +list_store_paths_for_cachix ${PATH} +list_store_paths_for_cachix ${PYTHONPATH} diff --git a/release.nix b/release.nix new file mode 100644 index 0000000000000000000000000000000000000000..20213b906f4df5bf478eed1be2a3f8c67ae6e879 --- /dev/null +++ b/release.nix @@ -0,0 +1,131 @@ +{ kapack ? import + (fetchTarball "https://github.com/oar-team/kapack/archive/master.tar.gz") + {} +, doCheck ? false +, simgrid ? kapack.simgrid322_2 +, batsim ? kapack.batsim +, batsim_dev ? kapack.batsim_dev +, batexpe ? kapack.batexpe +}: + +let + pkgs = kapack.pkgs; + pythonPackages = pkgs.python37Packages; + buildPythonPackage = pythonPackages.buildPythonPackage; + + jobs = rec { + # Batsim executable binary file. + batsched = kapack.batsched.overrideAttrs (attr: rec { + src = pkgs.lib.sourceByRegex ./. [ + "^src" + "^src/.*\.?pp" + "^src/algo" + "^src/algo/.*\.?pp" + "^src/external" + "^src/external/.*\.?pp" + "^CMakeLists.txt" + "^cmake" + "^cmake/Modules" + "^cmake/Modules/.*\.cmake" + "^cmake/Modules/.*\.cmake\.in" + ]; + # Debug build, without any Nix stripping magic. + mesonBuildType = "debug"; + hardeningDisable = [ "all" ]; + dontStrip = true; + }); + + # Batsim integration tests. + integration_tests = pkgs.stdenv.mkDerivation rec { + name = "batsched-integration-tests"; + src = pkgs.lib.sourceByRegex ./. [ + "^test" + "^test/.*\.py" + "^test/platforms" + "^test/platforms/.*\.xml" + "^test/workloads" + "^test/workloads/.*\.json" + ]; + buildInputs = with pkgs.python37Packages; [ + batsim batsched batexpe pkgs.redis + pytest pytest_html pandas]; + buildPhase = '' + set +e + (cd test && pytest -ra --html=../report/pytest_report.html) + echo $? > ./pytest_returncode + set -e + ''; + checkPhase = '' + pytest_return_code=$(cat ./pytest_returncode) + echo "pytest return code: $pytest_return_code" + if [ $pytest_return_code -ne 0 ] ; then + exit 1 + fi + ''; + inherit doCheck; + installPhase = '' + mkdir -p $out + mv ./report/* ./pytest_returncode $out/ + ''; + }; + integration_tests_batlatest = integration_tests.overrideAttrs (attr: rec { + buildInputs = with pkgs.python37Packages; [ + batsim_dev batsched batexpe pkgs.redis + pytest pytest_html pandas]; + }); + + # Batsim doxygen documentation. + doxydoc = pkgs.stdenv.mkDerivation rec { + name = "batsim-doxygen-documentation"; + src = pkgs.lib.sourceByRegex ./. [ + "^src" + "^src/.*\.?pp" + "^doc" + "^doc/Doxyfile" + "^doc/doxygen_mainpage.md" + ]; + buildInputs = [pkgs.doxygen]; + buildPhase = "(cd doc && doxygen)"; + installPhase = '' + mkdir -p $out + mv doc/doxygen_doc/html/* $out/ + ''; + checkPhase = '' + nb_warnings=$(cat doc/doxygen_warnings.log | wc -l) + if [[ $nb_warnings -gt 0 ]] ; then + echo "FAILURE: There are doxygen warnings!" + cat doc/doxygen_warnings.log + exit 1 + fi + ''; + doCheck = true; + }; + + # Dependencies not in nixpkgs as I write these lines. + pytest_metadata = buildPythonPackage { + name = "pytest-metadata-1.8.0"; + doCheck = false; + propagatedBuildInputs = [ + pythonPackages.pytest + ]; + src = builtins.fetchurl { + url = "https://files.pythonhosted.org/packages/12/38/eed3a1e00c765e4da61e4e833de41c3458cef5d18e819d09f0f160682993/pytest-metadata-1.8.0.tar.gz"; + sha256 = "1fk6icip2x1nh4kzhbc8cnqrs77avpqvj7ny3xadfh6yhn9aaw90"; + }; + }; + + pytest_html = buildPythonPackage { + name = "pytest-html-1.20.0"; + doCheck = false; + propagatedBuildInputs = [ + pythonPackages.pytest + pytest_metadata + ]; + src = builtins.fetchurl { + url = "https://files.pythonhosted.org/packages/08/3e/63d998f26c7846d3dac6da152d1b93db3670538c5e2fe18b88690c1f52a7/pytest-html-1.20.0.tar.gz"; + sha256 = "17jyn4czkihrs225nkpj0h113hc03y0cl07myb70jkaykpfmrim7"; + }; + }; + }; +in + jobs