Merge tag 'v1.4.0' into HEAD

Version 1.4.0 Added - New `fcfs` algorithm (copied from `fcfs_fast`) that takes into account the resources selector given in parameter. Fixed - The `easy_bf_fast` did not try to backfill previously submitted jobs in many events (when the priority job could not be executed).

Merge tag 'v1.4.0' into HEAD
eb8e0af9 · Millian Poquet · 343e3e31 · b020e48b · eb8e0af9 · eb8e0af9
Commit eb8e0af9 authored 5 years ago by Millian Poquet
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ build
 test-out
 test-instances
 cover
+result
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,86 +4,33 @@ variables:
  GIT_SUBMODULE_STRATEGY: none

 stages:
-  - build
-  - update_dependencies_cache
-  - test
-  - coverage
+  - big_stage
  - deploy

 ###############################################################################
-# Build stage
+# Do most tasks
 ###############################################################################
-build:
-  stage: build
+build_and_test:
+  stage: big_stage
  script:
-    - ./ci/build.bash
+    # Build batsched
+    - nix-shell --pure ./release.nix -A batsched --command ${CI_PROJECT_DIR}/ci/list-store-paths-for-cachix.bash | cachix push batsim
+    - nix-build ./release.nix -A batsched && cp -rL result ./batsched
+    # Test against pinned batsim
+    - nix-shell --pure ./release.nix -A integration_tests --command ${CI_PROJECT_DIR}/ci/list-store-paths-for-cachix.bash | cachix push batsim
+    - nix-build ./release.nix -A integration_tests && cp -rL result ./integration_tests
+    # Test against up-to-date batsim
+    - nix-shell --pure ./release.nix -A integration_tests_batlatest --command ${CI_PROJECT_DIR}/ci/list-store-paths-for-cachix.bash | cachix push batsim
+    - nix-build ./release.nix -A integration_tests_batlatest && rm result
+    # Fail job if tests failed
+    - if [[ "$(cat ./integration_tests/pytest_returncode)" -ne 0 ]] ; then echo "pytest returned non-zero (against pinned batsim), aborting" ; exit 1 ; fi
+    - if [[ "$(cat ./integration_tests_batlatest/pytest_returncode)" -ne 0 ]] ; then echo "pytest returned non-zero (against latest batsim), aborting" ; exit 1 ; fi
+    # Send coverage results to codecov.io
+    - nix-env -i gcc
+    - mkdir -p merged
+    - cp ./batsched/gcno/* ./integration_tests/gcda/* merged/
+    - bash <(curl -s https://codecov.io/bash)
  artifacts:
+    when: always
    paths:
-      - /builds/batsim/batsched/build
-
-###############################################################################
-# Dependencies cache stage
-###############################################################################
-update_dependencies_cache:
-  stage: update_dependencies_cache
-  script:
-    - ./ci/update-dependencies-cache.bash
-
-###############################################################################
-# Test stage
-###############################################################################
-test_pinned_batsim:
-  stage: test
-  script:
-    - nix-shell ./ci -A test_pinned --command 'bash ./ci/run-tests.bash'
-  dependencies:
-    - build
-  artifacts:
-    paths:
-      - /builds/batsim/batsched/build
-
-test_dev_batsim:
-  stage: test
-  script:
-    - nix-shell ./ci -A test_dev --command 'bash ./ci/run-tests.bash'
-  dependencies:
-    - build
-
-###############################################################################
-# Coverage stage
-###############################################################################
-coverage:
-  stage: coverage
-  script:
-    - nix-shell ./ci -A test_deps_pinned --command 'bash ./ci/analyze-coverage.bash'
-  dependencies:
-    - test_pinned_batsim
-  artifacts:
-    paths:
-      - /builds/batsim/batsched/cover
-
-###############################################################################
-# Deploy stage
-###############################################################################
-deploy_coverage:
-  stage: deploy
-  script:
-      # Pushes Batsim's code doc (doxygen) onto the gforge website.
-      # SSH setup (do NOT run these commands on your machine)
-      - eval $(ssh-agent -s)
-      - ssh-add <(echo "$SSH_PRIVATE_KEY")
-      - mkdir -p ~/.ssh
-      - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-      # Finally push the code documentation on the gforge website
-      - rsync -rlgoDz --delete cover/html/ mpoquet@scm.gforge.inria.fr:/home/groups/batsim/htdocs/batsched/coverage
-  dependencies:
-    - coverage
-  only:
-    - master
-
-deploy_batsched_dev_cachix:
-  stage: deploy
-  script:
-    - ./ci/update-batsched_dev-cache.bash
-  only:
-    - master
+      - /builds/batsim/batsched/integration_tests
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,16 @@ Batsched adheres to [Semantic Versioning][semver] and its public API is the foll
 [//]: =========================================================================
 ## [Unreleased]

+[//]: =========================================================================
+## [1.4.0] - 2020-07-29 - For [Batsim v4.0.0][Batsim v4.0.0]
+### Added
+- New `fcfs` algorithm (copied from `fcfs_fast`) that takes into account
+  the resources selector given in parameter.
+
+### Fixed
+- The `easy_bf_fast` did not try to backfill previously submitted jobs in many
+  events (when the priority job could not be executed).
+
 [//]: =========================================================================
 ## [1.3.0] - 2019-01-15 - For [Batsim v3.0.0][Batsim v3.0.0]
 ### Added
@@ -87,8 +97,10 @@ Initial release.

 [Batsim v2.0.0]: https://batsim.readthedocs.io/en/latest/changelog.html#v2-0-0
 [Batsim v3.0.0]: https://batsim.readthedocs.io/en/latest/changelog.html#v3-0-0
+[Batsim v4.0.0]: https://batsim.readthedocs.io/en/latest/changelog.html#v4-0-0

-[Unreleased]: https://gitlab.inria.fr/batsim/batsched/compare/v1.3.0...master
+[Unreleased]: https://gitlab.inria.fr/batsim/batsched/compare/v1.4.0...master
+[1.4.0]: https://gitlab.inria.fr/batsim/batsched/compare/v1.3.0...v1.4.0
 [1.3.0]: https://gitlab.inria.fr/batsim/batsched/compare/v1.2.1...v1.3.0
 [1.2.1]: https://gitlab.inria.fr/batsim/batsched/compare/v1.2.0...v1.2.1
 [1.2.0]: https://gitlab.inria.fr/batsim/batsched/compare/v1.1.0...v1.2.0

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -122,7 +122,7 @@ endif()
 ####################
 # Batsched version #
 ####################
-set(default_batsched_version "v1.3.0")
+set(default_batsched_version "v1.4.0")
 include(GetGitRevisionDescription)
 git_describe(batsched_version)
 message(STATUS "Batsched version from git: ${batsched_version}")

--- a/README.md
+++ b/README.md
 [![pipeline status](https://framagit.org/batsim/batsched/badges/master/pipeline.svg)](https://framagit.org/batsim/batsched/pipelines)
-[![coverage report](https://framagit.org/batsim/batsched/badges/master/coverage.svg)](http://batsim.gforge.inria.fr/batsched/coverage/)
+[![coverage](https://img.shields.io/codecov/c/github/oar-team/batsched.svg)](https://codecov.io/gh/oar-team/batsched)
 [![changelog](https://img.shields.io/badge/doc-changelog-blue.svg)](./CHANGELOG.md)

 **batsched** is a set of [Batsim]-compatible algorithms implemented in C++.
@@ -8,9 +8,9 @@
 ### For [Nix] users
 ``` bash
 # Up-to-date version
-nix-env -iA batsched_dev -f 'https://github.com/oar-team/kapack/archive/master.tar.gz'
+nix-env -iA batsched-master -f 'https://github.com/oar-team/nur-kapack/archive/master.tar.gz'
 # Latest release
-nix-env -iA batsched -f 'https://github.com/oar-team/kapack/archive/master.tar.gz'
+nix-env -iA batsched -f 'https://github.com/oar-team/nur-kapack/archive/master.tar.gz'
 ```

 ### Manually
@@ -23,14 +23,14 @@ make
 make install
 ```

-Up-to-date dependencies and versions are fully defined in [batsched's CI nix recipe](./default.nix).  
+Up-to-date dependencies and versions are fully defined in [batsched's CI nix recipe](./release.nix).  
 Here is a quick (and probably outdated) list:
 - decent clang/gcc and cmake
 - zmq (C and C++)
 - redox (hiredis + libev)
 - [loguru]
 - [intervalset]
- decent boost, gmp, rapidjson, openssl...
+- decent boost, gmp, rapidjson...

 [Batsim]: https://framagit.org/batsim/batsim/
 [intervalset]: https://framagit.org/batsim/intervalset

--- a/ci/README.md
+++ b/ci/README.md
-This directory is essentially a nix repository with some scripts.
-
-# Packages overview
-## batsched_local
-This is the current version of batsched (built from current file hierarchy).
-
-## batsim_pinned
-This is the current *reference* Batsim version for batsched.
-batsched should always work with this version.
-
-## batsim_dev
-This is the up-to-date Batsim version.
-batsched should work with this version.
-
-## test_deps_(pinned|dev)
-The list of packages needed to run tests.
-This is meant to be used as a shell, not meant to be installed.
-
-## test_(pinned|dev)
-A shell used to run tests. Essentially batsim_local + test_deps.
-Not meant to be installed either.
-
-# Useful commands
-In all the following examples, the current directory is expected to be
-batsched's root.
-
-## Building packages
-This can be done via `nix-build`. Result will be in `./result/`.
-Some examples:
-``` bash
-nix-build ./ci -A batsched_local
-nix-build ./ci -A batsim_pinned
-```
-
-## Install packages
-This is done via `nix-env`:
-``` bash
-nix-env -f ./ci -iA batsched_local
-nix-env -f ./ci -iA batsim_dev
-```
-
-To uninstall them, use `nix-env --uninstall`.
-
-## Get into a shell to build packages
-`nix-shell` is your friend here. Example:
-``` bash
-nix-shell ./ci -A batsched_local
-# your shell now has all batsched's build dependencies!
-# you can freely build the project (cmake, make...)
-```
-
-## Run the tests
-This is essentially "run the test script in the desired environment".
-``` bash
-# test current batsched with batsim_pinned:
-nix-shell ./ci -A test_pinned --command './ci/run-tests.bash'
-
-# or test with batsim_dev:
-nix-shell ./ci -A test_dev --command './ci/run-tests.bash'
-```
--- a/ci/analyze-coverage.bash
+++ b/ci/analyze-coverage.bash
-#!/usr/bin/env nix-shell
-#! nix-shell . -i bash -A test_deps_pinned
-set -eu
-
-echo "Prepare directories"
-rm -rf ./cover
-mkdir -p ./cover/tmp
-cd ./cover/tmp
-
-echo "Call gcov"
-gcov_files=$(find ../../build -name '*.gcda')
-for gcov_file in ${gcov_files[@]}; do
-    gcov ${gcov_file} 1>/dev/null 2>&1
-done
-
-echo "Only keep interesting files"
-interesting_sources=$(find ../../src -name '*.?pp' | sort | grep -v 'pempek_assert\|taywee_args')
-set +e
-for interesting_source in ${interesting_sources[@]}; do
-    interesting_file="./$(basename ${interesting_source}).gcov"
-    cp -f ${interesting_file} ../ 2>/dev/null
-done
-set -e
-
-cd ../..
-rm -rf ./cover/tmp
-
-echo "Run gcovr analysis (human-readable report)"
-gcovr -gk -o ./cover/summary.txt
-cat ./cover/summary.txt
-
-echo "Run gcovr analysis (html report)"
-rm -rf ./cover/html
-mkdir -p ./cover/html
-gcovr -gk --html-details -o ./cover/html/index.html
-
-exit 0
--- a/ci/build.bash
+++ b/ci/build.bash
-#!/usr/bin/env nix-shell
-#! nix-shell . -i bash -A batsched_local
-set -eux
-
-# Start from a clean build directory
-rm -rf ./build
-mkdir -p ./build
-
-# Usual cmake build stuff
-cd ./build
-cmake .. \
-    -DCMAKE_BUILD_TYPE=Debug \
-    -Denable_warnings=ON \
-    -Dtreat_warnings_as_errors=OFF \
-    -Ddo_coverage=ON
-make -j $(nproc)
--- a/ci/default.nix
+++ b/ci/default.nix
-let
-  pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/18.03.tar.gz") {};
-  kapack = import
-    ( fetchTarball "https://github.com/oar-team/kapack/archive/master.tar.gz") {};
-in
-
-let
-  callPackage = pkgs.lib.callPackageWith (pkgs // pkgs.xlibs // self // kapack);
-  self = rec {
-    inherit pkgs kapack;
-
-    # Redefine some packages for clarity's sake
-    batexpe = kapack.batexpe;
-    batsim_pinned = (kapack.batsim.override {simgrid = kapack.simgrid_dev_working; }).overrideAttrs (attrs: rec {
-      name = "batsim-${version}";
-      version = "3.0.0-pinned";
-      src = pkgs.fetchgit {
-        url = "https://framagit.org/batsim/batsim.git";
-        rev = "12db5085210ac24d82657b21fafe0ca198dcf48d";
-        sha256 = "07b9npm5qvrzanp14rwp743dxsh7dwpvpywmlpxla5j4kxk665hc";
-      };
-    });
-    batsim_dev = (kapack.batsim.override {simgrid = kapack.simgrid_dev_working; }).overrideAttrs (attrs: rec {
-      nativeBuildInputs = attrs.nativeBuildInputs ++ [kapack.intervalset];
-      name = "batsim-${version}";
-      version = "3.1.0-dev";
-      src = fetchTarball "https://gitlab.inria.fr/batsim/batsim/repository/master/archive.tar.gz";
-    });
-
-    pytest = pkgs.python36Packages.pytest;
-    gcovr = kapack.gcovr;
-
-    # Packages defined in this tree
-    batsched_local = callPackage ./local.nix {};
-    test_deps_pinned = callPackage ./test-deps.nix {
-      batsim = batsim_pinned;
-    };
-    test_deps_dev = callPackage ./test-deps.nix {
-      batsim = batsim_dev;
-    };
-
-    # Packages meant to be used as shells
-    test_pinned = callPackage ./test-env.nix {
-      batsched = batsched_local;
-      test_deps = test_deps_pinned;
-    };
-    test_dev = callPackage ./test-env.nix {
-      batsched = batsched_local;
-      test_deps = test_deps_dev;
-    };
-  };
-in
-  self
--- a/ci/list-store-paths-for-cachix.bash
+++ b/ci/list-store-paths-for-cachix.bash
+#!/usr/bin/env bash
+function list_store_paths_for_cachix {
+    var_path=$1
+    echo $var_path | tr ':' '\n' | sed -E -n 'sW(/nix/store/.*)/.*W\1Wp'
+}
+
+list_store_paths_for_cachix ${CMAKE_INCLUDE_PATH}
+list_store_paths_for_cachix ${CMAKE_LIBRARY_PATH}
+list_store_paths_for_cachix ${PATH}
+list_store_paths_for_cachix ${PYTHONPATH}
--- a/ci/local.nix
+++ b/ci/local.nix
-{ stdenv, batsched_dev }:
-
-(batsched_dev.override {}).overrideAttrs (attrs: rec {
-    name = "batsched-1.4.0-nix-ci";
-    src = stdenv.lib.sourceByRegex ../. [
-      "^src$"
-      "^src/algo$"
-      "^src/external$"
-        ".*\.cpp$" ".*\.hpp$"
-      "^cmake$"
-      "^cmake/Modules$"
-        ".*\.cmake"
-        ".*\.cmake.in"
-      "^CMakeLists\.txt$"
-    ];
-    enableParallelBuilding = true;
-    doCheck = false;
-
-    preConfigure = ''
-      # Always start from a clean build directory
-      rm -rf ./build
-    '';
-})
--- a/ci/pin-batsim.bash
+++ b/ci/pin-batsim.bash
-#!/usr/bin/env nix-shell
-#! nix-shell . -i bash -A test_deps_dev
-
-if [ "$#" -ne 1 ]; then
-    echo 'usage: pin-batsim.bash BATSIM-REV'
-    exit 1
-fi
-
-rev=$1
-nix-prefetch-git \
-    --url https://framagit.org/batsim/batsim.git \
-    --rev ${rev} \
-    > batsim-pinned.json
--- a/ci/run-tests.bash
+++ b/ci/run-tests.bash
-#!/usr/bin/env nix-shell
-#! nix-shell . -i bash -A test_deps_pinned
-set -eu
-
-initial_dir=$(realpath .)
-
-# Run a redis server if needed
-redis_launched_here=0
-r=$(ps faux | grep redis-server | grep -v grep | wc -l)
-if [ $r -eq 0 ]
-then
-    echo "Running a Redis server..."
-    redis-server>/dev/null &
-    redis_launched_here=1
-
-    while ! nc -z localhost 6379; do
-      sleep 1
-    done
-fi
-
-# Add built batsched in PATH
-export PATH=$(realpath ./build):${PATH}
-
-# Set TEST_ROOT so simulation input files can be found
-export TEST_ROOT=$(realpath ./test)
-
-# Print which versions are used
-echo "batsched realpath: $(realpath $(which batsched))"
-echo "batsim realpath: $(realpath $(which batsim))"
-echo "robin realpath: $(realpath $(which robin))"
-
-# Execute the tests (TODO: clean tests)
-cd test
-pytest
-failed=$?
-
-# Stop the redis server if it has been launched by this script
-if [ $redis_launched_here -eq 1 ]
-then
-    echo "Stopping the Redis server..."
-    killall redis-server
-fi
-
-cd ${initial_dir}
-exit ${failed}
--- a/ci/test-deps.nix
+++ b/ci/test-deps.nix
-{ stdenv, batsim, batexpe,
-  which, redis, procps, psmisc, pytest, gcovr,
-  nix-prefetch-git
-}:
-
-stdenv.mkDerivation rec {
-  name = "batsched-test-deps";
-
-  # This package is not meant to be built
-  unpackPhase = "true";
-  installPhase = "true";
-  propagatedBuildInputs = [ batsim batexpe
-    which redis procps psmisc pytest gcovr
-    nix-prefetch-git
-  ];
-}
--- a/ci/test-env.nix
+++ b/ci/test-env.nix
-{ stdenv, batsched, test_deps }:
-
-stdenv.mkDerivation rec {
-  name = "batsched-test-env";
-
-  # This package is not meant to be built
-  unpackPhase = "true";
-  installPhase = "true";
-  propagatedBuildInputs = [ batsched test_deps ];
-}
--- a/ci/update-batsched_dev-cache.bash
+++ b/ci/update-batsched_dev-cache.bash
-#!/usr/bin/env nix-shell
-#! nix-shell -i bash -p nix
-set -eu
-
-# Build up-to-date batsched_dev package, push it on binary cache
-nix-build https://github.com/oar-team/kapack/archive/master.tar.gz -A batsched_dev | cachix push batsim
--- a/ci/update-dependencies-cache.bash
+++ b/ci/update-dependencies-cache.bash
-#!/usr/bin/env nix-shell
-#! nix-shell -i bash -p nix
-set -eu
-
-# (re)build up-to-date CI batsched package + deps, push them on binary cache
-nix-build ./ci -A test_pinned | cachix push batsim
-nix-build ./ci -A test_dev | cachix push batsim
--- a/default.nix
+++ b/default.nix
-let
-  pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/18.03.tar.gz") {};
-  kapack = import
-    ( fetchTarball "https://github.com/oar-team/kapack/archive/master.tar.gz")
-  { inherit pkgs; };
-in
-
-with kapack;
-with pkgs;
-
-(batsched_dev.override {}).overrideAttrs (attrs: rec {
-    name = "batsched-1.4.0-nix-local";
-    src = ../.;
-    enableParallelBuilding = true;
-    doCheck = false;
-})
--- a/default.nix
+++ b/default.nix
+release.nix
\ No newline at end of file
--- a/meson.build
+++ b/meson.build
+project('batsched', 'cpp',
+    version: '1.4.0',
+    license: 'free',
+    default_options: ['cpp_std=c++11'],
+    meson_version: '>=0.40.0'
+)
+
+# Dependencies
+boost_dep = dependency('boost',
+    modules : ['locale', 'regex', 'system']
+)
+rapidjson_dep = dependency('RapidJSON')
+redox_dep = dependency('redox')
+libzmq_dep = dependency('libzmq')
+loguru_dep = dependency('loguru')
+intervalset_dep = dependency('intervalset')
+gmpxx_dep = dependency('gmpxx')
+
+batsched_deps = [
+    boost_dep,
+    rapidjson_dep,
+    redox_dep,
+    libzmq_dep,
+    loguru_dep,
+    intervalset_dep,
+    gmpxx_dep
+]
+
+# Source files
+src = [
+    'src/algo/conservative_bf.cpp',
+    'src/algo/conservative_bf.hpp',
+    'src/algo/crasher.cpp',
+    'src/algo/crasher.hpp',
+    'src/algo/easy_bf.cpp',
+    'src/algo/easy_bf_fast.cpp',
+    'src/algo/easy_bf_fast.hpp',
+    'src/algo/easy_bf.hpp',
+    'src/algo/easy_bf_plot_liquid_load_horizon.cpp',
+    'src/algo/easy_bf_plot_liquid_load_horizon.hpp',
+    'src/algo/energy_bf.cpp',
+    'src/algo/energy_bf_dicho.cpp',
+    'src/algo/energy_bf_dicho.hpp',
+    'src/algo/energy_bf.hpp',
+    'src/algo/energy_bf_idle_sleeper.cpp',
+    'src/algo/energy_bf_idle_sleeper.hpp',
+    'src/algo/energy_bf_machine_subpart_sleeper.cpp',
+    'src/algo/energy_bf_machine_subpart_sleeper.hpp',
+    'src/algo/energy_bf_monitoring_inertial_shutdown.cpp',
+    'src/algo/energy_bf_monitoring_inertial_shutdown.hpp',
+    'src/algo/energy_bf_monitoring_period.cpp',
+    'src/algo/energy_bf_monitoring_period.hpp',
+    'src/algo/energy_watcher.cpp',
+    'src/algo/energy_watcher.hpp',
+    'src/algo/fcfs_fast.cpp',
+    'src/algo/fcfs_fast.hpp',
+    'src/algo/fcfs.cpp',
+    'src/algo/fcfs.hpp',
+    'src/algo/filler.cpp',
+    'src/algo/filler.hpp',
+    'src/algo/killer2.cpp',
+    'src/algo/killer2.hpp',
+    'src/algo/killer.cpp',
+    'src/algo/killer.hpp',
+    'src/algo/random.cpp',
+    'src/algo/random.hpp',
+    'src/algo/rejecter.cpp',
+    'src/algo/rejecter.hpp',
+    'src/algo/sequencer.cpp',
+    'src/algo/sequencer.hpp',
+    'src/algo/sleeper.cpp',
+    'src/algo/sleeper.hpp',
+    'src/algo/submitter.cpp',
+    'src/algo/submitter.hpp',
+    'src/algo/wt_estimator.cpp',
+    'src/algo/wt_estimator.hpp',
+    'src/data_storage.cpp',
+    'src/data_storage.hpp',
+    'src/decision.cpp',
+    'src/decision.hpp',
+    'src/exact_numbers.hpp',
+    'src/external/taywee_args.hpp',
+    'src/isalgorithm.cpp',
+    'src/isalgorithm.hpp',
+    'src/json_workload.cpp',
+    'src/json_workload.hpp',
+    'src/locality.cpp',
+    'src/locality.hpp',
+    'src/main.cpp',
+    'src/network.cpp',
+    'src/network.hpp',
+    'src/pempek_assert.cpp',
+    'src/pempek_assert.hpp',
+    'src/protocol.cpp',
+    'src/protocol.hpp',
+    'src/queue.cpp',
+    'src/queue.hpp',
+    'src/queueing_theory_waiting_time_estimator.cpp',
+    'src/queueing_theory_waiting_time_estimator.hpp',
+    'src/schedule.cpp',
+    'src/schedule.hpp'
+]
+include_dir = include_directories('src')
+
+batsched = executable('batsched', src,
+    include_directories: include_dir,
+    dependencies: batsched_deps,
+    cpp_args: '-DBATSCHED_VERSION=@0@'.format(meson.project_version()),
+    install: true
+)