From aac7a87e63b9fad5ed783c96f71e2da462c29d5d Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 23 Jan 2023 07:13:14 -1000 Subject: [PATCH 01/18] Bandwidth Modifier --- CMakeLists.txt | 4 +++ src/BandwidthModifier.cpp | 68 +++++++++++++++++++++++++++++++++++++++ src/BandwidthModifier.h | 47 +++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 src/BandwidthModifier.cpp create mode 100644 src/BandwidthModifier.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f38e2f2..af38007b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,10 @@ include_directories(src/ ${SimGrid_INCLUDE_DIR}/include /usr/local/include /opt/ set(SOURCE_FILES src/WorkloadExecutionController.h src/WorkloadExecutionController.cpp + src/SimpleExecutionController.h + src/SimpleExecutionController.cpp + src/BandwidthModifier.h + src/BandwidthModifier.cpp src/SimpleSimulator.h src/SimpleSimulator.cpp src/JobSpecification.h diff --git a/src/BandwidthModifier.cpp b/src/BandwidthModifier.cpp new file mode 100644 index 00000000..0e85b4d9 --- /dev/null +++ b/src/BandwidthModifier.cpp @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2020. . + * Generated with the wrench-init.in tool. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ +#include +#include + +#include "BandwidthModifier.h" + +XBT_LOG_NEW_DEFAULT_CATEGORY(bandwidth_modifier, "Log category for BandwidthModifier"); + +/** + * @brief A "controller" that dynamically modifies a link's bandwidth + * + * @param hostname: the host on which this actor runs + * @param link_name: the name of the link whose bandwidth is to be modified + * @param period: the period in between two bandwidth modifications (in seconds) + * @param distribution: the normal distribution from which to same bandwidth modification values + */ +BandwidthModifier::BandwidthModifier( + const std::string &hostname, + const std::string &link_name, + double period, + std::normal_distribution* distribution, + unsigned long seed) : wrench::ExecutionController( + hostname, + "bandwidth-modifier"), link_name(link_name), period(period), distribution(distribution), seed(seed) { + +} + +/** + * @brief main method of the BandwidthModifier actor + * + * @return 0 on completion + * + * @throw std::runtime_error + */ +int BandwidthModifier::main() { + + wrench::TerminalOutput::setThisProcessLoggingColor(wrench::TerminalOutput::COLOR_GREEN); + + // Retrieve the link + auto the_link = simgrid::s4u::Link::by_name_or_null(this->link_name); + if (the_link == nullptr) { + throw std::invalid_argument("BandwidthModifier::main(): The platform does not contain a link named '" + this->link_name +"'"); + } + + // Get its (original) bandwidth + auto original_bandwidth = the_link->get_bandwidth(); + + // Create RNG + std::mt19937 rng(this->seed); + + while (true) { + // Sleep during the period + wrench::Simulation::sleep(period); + // Sample bandwidth modification factor + double factor = (*this->distribution)(rng); + // Update the link bandwidth + the_link->set_bandwidth(original_bandwidth * factor); + } +} + diff --git a/src/BandwidthModifier.h b/src/BandwidthModifier.h new file mode 100644 index 00000000..ef35f302 --- /dev/null +++ b/src/BandwidthModifier.h @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2020. . + * Generated with the wrench-init.in tool. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ +#ifndef MY_BANDWIDTH_MODIFIER_H +#define MY_BANDWIDTH_MODIFIER_H + +#include +#include +#include + +#include "JobSpecification.h" +#include "LRU_FileList.h" + +#include "util/Enums.h" + +class Simulation; + +class BandwidthModifier : public wrench::ExecutionController { +public: + // Constructor + BandwidthModifier( + const std::string &hostname, + const std::string &link_name, + double period, + std::normal_distribution* distribution, + unsigned long seed); + +private: + + std::string link_name; + double period; + std::normal_distribution* distribution; + std::mt19937 generator; + unsigned long seed; + + int main() override; + +}; + +#endif //MY_BANDWIDTH_MODIFIER_H + From bd64fa2a98675e6d4f58ad3370cf387192a661fa Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Wed, 19 Apr 2023 14:45:34 +0200 Subject: [PATCH 02/18] update to new main --- CMakeLists.txt | 2 -- src/BandwidthModifier.h | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index af38007b..336f4b5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,8 +22,6 @@ include_directories(src/ ${SimGrid_INCLUDE_DIR}/include /usr/local/include /opt/ set(SOURCE_FILES src/WorkloadExecutionController.h src/WorkloadExecutionController.cpp - src/SimpleExecutionController.h - src/SimpleExecutionController.cpp src/BandwidthModifier.h src/BandwidthModifier.cpp src/SimpleSimulator.h diff --git a/src/BandwidthModifier.h b/src/BandwidthModifier.h index ef35f302..d798a0eb 100644 --- a/src/BandwidthModifier.h +++ b/src/BandwidthModifier.h @@ -17,7 +17,7 @@ #include "JobSpecification.h" #include "LRU_FileList.h" -#include "util/Enums.h" +#include "util/Utils.h" class Simulation; From d4eb5326d5099835ee06242bd3b093656843ee54 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Wed, 19 Apr 2023 15:31:02 +0200 Subject: [PATCH 03/18] reduce bandwidth instead of multiplying factor --- src/BandwidthModifier.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/BandwidthModifier.cpp b/src/BandwidthModifier.cpp index 0e85b4d9..7f4c03d1 100644 --- a/src/BandwidthModifier.cpp +++ b/src/BandwidthModifier.cpp @@ -51,7 +51,7 @@ int BandwidthModifier::main() { } // Get its (original) bandwidth - auto original_bandwidth = the_link->get_bandwidth(); + const double original_bandwidth = the_link->get_bandwidth(); // Create RNG std::mt19937 rng(this->seed); @@ -59,10 +59,11 @@ int BandwidthModifier::main() { while (true) { // Sleep during the period wrench::Simulation::sleep(period); - // Sample bandwidth modification factor - double factor = (*this->distribution)(rng); + // Sample bandwidth modification subtrahend + double reduction = (*this->distribution)(rng); + while ((original_bandwidth - reduction < 0) || reduction < 0) reduction = (*this->distribution)(rng); // Update the link bandwidth - the_link->set_bandwidth(original_bandwidth * factor); + the_link->set_bandwidth(original_bandwidth - reduction); } } From 71867ab062d7257b47122f4fc3e036d6be0b44ac Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Wed, 19 Apr 2023 18:04:50 +0200 Subject: [PATCH 04/18] identify the links to vary --- src/SimpleSimulator.cpp | 24 ++++++++++++++++++++++++ src/SimpleSimulator.h | 3 +++ 2 files changed, 27 insertions(+) diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index 4289df06..fd9f9946 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -51,6 +51,7 @@ std::set SimpleSimulator::scheduler_hosts; std::set SimpleSimulator::executors; std::set SimpleSimulator::file_registries; std::set SimpleSimulator::network_monitors; +std::set SimpleSimulator::variable_links; std::map> SimpleSimulator::hosts_in_zones; bool SimpleSimulator::local_cache_scope = false; // flag to consider only local caches @@ -386,6 +387,29 @@ void SimpleSimulator::identifyHostTypes(std::shared_ptr simu } } + +/** + * @brief Identify variable links based on configured "variation" property tag + * + * @param simulation Simulation object with already instantiated hosts + * + * @throw std::runtime_error, std::invalid_argument + */ +void SimpleSimulator::identifyVariableLinks(std::shared_ptr simulation){ + std::vector linkname_list = simulation->getLinknameList(); + if (linkname_list.size() == 0) { + throw std::runtime_error("Empty linkname list! Have you instantiated the platform already?"); + } + for (const auto& linkname: linkname_list) { + std::string linkProperty = simgrid::s4u::Link::by_name_or_null(linkname)->get_property("variation"); + if (linkProperty == ""){ + throw std::runtime_error("Configuration property \"variation\" missing for link " + linkname); + } + SimpleSimulator::variable_links.insert(linkname); + } +} + + /** * @brief Method to be executed once at simulation start, * which finds all hosts in zone and all same level accopanying zones (siblings) diff --git a/src/SimpleSimulator.h b/src/SimpleSimulator.h index eb074e34..d0ab9f71 100644 --- a/src/SimpleSimulator.h +++ b/src/SimpleSimulator.h @@ -11,6 +11,7 @@ class SimpleSimulator { public: static void identifyHostTypes(std::shared_ptr simulation); + static void identifyVariableLinks(std::shared_ptr simulation); static std::set cache_hosts; // hosts configured to provide a cache static std::set storage_hosts; // hosts configured to provide GRID storage @@ -20,6 +21,8 @@ class SimpleSimulator { static std::set file_registries; // hosts configured to manage a file registry static std::set network_monitors; // hosts configured to monitor network + static std::set variable_links; // links configured to randomly vary their bandwidth + static void fillHostsInSiblingZonesMap(bool include_subzones); static bool local_cache_scope; From 83cb89a5b74d499d914d4e86096bf70cfe4523bc Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Wed, 19 Apr 2023 18:07:19 +0200 Subject: [PATCH 05/18] create an execution controller for each varied link --- src/SimpleSimulator.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index fd9f9946..100dc92a 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -11,6 +11,7 @@ #include "SimpleSimulator.h" #include "WorkloadExecutionController.h" #include "JobSpecification.h" +#include "BandwidthModifier.h" #include "util/Utils.h" @@ -19,6 +20,8 @@ #include #include +#include +#include #include #include @@ -770,6 +773,37 @@ int main(int argc, char **argv) { std::cerr << "The simulation now has " << std::to_string(num_total_jobs) << " jobs in total " << std::endl; + /* Identify the links which should be variied and add bandwidth modifiers to the simulation */ + std::cerr << "Setting varied link bandwidths ... " << "\n"; + std::set> bandwidth_modifiers; + SimpleSimulator::identifyVariableLinks(simulation); + for (auto varied_link: SimpleSimulator::variable_links) { + auto the_link = simgrid::s4u::Link::by_name_or_null(varied_link); + std::string property = the_link->get_property("variation"); + std::vector parameters; + boost::split(parameters, property, boost::is_any_of("\t, ")); + if (parameters.size() != 2) { + throw std::runtime_error( + "Property \"variation\":" + property + " for link " + varied_link + " misconfigured! Should only contain two splitable values!" + ); + } + double mu = std::stod(parameters.at(0)); + double sigma = std::stod(parameters.at(1)); + std::normal_distribution reduction(mu, sigma); + for (auto host: SimpleSimulator::executors) { + auto bm = simulation->add( + new BandwidthModifier( + host, + varied_link, + 3600., + &reduction, + 42 + ) + ); + } + } + + /* Launch the simulation */ try { /* initialize output-dump file */ From b5116b87fc1683b29df9eba0d868d9e5319a7893 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Thu, 20 Apr 2023 14:19:10 +0200 Subject: [PATCH 06/18] add print out --- src/SimpleSimulator.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index 100dc92a..de8c365f 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -800,6 +800,7 @@ int main(int argc, char **argv) { 42 ) ); + std::cerr << "\t Created a BandwidthModifier on host " << host << " for link " << varied_link << "\n"; } } From 2c1d2674c43369ba858a26bfddb84bab305cdb24 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Thu, 20 Apr 2023 14:59:20 +0200 Subject: [PATCH 07/18] use helper method for link properties --- src/SimpleSimulator.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index de8c365f..dfcaeb4f 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -390,6 +390,16 @@ void SimpleSimulator::identifyHostTypes(std::shared_ptr simu } } +std::string getLinkProperty(const std::string &linkname, const std::string &property_name) { + auto link = simgrid::s4u::Link::by_name_or_null(linkname); + if (link == nullptr) { + throw std::invalid_argument("getLinkProperty(): Unknown linkname " + linkname); + } + if (link->get_properties()->find(property_name) == link->get_properties()->end()) { + throw std::invalid_argument("getLinkProperty(): Unknown property \"" + property_name + "\" for link " + linkname); + } + return link->get_property(property_name); + } /** * @brief Identify variable links based on configured "variation" property tag @@ -404,11 +414,13 @@ void SimpleSimulator::identifyVariableLinks(std::shared_ptr throw std::runtime_error("Empty linkname list! Have you instantiated the platform already?"); } for (const auto& linkname: linkname_list) { - std::string linkProperty = simgrid::s4u::Link::by_name_or_null(linkname)->get_property("variation"); - if (linkProperty == ""){ - throw std::runtime_error("Configuration property \"variation\" missing for link " + linkname); + try { + std::string linkProperties = getLinkProperty(linkname, "variation"); + SimpleSimulator::variable_links.insert(linkname); + } catch (std::invalid_argument& e) { + std::cerr << e.what() << "\t->\t" << "Skip link " << linkname << "\n"; + continue; } - SimpleSimulator::variable_links.insert(linkname); } } @@ -778,8 +790,7 @@ int main(int argc, char **argv) { std::set> bandwidth_modifiers; SimpleSimulator::identifyVariableLinks(simulation); for (auto varied_link: SimpleSimulator::variable_links) { - auto the_link = simgrid::s4u::Link::by_name_or_null(varied_link); - std::string property = the_link->get_property("variation"); + std::string property = getLinkProperty(varied_link, "variation"); std::vector parameters; boost::split(parameters, property, boost::is_any_of("\t, ")); if (parameters.size() != 2) { From 6bacca25486f986c7d42428ae345ecbbc14c0d5d Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Thu, 20 Apr 2023 15:04:52 +0200 Subject: [PATCH 08/18] comment debug print outs --- src/SimpleSimulator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index dfcaeb4f..15c83fd2 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -418,7 +418,7 @@ void SimpleSimulator::identifyVariableLinks(std::shared_ptr std::string linkProperties = getLinkProperty(linkname, "variation"); SimpleSimulator::variable_links.insert(linkname); } catch (std::invalid_argument& e) { - std::cerr << e.what() << "\t->\t" << "Skip link " << linkname << "\n"; + // std::cerr << e.what() << "\t->\t" << "Skip link " << linkname << "\n"; continue; } } From dbcbe74b32e451becf8a96bb0b112bc5c2cf5de6 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Fri, 21 Apr 2023 15:41:35 +0200 Subject: [PATCH 09/18] first vary bandwidth then wait --- src/BandwidthModifier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BandwidthModifier.cpp b/src/BandwidthModifier.cpp index 7f4c03d1..8aaa3899 100644 --- a/src/BandwidthModifier.cpp +++ b/src/BandwidthModifier.cpp @@ -57,13 +57,13 @@ int BandwidthModifier::main() { std::mt19937 rng(this->seed); while (true) { - // Sleep during the period - wrench::Simulation::sleep(period); // Sample bandwidth modification subtrahend double reduction = (*this->distribution)(rng); while ((original_bandwidth - reduction < 0) || reduction < 0) reduction = (*this->distribution)(rng); // Update the link bandwidth the_link->set_bandwidth(original_bandwidth - reduction); + // Sleep during the period + wrench::Simulation::sleep(period); } } From fd5cef02284987ddf13348f74c8545389475c11e Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Mon, 24 Apr 2023 14:10:24 +0200 Subject: [PATCH 10/18] scale up WLCG for testing --- .../WLCG_disklessTier2_reduced50.xml | 143 ++++++++++++++++++ .../workload-configs/T1_DE_KIT_workloads.json | 8 +- .../T2_DE_DESY_workloads.json | 8 +- tools/prefetchPerformancePlots.py | 6 +- tools/prefetchScan.sh | 9 +- 5 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 data/platform-files/WLCG_disklessTier2_reduced50.xml diff --git a/data/platform-files/WLCG_disklessTier2_reduced50.xml b/data/platform-files/WLCG_disklessTier2_reduced50.xml new file mode 100644 index 00000000..e1637143 --- /dev/null +++ b/data/platform-files/WLCG_disklessTier2_reduced50.xml @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data/workload-configs/T1_DE_KIT_workloads.json b/data/workload-configs/T1_DE_KIT_workloads.json index e29e0d57..507c75ab 100644 --- a/data/workload-configs/T1_DE_KIT_workloads.json +++ b/data/workload-configs/T1_DE_KIT_workloads.json @@ -1,6 +1,6 @@ { "Analysis_T1": { - "num_jobs": 48, + "num_jobs": 96, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -31,7 +31,7 @@ "submission_time": 10 }, "Digi_T1": { - "num_jobs": 50, + "num_jobs": 100, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -61,7 +61,7 @@ "submission_time": 10 }, "DataProcessing_T1": { - "num_jobs": 13, + "num_jobs": 26, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -121,7 +121,7 @@ "submission_time": 10 }, "Merge_T1": { - "num_jobs": 5, + "num_jobs": 10, "infiles_per_job": 10, "cores": { "type": "histogram", diff --git a/data/workload-configs/T2_DE_DESY_workloads.json b/data/workload-configs/T2_DE_DESY_workloads.json index 0e25e876..d1f28df0 100644 --- a/data/workload-configs/T2_DE_DESY_workloads.json +++ b/data/workload-configs/T2_DE_DESY_workloads.json @@ -1,6 +1,6 @@ { "Analysis_T2": { - "num_jobs": 108, + "num_jobs": 216, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -31,7 +31,7 @@ "submission_time": 10 }, "Digi_T2": { - "num_jobs": 74, + "num_jobs": 148, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -61,7 +61,7 @@ "submission_time": 10 }, "DataProcessing_T2": { - "num_jobs": 11, + "num_jobs": 22, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -121,7 +121,7 @@ "submission_time": 10 }, "Merge_T2": { - "num_jobs": 4, + "num_jobs": 8, "infiles_per_job": 10, "cores": { "type": "histogram", diff --git a/tools/prefetchPerformancePlots.py b/tools/prefetchPerformancePlots.py index 475afa70..5ba83b8f 100644 --- a/tools/prefetchPerformancePlots.py +++ b/tools/prefetchPerformancePlots.py @@ -37,17 +37,17 @@ "Walltime": { "ident": "Walltime", "label": "jobtime / min", - "ylim": [0.,2000.], + "ylim": [0.,5500.], }, "IOtime": { "ident": "IOtime", "label": "transfer time / min", - "ylim": [0.,1500.], + "ylim": [0.,4500.], }, "CPUtime": { "ident": "CPUtime", "label": "CPU time / min", - "ylim": [0.,2000.] + "ylim": [0.,4000.] }, "CPUEfficiency": { "ident": "Efficiency", diff --git a/tools/prefetchScan.sh b/tools/prefetchScan.sh index 6a50daa6..115e9c20 100755 --- a/tools/prefetchScan.sh +++ b/tools/prefetchScan.sh @@ -15,7 +15,7 @@ action() { local base="$( cd "$( dirname "$this_file" )" && pwd )" local parent="$( dirname "$base" )" - local PLATFORM="$parent/data/platform-files/WLCG_disklessTier2_reduced100.xml" + local PLATFORM="$parent/data/platform-files/WLCG_disklessTier2_reduced50.xml" # local PLATFORM="$parent/data/platform-files/WLCG_disklessTier2_reduced1000.xml" # local WORKLOADS="$parent/data/workload-configs/Dummy_workloads.json $parent/data/workload-configs/T?_DE_*_workloads.json" local WORKLOADS="$parent/data/workload-configs/T?_DE_*_workloads.json" @@ -23,15 +23,17 @@ action() { local XRD_BLOCKSIZE=100000000 local STORAGE_BUFFER_SIZE=0 #1048576 - local SCENARIO="prefetchScanScaled100" + local SCENARIO="prefetchScanScaled10" # local SCENARIO="prefetchScanScaled1000" + local DUPLICATIONS=1 + local OUTDIR="$parent/tmp/outputs/WLCG" if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR fi - for prefetchrate in $(LANG=en_US seq 0.0 0.1 1.0) + for prefetchrate in $(LANG=en seq 0.0 0.1 0.0) do dc-sim --platform "$PLATFORM" \ --hitrate ${prefetchrate} \ @@ -41,6 +43,7 @@ action() { --storage-buffer-size $STORAGE_BUFFER_SIZE \ --cache-scope network \ --no-caching \ + --duplications $DUPLICATIONS \ --workload-configurations $WORKLOADS #\ # --no-streaming \ # --wrench-full-log From 738aab5ee4444e759734106c32f3f9fdf14a6b3d Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Mon, 24 Apr 2023 15:25:44 +0200 Subject: [PATCH 11/18] add htcondor configs for prefetch scan --- .../wlcg_disklessTier2/prefetch_scan.jdl | 31 +++++++++ .../wlcg_disklessTier2/prefetch_scan.sh | 69 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl create mode 100644 tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh diff --git a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl new file mode 100644 index 00000000..ab0ab658 --- /dev/null +++ b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl @@ -0,0 +1,31 @@ +# submit multiple simulations and harvest scaling and job information + +executable = ./prefetch_scan.sh +arguments = $(PREFETCHRATE) + +should_transfer_files = YES +transfer_input_files = ../../../data/platform-files/WLCG_disklessTier2_reduced50.xml ../../../data/workload-configs/T?_DE_*_workloads.json +transfer_output_files = scaling_dump_WLCG_disklessTier2_reduced50.xml_rate$(PREFETCHRATE).txt, prefetchScanScaled50_rate$(PREFETCHRATE).csv +when_to_transfer_output = ON_EXIT + +log = logs/log.$(ClusterId).$(ProcId) +output = logs/out.$(ClusterId).$(ProcId) +error = logs/err.$(ClusterId).$(ProcId) + + +accounting_group=cms.production +Requirements = TARGET.ProvidesIO && (TARGET.Machine=?="mdm1.ekp.kit.edu") ++RemoteJob = True + +universe = docker +docker_image = mschnepf/slc7-condocker + ++RequestWalltime = 3600 + (24*3600*NumJobStarts) +request_cpus = 1 +RequestMemory = 4000 +periodic_release = (HoldReasonCode == 34) +RequestDisk = 4000000 + +x509userproxy = $ENV(X509_USER_PROXY) + +queue PREFETCHRATE from seq 0.0 0.1 1.0 | diff --git a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh new file mode 100644 index 00000000..dbed5a53 --- /dev/null +++ b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh @@ -0,0 +1,69 @@ +#! /bin/bash + +ulimit -s unlimited +set -e + +echo "INITIAL ENVIRONMENT START" +env +echo "INITIAL ENVIRONMENT END" +echo "" + +echo "CURRENT DIRECTORY CONTENT:" +ls $(pwd) + +echo "SETTING GRID ENVIRONMENT" +source /cvmfs/grid.cern.ch/umd-c7ui-latest/etc/profile.d/setup-c7-ui-example.sh + +# echo "SOURCING CONDA ENVIRONMENT FROM CVMFS" +# source /cvmfs/etp.kit.edu/DCSim/pre_0.2/setup.sh + +echo "GETTING CONDA ENVIRONMENT FROM REMOTE STORAGE" +gfal-copy davs://cmswebdav-kit.gridka.de:2880/pnfs/gridka.de/cms/disk-only/store/user/mhorzela/dcsim-env.tar.gz dcsim-env.tar.gz + +echo "EXTRACTING AND SETTING CONDA ENVIRONMENT" +mkdir -p dcsim-env +tar -zxvf dcsim-env.tar.gz -C dcsim-env + +source dcsim-env/bin/activate +conda-unpack + +echo "UPDATING LIBRARIES" + +echo "Old LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib:${CONDA_PREFIX}/lib64:${CONDA_PREFIX}/lib32:${LD_LIBRARY_PATH} + +ldd ${CONDA_PREFIX}/bin/dc-sim + +echo "START SIMULATION" +DUPLICATIONS=1 +PREFETCHRATE="${1}" +XRDBLOCKSIZE=100000000 +STORAGE_BUFFER_SIZE=0 #1048576 + +PLATFORM="WLCG_disklessTier2_reduced50.xml" +WORKLOADS="T?_DE_*_workloads.json" + +SCENARIO="prefetchScanScaled50" + + dc-sim --platform "$PLATFORM" \ + --hitrate $PREFETCHRATE \ + --xrd-blocksize $XRD_BLOCKSIZE \ + --output-file ${SCENARIO}_rate$PREFETCHRATE.csv \ + --cfg=network/loopback-bw:100000000000000 \ + --storage-buffer-size $STORAGE_BUFFER_SIZE \ + --cache-scope network \ + --no-caching \ + --duplications $DUPLICATIONS \ + --workload-configurations $WORKLOADS \ + & TEST_PID=$! + + (while [[ True ]]; \ + do ps -aux | grep " ${TEST_PID} " | grep "dc-sim" \ + >> scaling_dump_${PLATFORM}_rate${PREFETCHRATE}.txt; \ + sleep 10; done;)\ + & MONITOR_PID=$! + echo "Simulation process to monitor: $TEST_PID" + echo "Monitoring process: $MONITOR_PID" + + wait $TEST_PID + kill -9 ${MONITOR_PID} From 40248e727ae4a54414c39bb8db6eaee161c5c619 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Tue, 25 Apr 2023 09:43:23 +0200 Subject: [PATCH 12/18] add configuration for bandwidth variations --- data/platform-files/WLCG_disklessTier2_reduced100.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/platform-files/WLCG_disklessTier2_reduced100.xml b/data/platform-files/WLCG_disklessTier2_reduced100.xml index 55109ee0..d0f90eb1 100644 --- a/data/platform-files/WLCG_disklessTier2_reduced100.xml +++ b/data/platform-files/WLCG_disklessTier2_reduced100.xml @@ -72,7 +72,9 @@ - + + + From 7d465a693059f31b7390439193f3972eb7b1bf67 Mon Sep 17 00:00:00 2001 From: Maximilian Maria Horzela Date: Tue, 25 Apr 2023 10:59:11 +0200 Subject: [PATCH 13/18] add htcondor configs for prefetch scan --- tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl | 4 ++-- tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl index ab0ab658..3fd5af8e 100644 --- a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl +++ b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl @@ -4,7 +4,7 @@ executable = ./prefetch_scan.sh arguments = $(PREFETCHRATE) should_transfer_files = YES -transfer_input_files = ../../../data/platform-files/WLCG_disklessTier2_reduced50.xml ../../../data/workload-configs/T?_DE_*_workloads.json +transfer_input_files = ../../../data/platform-files/WLCG_disklessTier2_reduced50.xml, ../../../data/workload-configs/T1_DE_KIT_workloads.json, ../../../data/workload-configs/T2_DE_DESY_workloads.json transfer_output_files = scaling_dump_WLCG_disklessTier2_reduced50.xml_rate$(PREFETCHRATE).txt, prefetchScanScaled50_rate$(PREFETCHRATE).csv when_to_transfer_output = ON_EXIT @@ -28,4 +28,4 @@ RequestDisk = 4000000 x509userproxy = $ENV(X509_USER_PROXY) -queue PREFETCHRATE from seq 0.0 0.1 1.0 | +queue PREFETCHRATE in 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0 diff --git a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh index dbed5a53..aeb83c3d 100644 --- a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh +++ b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh @@ -47,7 +47,7 @@ SCENARIO="prefetchScanScaled50" dc-sim --platform "$PLATFORM" \ --hitrate $PREFETCHRATE \ - --xrd-blocksize $XRD_BLOCKSIZE \ + --xrd-blocksize $XRDBLOCKSIZE \ --output-file ${SCENARIO}_rate$PREFETCHRATE.csv \ --cfg=network/loopback-bw:100000000000000 \ --storage-buffer-size $STORAGE_BUFFER_SIZE \ From 48944aa7b877694de857973fc6245a78acfb354c Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Wed, 26 Apr 2023 16:13:18 +0200 Subject: [PATCH 14/18] update plot ranges for legend positioning --- tools/prefetchPerformancePlots.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/prefetchPerformancePlots.py b/tools/prefetchPerformancePlots.py index 5ba83b8f..228ac795 100644 --- a/tools/prefetchPerformancePlots.py +++ b/tools/prefetchPerformancePlots.py @@ -37,22 +37,22 @@ "Walltime": { "ident": "Walltime", "label": "jobtime / min", - "ylim": [0.,5500.], + "ylim": [0.,7000.], }, "IOtime": { "ident": "IOtime", "label": "transfer time / min", - "ylim": [0.,4500.], + "ylim": [0.,5500.], }, "CPUtime": { "ident": "CPUtime", "label": "CPU time / min", - "ylim": [0.,4000.] + "ylim": [0.,5500.] }, "CPUEfficiency": { "ident": "Efficiency", "label": "CPU efficiency", - "ylim": [0,1.05], + "ylim": [0,1.3], }, "Hitrate": { "ident": "hitrate", From af2f6c6753e8b89614170c5f2955fa078b8625be Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 26 Apr 2023 15:59:52 -1000 Subject: [PATCH 15/18] Fixed daemonization bug for bandwidth modifier --- src/SimpleSimulator.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index 15c83fd2..d445878a 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -811,6 +811,7 @@ int main(int argc, char **argv) { 42 ) ); + bm->setDaemonized(true); std::cerr << "\t Created a BandwidthModifier on host " << host << " for link " << varied_link << "\n"; } } From aebfc1deb586caada0aaaa2c7ea0a8245c3fe796 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Tue, 2 May 2023 13:32:43 +0200 Subject: [PATCH 16/18] add explanation plot to pointplots/boxplots --- tools/hitratePerformancePlots.py | 22 +++++++++++++++++++++- tools/prefetchPerformancePlots.py | 26 +++++++++++++++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/tools/hitratePerformancePlots.py b/tools/hitratePerformancePlots.py index 6db2c5a7..02d3375f 100755 --- a/tools/hitratePerformancePlots.py +++ b/tools/hitratePerformancePlots.py @@ -3,6 +3,7 @@ import pandas as pd import numpy as np from matplotlib import pyplot as plt +import matplotlib.lines as mlines import seaborn as sns import os.path import argparse @@ -219,16 +220,25 @@ def mapHostToSite(test: str, mapping: 'dict[str,str]',): markers = markers[0:len(sites)] fig = plt.figure(f"hitrate-{quantity}", figsize=(6,4)) ax1 = fig.add_subplot(1,1,1) + percentileinterval=95 sns.pointplot( x="hitrate", y=quantity, hue="Site", hue_order=sites, data=df, - estimator="median", errorbar=("pi",95), # ci = Confidence Interval, pi = Percentile Interval, sd = Standard Deviation, se = Standard Error of Mean + estimator="median", errorbar=("pi",percentileinterval), # ci = Confidence Interval, pi = Percentile Interval, sd = Standard Deviation, se = Standard Error of Mean dodge=True, join=False, markers=markers, capsize=0.5/len(sites), errwidth=1., palette=sns.color_palette("colorblind", n_colors=len(sites)), ax=ax1 ) + ax_in = ax1.inset_axes([0.56,0.03,0.15,0.15]) + ax_in.errorbar(1,1,1, marker="o", linestyle="none", capsize=5, color="black") + ax_in.axis("off") + ax_in.annotate(f"{(0.5*percentileinterval)+50} percentile", xy=(1.01,2), xytext=(1.05,1.8), arrowprops=dict(arrowstyle="-|>", color="black",), ha="left") + ax_in.annotate("median", xy=(1.01,1), xytext=(1.05,0.8), arrowprops=dict(arrowstyle="-|>", color="black",)) + ax_in.annotate(f"{0.5*(100-percentileinterval)} percentile", xy=(1.01,0), xytext=(1.05,-0.2), arrowprops=dict(arrowstyle="-|>", color="black",), ha="left") + ax_in.set_yticklabels([]) + ax_in.set_xticklabels([]) ax1.set_title(scenario_plotlabel_dict[scenario]) ax1.set_xlabel("hitrate", loc="right", color="black") scale_xticks(ax1, hitrateticks) @@ -251,6 +261,16 @@ def mapHostToSite(test: str, mapping: 'dict[str,str]',): flierprops=dict(marker="x"), palette=sns.color_palette("colorblind", n_colors=len(sites)), ) + ax_in = ax1.inset_axes([0.56,0.03,0.3,0.3]) + ax_in.boxplot(np.linspace(0.,4.,100), medianprops = dict(color="black")) + ax_in.axis("off") + ax_in.annotate("Q3 + 1.5IQR", xy=(1.1,4), xytext=(1.2,3.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("Q3 = 75 percentile", xy=(1.1,3), xytext=(1.2,2.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("median", xy=(1.1,2), xytext=(1.2,1.8), arrowprops=dict(arrowstyle="-", color="black",)) + ax_in.annotate(f"Q1 = 25 percentile", xy=(1.1,1), xytext=(1.2,0.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate(f"Q1 - 1.5IQR", xy=(1.1,0), xytext=(1.2,-0.2), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.set_yticklabels([]) + ax_in.set_xticklabels([]) ax1.set_title(scenario_plotlabel_dict[scenario]) ax1.set_xlabel("hitrate", loc="right") scale_xticks(ax1, hitrateticks) diff --git a/tools/prefetchPerformancePlots.py b/tools/prefetchPerformancePlots.py index 228ac795..c33a6c1c 100644 --- a/tools/prefetchPerformancePlots.py +++ b/tools/prefetchPerformancePlots.py @@ -42,7 +42,7 @@ "IOtime": { "ident": "IOtime", "label": "transfer time / min", - "ylim": [0.,5500.], + "ylim": [0.,3500.], }, "CPUtime": { "ident": "CPUtime", @@ -185,7 +185,8 @@ def plotBoxes( prefix="", suffix="", figsize=(6,4), emptyBoxes=False, - plot_dir=os.path.join(os.path.dirname(__file__),"..","plots") + plot_dir=os.path.join(os.path.dirname(__file__),"..","plots"), + boxplot_legend=False ): plot_dir = os.path.abspath(plot_dir) if not os.path.exists(plot_dir): @@ -206,6 +207,17 @@ def plotBoxes( flierprops=dict(marker="x"), palette=sns.color_palette("colorblind", n_colors=len(sites)) ) + if boxplot_legend: + ax_in = ax1.inset_axes([0.36,0.67,0.3,0.3]) + ax_in.boxplot(np.linspace(0.,4.,100), medianprops = dict(color="black")) + ax_in.axis("off") + ax_in.annotate("Q3 + 1.5IQR", xy=(1.1,4), xytext=(1.2,3.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("Q3 = 75 percentile", xy=(1.1,3), xytext=(1.2,2.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("median", xy=(1.1,2), xytext=(1.2,1.8), arrowprops=dict(arrowstyle="-", color="black",)) + ax_in.annotate(f"Q1 = 25 percentile", xy=(1.1,1), xytext=(1.2,0.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate(f"Q1 - 1.5IQR", xy=(1.1,0), xytext=(1.2,-0.2), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.set_yticklabels([]) + ax_in.set_xticklabels([]) ax1.set_title(title) ax1.set_xlabel("fraction of prefetched files in cache",color="black") hitrateticks = [x*0.1 for x in range(0,11)] @@ -262,11 +274,11 @@ def run(args=parser.parse_args()): else: suffix = args.suffix plotBoxes(data, - sites=sites, - title=args.scenario, - emptyBoxes=False, - plot_dir=out_dir, - suffix=suffix) + sites=sites, + title=args.scenario, + emptyBoxes=False, + plot_dir=out_dir, + suffix=suffix) if __name__ == "__main__": From 92039cc3b10dfbedc0b8e074366bc5cd72351032 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Tue, 2 May 2023 17:00:28 +0200 Subject: [PATCH 17/18] add script for creating standalone boxplot legend --- tools/boxplot_legend.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tools/boxplot_legend.py diff --git a/tools/boxplot_legend.py b/tools/boxplot_legend.py new file mode 100644 index 00000000..025215b9 --- /dev/null +++ b/tools/boxplot_legend.py @@ -0,0 +1,43 @@ + + +import numpy as np +from matplotlib import pyplot as plt +import os + +plt.rcParams['figure.autolayout'] = True +plt.rcParams['axes.facecolor'] = 'white' +plt.rcParams['axes.spines.left'] = True +plt.rcParams['axes.spines.right'] = False +plt.rcParams['axes.spines.top'] = False +plt.rcParams['axes.spines.bottom'] = True +plt.rcParams['axes.grid'] = False +plt.rcParams['axes.grid.axis'] = 'both' +plt.rcParams['axes.labelcolor'] = 'black' +plt.rcParams['axes.labelsize'] = 13 +plt.rcParams['text.color'] = 'black' +plt.rcParams['figure.figsize'] = 6,4 +plt.rcParams['figure.dpi'] = 100 +plt.rcParams['figure.titleweight'] = 'normal' +plt.rcParams['font.family'] = 'sans-serif' +# plt.rcParams['font.weight'] = 'bold' +plt.rcParams['font.size'] = 13 + +fig = plt.figure(f"boxplot_legend", figsize=(3,2)) + +ax_in = fig.add_subplot(1,1,1) + +ax_in.boxplot(np.linspace(0.,4.,100), medianprops = dict(color="black")) +ax_in.axis("off") +ax_in.annotate("Q3$+ 1.5$IQR", xy=(1.1,4), xytext=(1.2,3.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate("Q3$= 75$ percentile", xy=(1.1,3), xytext=(1.2,2.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate("median", xy=(1.1,2), xytext=(1.2,1.8), arrowprops=dict(arrowstyle="-", color="black",)) +ax_in.annotate(f"Q1$= 25$ percentile", xy=(1.1,1), xytext=(1.2,0.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate(f"Q1$- 1.5$IQR", xy=(1.1,0), xytext=(1.2,-0.2), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate('', xy=(0.85,0.898), xytext=(0.85, 3.079), arrowprops={'arrowstyle': '<->'}) +ax_in.annotate('IQR', xy=(0.8, 1.8), xytext=(0.8, 1.8), ha="right") +ax_in.set_yticklabels([]) +ax_in.set_xticklabels([]) + +fig.savefig(os.path.join(os.getcwd(), f"{fig.get_label()}.pdf")) +fig.savefig(os.path.join(os.getcwd(), f"{fig.get_label()}.png")) +plt.close() From 82d24cb8379fe0c4fe5223bbab3c6f6755ad9b25 Mon Sep 17 00:00:00 2001 From: HerrHorizontal Date: Mon, 17 Jul 2023 11:06:30 +0200 Subject: [PATCH 18/18] fix helper script --- tools/prefetchScan.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/prefetchScan.sh b/tools/prefetchScan.sh index 115e9c20..c89a2248 100755 --- a/tools/prefetchScan.sh +++ b/tools/prefetchScan.sh @@ -15,7 +15,7 @@ action() { local base="$( cd "$( dirname "$this_file" )" && pwd )" local parent="$( dirname "$base" )" - local PLATFORM="$parent/data/platform-files/WLCG_disklessTier2_reduced50.xml" + local PLATFORM="$parent/data/platform-files/WLCG_disklessTier2_reduced100.xml" # local PLATFORM="$parent/data/platform-files/WLCG_disklessTier2_reduced1000.xml" # local WORKLOADS="$parent/data/workload-configs/Dummy_workloads.json $parent/data/workload-configs/T?_DE_*_workloads.json" local WORKLOADS="$parent/data/workload-configs/T?_DE_*_workloads.json" @@ -23,7 +23,7 @@ action() { local XRD_BLOCKSIZE=100000000 local STORAGE_BUFFER_SIZE=0 #1048576 - local SCENARIO="prefetchScanScaled10" + local SCENARIO="prefetchScanScaled100" # local SCENARIO="prefetchScanScaled1000" local DUPLICATIONS=1 @@ -33,8 +33,8 @@ action() { mkdir -p $OUTDIR fi - for prefetchrate in $(LANG=en seq 0.0 0.1 0.0) - do + for prefetchrate in $(LANG=en seq 0.0 0.1 1.0) + do dc-sim --platform "$PLATFORM" \ --hitrate ${prefetchrate} \ --xrd-blocksize $XRD_BLOCKSIZE \