diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f38e2f2..336f4b5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,8 @@ include_directories(src/ ${SimGrid_INCLUDE_DIR}/include /usr/local/include /opt/ set(SOURCE_FILES src/WorkloadExecutionController.h src/WorkloadExecutionController.cpp + src/BandwidthModifier.h + src/BandwidthModifier.cpp src/SimpleSimulator.h src/SimpleSimulator.cpp src/JobSpecification.h diff --git a/data/platform-files/WLCG_disklessTier2_reduced100.xml b/data/platform-files/WLCG_disklessTier2_reduced100.xml index 55109ee0..d0f90eb1 100644 --- a/data/platform-files/WLCG_disklessTier2_reduced100.xml +++ b/data/platform-files/WLCG_disklessTier2_reduced100.xml @@ -72,7 +72,9 @@ - + + + diff --git a/data/platform-files/WLCG_disklessTier2_reduced50.xml b/data/platform-files/WLCG_disklessTier2_reduced50.xml new file mode 100644 index 00000000..e1637143 --- /dev/null +++ b/data/platform-files/WLCG_disklessTier2_reduced50.xml @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data/workload-configs/T1_DE_KIT_workloads.json b/data/workload-configs/T1_DE_KIT_workloads.json index e29e0d57..507c75ab 100644 --- a/data/workload-configs/T1_DE_KIT_workloads.json +++ b/data/workload-configs/T1_DE_KIT_workloads.json @@ -1,6 +1,6 @@ { "Analysis_T1": { - "num_jobs": 48, + "num_jobs": 96, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -31,7 +31,7 @@ "submission_time": 10 }, "Digi_T1": { - "num_jobs": 50, + "num_jobs": 100, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -61,7 +61,7 @@ "submission_time": 10 }, "DataProcessing_T1": { - "num_jobs": 13, + "num_jobs": 26, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -121,7 +121,7 @@ "submission_time": 10 }, "Merge_T1": { - "num_jobs": 5, + "num_jobs": 10, "infiles_per_job": 10, "cores": { "type": "histogram", diff --git a/data/workload-configs/T2_DE_DESY_workloads.json b/data/workload-configs/T2_DE_DESY_workloads.json index 0e25e876..d1f28df0 100644 --- a/data/workload-configs/T2_DE_DESY_workloads.json +++ b/data/workload-configs/T2_DE_DESY_workloads.json @@ -1,6 +1,6 @@ { "Analysis_T2": { - "num_jobs": 108, + "num_jobs": 216, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -31,7 +31,7 @@ "submission_time": 10 }, "Digi_T2": { - "num_jobs": 74, + "num_jobs": 148, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -61,7 +61,7 @@ "submission_time": 10 }, "DataProcessing_T2": { - "num_jobs": 11, + "num_jobs": 22, "infiles_per_job": 10, "cores": { "type": "histogram", @@ -121,7 +121,7 @@ "submission_time": 10 }, "Merge_T2": { - "num_jobs": 4, + "num_jobs": 8, "infiles_per_job": 10, "cores": { "type": "histogram", diff --git a/src/BandwidthModifier.cpp b/src/BandwidthModifier.cpp new file mode 100644 index 00000000..8aaa3899 --- /dev/null +++ b/src/BandwidthModifier.cpp @@ -0,0 +1,69 @@ +/** + * Copyright (c) 2020. . + * Generated with the wrench-init.in tool. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ +#include +#include + +#include "BandwidthModifier.h" + +XBT_LOG_NEW_DEFAULT_CATEGORY(bandwidth_modifier, "Log category for BandwidthModifier"); + +/** + * @brief A "controller" that dynamically modifies a link's bandwidth + * + * @param hostname: the host on which this actor runs + * @param link_name: the name of the link whose bandwidth is to be modified + * @param period: the period in between two bandwidth modifications (in seconds) + * @param distribution: the normal distribution from which to same bandwidth modification values + */ +BandwidthModifier::BandwidthModifier( + const std::string &hostname, + const std::string &link_name, + double period, + std::normal_distribution* distribution, + unsigned long seed) : wrench::ExecutionController( + hostname, + "bandwidth-modifier"), link_name(link_name), period(period), distribution(distribution), seed(seed) { + +} + +/** + * @brief main method of the BandwidthModifier actor + * + * @return 0 on completion + * + * @throw std::runtime_error + */ +int BandwidthModifier::main() { + + wrench::TerminalOutput::setThisProcessLoggingColor(wrench::TerminalOutput::COLOR_GREEN); + + // Retrieve the link + auto the_link = simgrid::s4u::Link::by_name_or_null(this->link_name); + if (the_link == nullptr) { + throw std::invalid_argument("BandwidthModifier::main(): The platform does not contain a link named '" + this->link_name +"'"); + } + + // Get its (original) bandwidth + const double original_bandwidth = the_link->get_bandwidth(); + + // Create RNG + std::mt19937 rng(this->seed); + + while (true) { + // Sample bandwidth modification subtrahend + double reduction = (*this->distribution)(rng); + while ((original_bandwidth - reduction < 0) || reduction < 0) reduction = (*this->distribution)(rng); + // Update the link bandwidth + the_link->set_bandwidth(original_bandwidth - reduction); + // Sleep during the period + wrench::Simulation::sleep(period); + } +} + diff --git a/src/BandwidthModifier.h b/src/BandwidthModifier.h new file mode 100644 index 00000000..d798a0eb --- /dev/null +++ b/src/BandwidthModifier.h @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2020. . + * Generated with the wrench-init.in tool. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ +#ifndef MY_BANDWIDTH_MODIFIER_H +#define MY_BANDWIDTH_MODIFIER_H + +#include +#include +#include + +#include "JobSpecification.h" +#include "LRU_FileList.h" + +#include "util/Utils.h" + +class Simulation; + +class BandwidthModifier : public wrench::ExecutionController { +public: + // Constructor + BandwidthModifier( + const std::string &hostname, + const std::string &link_name, + double period, + std::normal_distribution* distribution, + unsigned long seed); + +private: + + std::string link_name; + double period; + std::normal_distribution* distribution; + std::mt19937 generator; + unsigned long seed; + + int main() override; + +}; + +#endif //MY_BANDWIDTH_MODIFIER_H + diff --git a/src/SimpleSimulator.cpp b/src/SimpleSimulator.cpp index 4289df06..d445878a 100644 --- a/src/SimpleSimulator.cpp +++ b/src/SimpleSimulator.cpp @@ -11,6 +11,7 @@ #include "SimpleSimulator.h" #include "WorkloadExecutionController.h" #include "JobSpecification.h" +#include "BandwidthModifier.h" #include "util/Utils.h" @@ -19,6 +20,8 @@ #include #include +#include +#include #include #include @@ -51,6 +54,7 @@ std::set SimpleSimulator::scheduler_hosts; std::set SimpleSimulator::executors; std::set SimpleSimulator::file_registries; std::set SimpleSimulator::network_monitors; +std::set SimpleSimulator::variable_links; std::map> SimpleSimulator::hosts_in_zones; bool SimpleSimulator::local_cache_scope = false; // flag to consider only local caches @@ -386,6 +390,41 @@ void SimpleSimulator::identifyHostTypes(std::shared_ptr simu } } +std::string getLinkProperty(const std::string &linkname, const std::string &property_name) { + auto link = simgrid::s4u::Link::by_name_or_null(linkname); + if (link == nullptr) { + throw std::invalid_argument("getLinkProperty(): Unknown linkname " + linkname); + } + if (link->get_properties()->find(property_name) == link->get_properties()->end()) { + throw std::invalid_argument("getLinkProperty(): Unknown property \"" + property_name + "\" for link " + linkname); + } + return link->get_property(property_name); + } + +/** + * @brief Identify variable links based on configured "variation" property tag + * + * @param simulation Simulation object with already instantiated hosts + * + * @throw std::runtime_error, std::invalid_argument + */ +void SimpleSimulator::identifyVariableLinks(std::shared_ptr simulation){ + std::vector linkname_list = simulation->getLinknameList(); + if (linkname_list.size() == 0) { + throw std::runtime_error("Empty linkname list! Have you instantiated the platform already?"); + } + for (const auto& linkname: linkname_list) { + try { + std::string linkProperties = getLinkProperty(linkname, "variation"); + SimpleSimulator::variable_links.insert(linkname); + } catch (std::invalid_argument& e) { + // std::cerr << e.what() << "\t->\t" << "Skip link " << linkname << "\n"; + continue; + } + } +} + + /** * @brief Method to be executed once at simulation start, * which finds all hosts in zone and all same level accopanying zones (siblings) @@ -746,6 +785,38 @@ int main(int argc, char **argv) { std::cerr << "The simulation now has " << std::to_string(num_total_jobs) << " jobs in total " << std::endl; + /* Identify the links which should be variied and add bandwidth modifiers to the simulation */ + std::cerr << "Setting varied link bandwidths ... " << "\n"; + std::set> bandwidth_modifiers; + SimpleSimulator::identifyVariableLinks(simulation); + for (auto varied_link: SimpleSimulator::variable_links) { + std::string property = getLinkProperty(varied_link, "variation"); + std::vector parameters; + boost::split(parameters, property, boost::is_any_of("\t, ")); + if (parameters.size() != 2) { + throw std::runtime_error( + "Property \"variation\":" + property + " for link " + varied_link + " misconfigured! Should only contain two splitable values!" + ); + } + double mu = std::stod(parameters.at(0)); + double sigma = std::stod(parameters.at(1)); + std::normal_distribution reduction(mu, sigma); + for (auto host: SimpleSimulator::executors) { + auto bm = simulation->add( + new BandwidthModifier( + host, + varied_link, + 3600., + &reduction, + 42 + ) + ); + bm->setDaemonized(true); + std::cerr << "\t Created a BandwidthModifier on host " << host << " for link " << varied_link << "\n"; + } + } + + /* Launch the simulation */ try { /* initialize output-dump file */ diff --git a/src/SimpleSimulator.h b/src/SimpleSimulator.h index eb074e34..d0ab9f71 100644 --- a/src/SimpleSimulator.h +++ b/src/SimpleSimulator.h @@ -11,6 +11,7 @@ class SimpleSimulator { public: static void identifyHostTypes(std::shared_ptr simulation); + static void identifyVariableLinks(std::shared_ptr simulation); static std::set cache_hosts; // hosts configured to provide a cache static std::set storage_hosts; // hosts configured to provide GRID storage @@ -20,6 +21,8 @@ class SimpleSimulator { static std::set file_registries; // hosts configured to manage a file registry static std::set network_monitors; // hosts configured to monitor network + static std::set variable_links; // links configured to randomly vary their bandwidth + static void fillHostsInSiblingZonesMap(bool include_subzones); static bool local_cache_scope; diff --git a/tools/boxplot_legend.py b/tools/boxplot_legend.py new file mode 100644 index 00000000..025215b9 --- /dev/null +++ b/tools/boxplot_legend.py @@ -0,0 +1,43 @@ + + +import numpy as np +from matplotlib import pyplot as plt +import os + +plt.rcParams['figure.autolayout'] = True +plt.rcParams['axes.facecolor'] = 'white' +plt.rcParams['axes.spines.left'] = True +plt.rcParams['axes.spines.right'] = False +plt.rcParams['axes.spines.top'] = False +plt.rcParams['axes.spines.bottom'] = True +plt.rcParams['axes.grid'] = False +plt.rcParams['axes.grid.axis'] = 'both' +plt.rcParams['axes.labelcolor'] = 'black' +plt.rcParams['axes.labelsize'] = 13 +plt.rcParams['text.color'] = 'black' +plt.rcParams['figure.figsize'] = 6,4 +plt.rcParams['figure.dpi'] = 100 +plt.rcParams['figure.titleweight'] = 'normal' +plt.rcParams['font.family'] = 'sans-serif' +# plt.rcParams['font.weight'] = 'bold' +plt.rcParams['font.size'] = 13 + +fig = plt.figure(f"boxplot_legend", figsize=(3,2)) + +ax_in = fig.add_subplot(1,1,1) + +ax_in.boxplot(np.linspace(0.,4.,100), medianprops = dict(color="black")) +ax_in.axis("off") +ax_in.annotate("Q3$+ 1.5$IQR", xy=(1.1,4), xytext=(1.2,3.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate("Q3$= 75$ percentile", xy=(1.1,3), xytext=(1.2,2.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate("median", xy=(1.1,2), xytext=(1.2,1.8), arrowprops=dict(arrowstyle="-", color="black",)) +ax_in.annotate(f"Q1$= 25$ percentile", xy=(1.1,1), xytext=(1.2,0.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate(f"Q1$- 1.5$IQR", xy=(1.1,0), xytext=(1.2,-0.2), arrowprops=dict(arrowstyle="-", color="black",), ha="left") +ax_in.annotate('', xy=(0.85,0.898), xytext=(0.85, 3.079), arrowprops={'arrowstyle': '<->'}) +ax_in.annotate('IQR', xy=(0.8, 1.8), xytext=(0.8, 1.8), ha="right") +ax_in.set_yticklabels([]) +ax_in.set_xticklabels([]) + +fig.savefig(os.path.join(os.getcwd(), f"{fig.get_label()}.pdf")) +fig.savefig(os.path.join(os.getcwd(), f"{fig.get_label()}.png")) +plt.close() diff --git a/tools/hitratePerformancePlots.py b/tools/hitratePerformancePlots.py index 6db2c5a7..02d3375f 100755 --- a/tools/hitratePerformancePlots.py +++ b/tools/hitratePerformancePlots.py @@ -3,6 +3,7 @@ import pandas as pd import numpy as np from matplotlib import pyplot as plt +import matplotlib.lines as mlines import seaborn as sns import os.path import argparse @@ -219,16 +220,25 @@ def mapHostToSite(test: str, mapping: 'dict[str,str]',): markers = markers[0:len(sites)] fig = plt.figure(f"hitrate-{quantity}", figsize=(6,4)) ax1 = fig.add_subplot(1,1,1) + percentileinterval=95 sns.pointplot( x="hitrate", y=quantity, hue="Site", hue_order=sites, data=df, - estimator="median", errorbar=("pi",95), # ci = Confidence Interval, pi = Percentile Interval, sd = Standard Deviation, se = Standard Error of Mean + estimator="median", errorbar=("pi",percentileinterval), # ci = Confidence Interval, pi = Percentile Interval, sd = Standard Deviation, se = Standard Error of Mean dodge=True, join=False, markers=markers, capsize=0.5/len(sites), errwidth=1., palette=sns.color_palette("colorblind", n_colors=len(sites)), ax=ax1 ) + ax_in = ax1.inset_axes([0.56,0.03,0.15,0.15]) + ax_in.errorbar(1,1,1, marker="o", linestyle="none", capsize=5, color="black") + ax_in.axis("off") + ax_in.annotate(f"{(0.5*percentileinterval)+50} percentile", xy=(1.01,2), xytext=(1.05,1.8), arrowprops=dict(arrowstyle="-|>", color="black",), ha="left") + ax_in.annotate("median", xy=(1.01,1), xytext=(1.05,0.8), arrowprops=dict(arrowstyle="-|>", color="black",)) + ax_in.annotate(f"{0.5*(100-percentileinterval)} percentile", xy=(1.01,0), xytext=(1.05,-0.2), arrowprops=dict(arrowstyle="-|>", color="black",), ha="left") + ax_in.set_yticklabels([]) + ax_in.set_xticklabels([]) ax1.set_title(scenario_plotlabel_dict[scenario]) ax1.set_xlabel("hitrate", loc="right", color="black") scale_xticks(ax1, hitrateticks) @@ -251,6 +261,16 @@ def mapHostToSite(test: str, mapping: 'dict[str,str]',): flierprops=dict(marker="x"), palette=sns.color_palette("colorblind", n_colors=len(sites)), ) + ax_in = ax1.inset_axes([0.56,0.03,0.3,0.3]) + ax_in.boxplot(np.linspace(0.,4.,100), medianprops = dict(color="black")) + ax_in.axis("off") + ax_in.annotate("Q3 + 1.5IQR", xy=(1.1,4), xytext=(1.2,3.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("Q3 = 75 percentile", xy=(1.1,3), xytext=(1.2,2.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("median", xy=(1.1,2), xytext=(1.2,1.8), arrowprops=dict(arrowstyle="-", color="black",)) + ax_in.annotate(f"Q1 = 25 percentile", xy=(1.1,1), xytext=(1.2,0.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate(f"Q1 - 1.5IQR", xy=(1.1,0), xytext=(1.2,-0.2), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.set_yticklabels([]) + ax_in.set_xticklabels([]) ax1.set_title(scenario_plotlabel_dict[scenario]) ax1.set_xlabel("hitrate", loc="right") scale_xticks(ax1, hitrateticks) diff --git a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl new file mode 100644 index 00000000..3fd5af8e --- /dev/null +++ b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.jdl @@ -0,0 +1,31 @@ +# submit multiple simulations and harvest scaling and job information + +executable = ./prefetch_scan.sh +arguments = $(PREFETCHRATE) + +should_transfer_files = YES +transfer_input_files = ../../../data/platform-files/WLCG_disklessTier2_reduced50.xml, ../../../data/workload-configs/T1_DE_KIT_workloads.json, ../../../data/workload-configs/T2_DE_DESY_workloads.json +transfer_output_files = scaling_dump_WLCG_disklessTier2_reduced50.xml_rate$(PREFETCHRATE).txt, prefetchScanScaled50_rate$(PREFETCHRATE).csv +when_to_transfer_output = ON_EXIT + +log = logs/log.$(ClusterId).$(ProcId) +output = logs/out.$(ClusterId).$(ProcId) +error = logs/err.$(ClusterId).$(ProcId) + + +accounting_group=cms.production +Requirements = TARGET.ProvidesIO && (TARGET.Machine=?="mdm1.ekp.kit.edu") ++RemoteJob = True + +universe = docker +docker_image = mschnepf/slc7-condocker + ++RequestWalltime = 3600 + (24*3600*NumJobStarts) +request_cpus = 1 +RequestMemory = 4000 +periodic_release = (HoldReasonCode == 34) +RequestDisk = 4000000 + +x509userproxy = $ENV(X509_USER_PROXY) + +queue PREFETCHRATE in 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0 diff --git a/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh new file mode 100644 index 00000000..aeb83c3d --- /dev/null +++ b/tools/htcondor/wlcg_disklessTier2/prefetch_scan.sh @@ -0,0 +1,69 @@ +#! /bin/bash + +ulimit -s unlimited +set -e + +echo "INITIAL ENVIRONMENT START" +env +echo "INITIAL ENVIRONMENT END" +echo "" + +echo "CURRENT DIRECTORY CONTENT:" +ls $(pwd) + +echo "SETTING GRID ENVIRONMENT" +source /cvmfs/grid.cern.ch/umd-c7ui-latest/etc/profile.d/setup-c7-ui-example.sh + +# echo "SOURCING CONDA ENVIRONMENT FROM CVMFS" +# source /cvmfs/etp.kit.edu/DCSim/pre_0.2/setup.sh + +echo "GETTING CONDA ENVIRONMENT FROM REMOTE STORAGE" +gfal-copy davs://cmswebdav-kit.gridka.de:2880/pnfs/gridka.de/cms/disk-only/store/user/mhorzela/dcsim-env.tar.gz dcsim-env.tar.gz + +echo "EXTRACTING AND SETTING CONDA ENVIRONMENT" +mkdir -p dcsim-env +tar -zxvf dcsim-env.tar.gz -C dcsim-env + +source dcsim-env/bin/activate +conda-unpack + +echo "UPDATING LIBRARIES" + +echo "Old LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}" +export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib:${CONDA_PREFIX}/lib64:${CONDA_PREFIX}/lib32:${LD_LIBRARY_PATH} + +ldd ${CONDA_PREFIX}/bin/dc-sim + +echo "START SIMULATION" +DUPLICATIONS=1 +PREFETCHRATE="${1}" +XRDBLOCKSIZE=100000000 +STORAGE_BUFFER_SIZE=0 #1048576 + +PLATFORM="WLCG_disklessTier2_reduced50.xml" +WORKLOADS="T?_DE_*_workloads.json" + +SCENARIO="prefetchScanScaled50" + + dc-sim --platform "$PLATFORM" \ + --hitrate $PREFETCHRATE \ + --xrd-blocksize $XRDBLOCKSIZE \ + --output-file ${SCENARIO}_rate$PREFETCHRATE.csv \ + --cfg=network/loopback-bw:100000000000000 \ + --storage-buffer-size $STORAGE_BUFFER_SIZE \ + --cache-scope network \ + --no-caching \ + --duplications $DUPLICATIONS \ + --workload-configurations $WORKLOADS \ + & TEST_PID=$! + + (while [[ True ]]; \ + do ps -aux | grep " ${TEST_PID} " | grep "dc-sim" \ + >> scaling_dump_${PLATFORM}_rate${PREFETCHRATE}.txt; \ + sleep 10; done;)\ + & MONITOR_PID=$! + echo "Simulation process to monitor: $TEST_PID" + echo "Monitoring process: $MONITOR_PID" + + wait $TEST_PID + kill -9 ${MONITOR_PID} diff --git a/tools/prefetchPerformancePlots.py b/tools/prefetchPerformancePlots.py index 475afa70..c33a6c1c 100644 --- a/tools/prefetchPerformancePlots.py +++ b/tools/prefetchPerformancePlots.py @@ -37,22 +37,22 @@ "Walltime": { "ident": "Walltime", "label": "jobtime / min", - "ylim": [0.,2000.], + "ylim": [0.,7000.], }, "IOtime": { "ident": "IOtime", "label": "transfer time / min", - "ylim": [0.,1500.], + "ylim": [0.,3500.], }, "CPUtime": { "ident": "CPUtime", "label": "CPU time / min", - "ylim": [0.,2000.] + "ylim": [0.,5500.] }, "CPUEfficiency": { "ident": "Efficiency", "label": "CPU efficiency", - "ylim": [0,1.05], + "ylim": [0,1.3], }, "Hitrate": { "ident": "hitrate", @@ -185,7 +185,8 @@ def plotBoxes( prefix="", suffix="", figsize=(6,4), emptyBoxes=False, - plot_dir=os.path.join(os.path.dirname(__file__),"..","plots") + plot_dir=os.path.join(os.path.dirname(__file__),"..","plots"), + boxplot_legend=False ): plot_dir = os.path.abspath(plot_dir) if not os.path.exists(plot_dir): @@ -206,6 +207,17 @@ def plotBoxes( flierprops=dict(marker="x"), palette=sns.color_palette("colorblind", n_colors=len(sites)) ) + if boxplot_legend: + ax_in = ax1.inset_axes([0.36,0.67,0.3,0.3]) + ax_in.boxplot(np.linspace(0.,4.,100), medianprops = dict(color="black")) + ax_in.axis("off") + ax_in.annotate("Q3 + 1.5IQR", xy=(1.1,4), xytext=(1.2,3.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("Q3 = 75 percentile", xy=(1.1,3), xytext=(1.2,2.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate("median", xy=(1.1,2), xytext=(1.2,1.8), arrowprops=dict(arrowstyle="-", color="black",)) + ax_in.annotate(f"Q1 = 25 percentile", xy=(1.1,1), xytext=(1.2,0.8), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.annotate(f"Q1 - 1.5IQR", xy=(1.1,0), xytext=(1.2,-0.2), arrowprops=dict(arrowstyle="-", color="black",), ha="left") + ax_in.set_yticklabels([]) + ax_in.set_xticklabels([]) ax1.set_title(title) ax1.set_xlabel("fraction of prefetched files in cache",color="black") hitrateticks = [x*0.1 for x in range(0,11)] @@ -262,11 +274,11 @@ def run(args=parser.parse_args()): else: suffix = args.suffix plotBoxes(data, - sites=sites, - title=args.scenario, - emptyBoxes=False, - plot_dir=out_dir, - suffix=suffix) + sites=sites, + title=args.scenario, + emptyBoxes=False, + plot_dir=out_dir, + suffix=suffix) if __name__ == "__main__": diff --git a/tools/prefetchScan.sh b/tools/prefetchScan.sh index 6a50daa6..c89a2248 100755 --- a/tools/prefetchScan.sh +++ b/tools/prefetchScan.sh @@ -26,13 +26,15 @@ action() { local SCENARIO="prefetchScanScaled100" # local SCENARIO="prefetchScanScaled1000" + local DUPLICATIONS=1 + local OUTDIR="$parent/tmp/outputs/WLCG" if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR fi - for prefetchrate in $(LANG=en_US seq 0.0 0.1 1.0) - do + for prefetchrate in $(LANG=en seq 0.0 0.1 1.0) + do dc-sim --platform "$PLATFORM" \ --hitrate ${prefetchrate} \ --xrd-blocksize $XRD_BLOCKSIZE \ @@ -41,6 +43,7 @@ action() { --storage-buffer-size $STORAGE_BUFFER_SIZE \ --cache-scope network \ --no-caching \ + --duplications $DUPLICATIONS \ --workload-configurations $WORKLOADS #\ # --no-streaming \ # --wrench-full-log