From a8bc00f3d24dcc1ff747cec6cf23effc9b263c25 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 30 Jan 2025 15:44:44 +0000 Subject: [PATCH 1/3] Add a yotal_iodepth option to be slit over total volumes Signed-off-by: Chris Harris(harriscr@uk.ibm.com --- benchmark/librbdfio.py | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/benchmark/librbdfio.py b/benchmark/librbdfio.py index 44b9b660..8fffaee7 100644 --- a/benchmark/librbdfio.py +++ b/benchmark/librbdfio.py @@ -9,6 +9,7 @@ import settings import monitoring +from typing import Optional from .benchmark import Benchmark logger = logging.getLogger("cbt") @@ -50,6 +51,14 @@ def __init__(self, archive_dir, cluster, config): self.rate_iops = config.get('rate_iops', None) self.fio_out_format = config.get('fio_out_format', 'json,normal') self.data_pool = None + + self._ioddepth_per_volume: dict[int, int] = {} + total_iodepth: Optional[str] = config.get("total_iodepth", None) + if total_iodepth is not None: + self._ioddepth_per_volume = self._calculate_iodepth_per_volume( + int(self.volumes_per_client), int(total_iodepth) + ) + # use_existing_volumes needs to be true to set the pool and rbd names self.use_existing_volumes = bool(config.get('use_existing_volumes', False)) self.no_sudo = bool(config.get('no_sudo', False)) @@ -244,7 +253,7 @@ def run(self): self.analyze(self.out_dir) - def mkfiocmd(self, volnum): + def mkfiocmd(self, volnum: int) -> str: """ Construct a FIO cmd (note the shell interpolation for the host executing FIO). @@ -257,7 +266,7 @@ def mkfiocmd(self, volnum): logger.debug('Using rbdname %s', rbdname) out_file = f'{self.run_dir}/output.{volnum:d}' - fio_cmd = '' + fio_cmd: str = '' if not self.no_sudo: fio_cmd = 'sudo ' fio_cmd += '%s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0' % (self.cmd_path, self.pool_name, rbdname) @@ -274,7 +283,12 @@ def mkfiocmd(self, volnum): fio_cmd += ' --numjobs=%s' % self.numjobs fio_cmd += ' --direct=1' fio_cmd += ' --bs=%dB' % self.op_size - fio_cmd += ' --iodepth=%d' % self.iodepth + + iodepth: str = f"{self.iodepth}" + if self._ioddepth_per_volume != {}: + iodepth = f"{self._ioddepth_per_volume[volnum]}" + + fio_cmd += ' --iodepth=%s' % iodepth fio_cmd += ' --end_fsync=%d' % self.end_fsync # if self.vol_size: # fio_cmd += ' -- size=%dM' % self.vol_size @@ -401,6 +415,41 @@ def analyze(self, out_dir): logger.info('Convert results to json format.') self.parse(out_dir) + def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_iodepth: int) -> dict[int, int]: + """ + Given the total desired iodepth and the number of volumes from the + configuration yaml file, calculate the iodepth for each volume + + If the iodepth specified in total_iodepth is too small to allow + an iodepth of 1 per volume, then reduce the number of volumes + used to allow an iodepth of 1 per volume. + """ + queue_depths: dict[int, int] = {} + + if number_of_volumes > total_desired_iodepth: + logger.warning( + "The total iodepth requested: %s is less than 1 per volume (%s)", + total_desired_iodepth, + number_of_volumes, + ) + logger.warning( + "Number of volumes per client will be reduced from %s to %s", number_of_volumes, total_desired_iodepth + ) + number_of_volumes = total_desired_iodepth + self.volumes_per_client = number_of_volumes + + iodepth_per_volume: int = total_desired_iodepth // number_of_volumes + remainder: int = total_desired_iodepth % number_of_volumes + + for volume_id in range(number_of_volumes): + iodepth: int = iodepth_per_volume + + if remainder > 0: + iodepth += 1 + remainder -= 1 + queue_depths[volume_id] = iodepth + + return queue_depths def __str__(self): return "%s\n%s\n%s" % (self.run_dir, self.out_dir, super(LibrbdFio, self).__str__()) From dc1a795a3e4592e22c143680a49d1509a5a48c68 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 6 Feb 2025 14:25:53 +0000 Subject: [PATCH 2/3] Workload fixes for total_iodepth Signed-off-by: CHris Harris(harriscr@uk.ibm.com) --- benchmark/librbdfio.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/benchmark/librbdfio.py b/benchmark/librbdfio.py index 8fffaee7..0ee13343 100644 --- a/benchmark/librbdfio.py +++ b/benchmark/librbdfio.py @@ -172,7 +172,18 @@ def run_workloads(self): enable_monitor = bool(test['monitor']) # TODO: simplify this loop to have a single iterator for general queu depth for job in test['numjobs']: - for iod in test['iodepth']: + iodepth: list[str] = [] + use_total_iodepth: bool = False + if "total_iodepth" in test.keys(): + iodepth = test["total_iodepth"] + use_total_iodepth = True + else: + iodepth = test["iodepth"] + for iod in iodepth: + if use_total_iodepth: + self._ioddepth_per_volume = self._calculate_iodepth_per_volume( + int(self.volumes_per_client), int(iod) + ) self.mode = test['mode'] if 'op_size' in test: self.op_size = test['op_size'] @@ -183,7 +194,10 @@ def run_workloads(self): f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) common.make_remote_dir(self.run_dir) - for i in range(self.volumes_per_client): + number_of_volumes: int = int(self.volumes_per_client) + if use_total_iodepth: + number_of_volumes = len(self._ioddepth_per_volume.keys()) + for i in range(number_of_volumes): fio_cmd = self.mkfiocmd(i) p = common.pdsh(settings.getnodes('clients'), fio_cmd) ps.append(p) @@ -235,7 +249,10 @@ def run(self): monitoring.start(self.run_dir) logger.info('Running rbd fio %s test.', self.mode) ps = [] - for i in range(self.volumes_per_client): + number_of_volumes: int = int(self.volumes_per_client) + if self._ioddepth_per_volume != {}: + number_of_volumes = len(self._ioddepth_per_volume.keys()) + for i in range(number_of_volumes): fio_cmd = self.mkfiocmd(i) p = common.pdsh(settings.getnodes('clients'), fio_cmd) ps.append(p) @@ -436,7 +453,6 @@ def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_io "Number of volumes per client will be reduced from %s to %s", number_of_volumes, total_desired_iodepth ) number_of_volumes = total_desired_iodepth - self.volumes_per_client = number_of_volumes iodepth_per_volume: int = total_desired_iodepth // number_of_volumes remainder: int = total_desired_iodepth % number_of_volumes From e048c28dd549128e97c66dfa9e9d5f8fd851a4dd Mon Sep 17 00:00:00 2001 From: lee-j-sanders Date: Mon, 17 Feb 2025 16:19:53 +0000 Subject: [PATCH 3/3] Allow time within a test to override global time setting in YAML and allow multiple mixed read/write workloads within a YAML This commit only changes the functionality of workloads in CBT. This commit adds two pieces of functionality: 1) Workloads with randrw only lets you specify 1 mixed workload within a YAML as the same directory name was used for subsequent randrw workloads, thus overwriting all the previous resuilts in the archive directory with the later tests. This commit adds the readwrite ratio into the directory name if the mode is randrw. The directory name for 100% read and 100% write tests are unaffected. 2) Prior to this commit, all workloads inherited "time" from the outer/global part of the YAML. This meant you couldn't set different time for each test within the "workloads". Typically for preconditioning you'd want to precondition for a set amount of time ( 600 seconds - 10minutes), then set the workload time to be 120 seconds (2 minutes). If you do not set a "time" within the workload, the time for that specific test will be inherited from the global "time" within the YAML. Signed-off-by: lee-j-sanders --- benchmark/librbdfio.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/benchmark/librbdfio.py b/benchmark/librbdfio.py index 0ee13343..9d81a9ca 100644 --- a/benchmark/librbdfio.py +++ b/benchmark/librbdfio.py @@ -99,6 +99,7 @@ def backup_global_fio_options(self): Backup/copy the FIO global options into a dictionary """ self.global_fio_options['time_based'] = self.time_based + self.global_fio_options['time'] = self.time self.global_fio_options['ramp'] = self.ramp self.global_fio_options['iodepth'] = self.iodepth self.global_fio_options['numjobs'] = self.numjobs @@ -124,7 +125,7 @@ def restore_global_fio_options(self): self.log_avg_msec = self.global_fio_options['log_avg_msec'] self.op_size = self.global_fio_options['op_size'] self.time_based = self.global_fio_options['time_based'] - + self.time = self.global_fio_options['time'] def exists(self): """ @@ -184,14 +185,28 @@ def run_workloads(self): self._ioddepth_per_volume = self._calculate_iodepth_per_volume( int(self.volumes_per_client), int(iod) ) + + self.time = test['time'] self.mode = test['mode'] if 'op_size' in test: self.op_size = test['op_size'] - self.mode = test['mode'] self.numjobs = job self.iodepth = iod - self.run_dir = ( f'{self.base_run_dir}/{self.mode}_{int(self.op_size)}/' - f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) + + # Needed to allow for different mixed ratio results with the same block size, we + # store the ratio within the directory name. Otherwise workloads would only support + # 1 mixed workload for a given block size. For 100% read, 100% write don't need to + # store the read/write ratio. + + if self.mode == 'randrw': + self.rwmixread = test['rwmixread'] + self.rwmixwrite = 100 - self.rwmixread + self.run_dir = ( f'{self.base_run_dir}/{self.mode}{self.rwmixread}{self.rwmixwrite}_{int(self.op_size)}/' + f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) + else: + self.run_dir = ( f'{self.base_run_dir}/{self.mode}_{int(self.op_size)}/' + f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) + common.make_remote_dir(self.run_dir) number_of_volumes: int = int(self.volumes_per_client)