From 32260f8965d4ab6bc220d265688fb8d356b4c4ca Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 8 May 2021 11:25:11 +0200 Subject: [PATCH 01/12] refactor: fix typo and magic number, rearrange for readability --- tcav/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tcav/utils.py b/tcav/utils.py index 6acbb6946d..27ebf213e6 100644 --- a/tcav/utils.py +++ b/tcav/utils.py @@ -25,6 +25,8 @@ "val_directional_dirs_std", "note", "alpha", "bottleneck" ] +MAX_RANDOM_EXPERIMENTS = 100 + def create_session(timeout=10000, interactive=True): """Create a tf session for the model. @@ -87,17 +89,15 @@ def get_random_concept(i): # if only one element was given, this is to test with random. if len(concept_set) == 1: i = 0 - while len(new_pairs_to_test_t) < min(100, num_random_exp): + while len(new_pairs_to_test_t) < min(MAX_RANDOM_EXPERIMENTS, num_random_exp): # fixme: why max 100? magic number # make sure that we are not comparing the same thing to each other. - if concept_set[0] != get_random_concept( - i) and random_counterpart != get_random_concept(i): - new_pairs_to_test_t.append( - (target, [concept_set[0], get_random_concept(i)])) + if concept_set[0] != get_random_concept(i) and random_counterpart != get_random_concept(i): + new_pairs_to_test_t.append((target, [concept_set[0], get_random_concept(i)])) i += 1 elif len(concept_set) > 1: new_pairs_to_test_t.append((target, concept_set)) else: - tf.compat.v1.logging.info('PAIR NOT PROCCESSED') + tf.compat.v1.logging.info('PAIR NOT PROCESSED') new_pairs_to_test.extend(new_pairs_to_test_t) all_concepts = list(set(flatten([cs + [tc] for tc, cs in new_pairs_to_test]))) From 753e7dadf6a0b7076a22b6b756d430504c46bf0e Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 8 May 2021 12:04:58 +0200 Subject: [PATCH 02/12] add: test for max_random_experiments limit --- tcav/utils_test.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tcav/utils_test.py b/tcav/utils_test.py index 99109f2934..56dd510f7e 100644 --- a/tcav/utils_test.py +++ b/tcav/utils_test.py @@ -16,7 +16,7 @@ from __future__ import print_function from tensorflow.python.platform import googletest from tcav.tcav_results.results_pb2 import Result, Results -from tcav.utils import flatten, process_what_to_run_expand, process_what_to_run_concepts, process_what_to_run_randoms, results_to_proto +from tcav.utils import flatten, process_what_to_run_expand, process_what_to_run_concepts, process_what_to_run_randoms, results_to_proto, MAX_RANDOM_EXPERIMENTS class UtilsTest(googletest.TestCase): @@ -42,6 +42,21 @@ def test_process_what_to_run_expand(self): ('t1', ['c2', 'random500_0']), ('t1', ['c2', 'random500_1'])])) + def test_process_what_to_run_expand_max_experiments(self): + num_random_exp = MAX_RANDOM_EXPERIMENTS + 2 # just add some number to be bigger than max value + all_concepts, pairs_to_test = process_what_to_run_expand( + self.pair_to_test_one_concept, + num_random_exp=num_random_exp) + + concepts = ['c1', 'c2'] + expected_max_size = MAX_RANDOM_EXPERIMENTS + self.assertEqual( + sorted(all_concepts), + sorted(['t1'] + concepts + [f'random500_{i}' for i in range(expected_max_size)])) + + # expect max experiments = num_concepts * expected_max_size + self.assertEqual(len(pairs_to_test), len(concepts*MAX_RANDOM_EXPERIMENTS)) + def test_process_what_to_run_expand_specify_dirs(self): all_concepts, pairs_to_test = process_what_to_run_expand( self.pair_to_test_one_concept, From c8003bcd12cfa80e1c955a73abb9d8f42d184711 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 8 May 2021 12:05:49 +0200 Subject: [PATCH 03/12] cleanup: relative tcav test --- tcav/tcav_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcav/tcav_test.py b/tcav/tcav_test.py index fd6dc46ad3..7945c2f2bf 100644 --- a/tcav/tcav_test.py +++ b/tcav/tcav_test.py @@ -221,8 +221,7 @@ def test__process_what_to_run_expand_relative_tcav(self): self.act_gen, [self.hparams['alpha']], random_concepts=concepts_relative) - self.mytcav_random_counterpart._process_what_to_run_expand( - num_random_exp=2, random_concepts=concepts_relative) + self.assertEqual(sorted(my_relative_tcav.all_concepts), sorted(['t1', 'c1', 'c2', 'c3'])) self.assertEqual(sorted(my_relative_tcav.pairs_to_test), From 0a0544aabf02bd25d725293f599f6e6c44652118 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 8 May 2021 15:09:59 +0200 Subject: [PATCH 04/12] refactor: make code more readable --- tcav/activation_generator.py | 21 +++++++++++---------- tcav/tcav.py | 32 +++++++++++++++----------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/tcav/activation_generator.py b/tcav/activation_generator.py index 21671f962e..541c4cd5f2 100644 --- a/tcav/activation_generator.py +++ b/tcav/activation_generator.py @@ -69,24 +69,25 @@ def process_and_load_activations(self, bottleneck_names, concepts): for concept in concepts: if concept not in acts: acts[concept] = {} + for bottleneck_name in bottleneck_names: - acts_path = os.path.join(self.acts_dir, 'acts_{}_{}'.format( - concept, bottleneck_name)) if self.acts_dir else None + acts_path = os.path.join(self.acts_dir, f'acts_{concept}_{bottleneck_name}') if self.acts_dir else None + if acts_path and tf.io.gfile.exists(acts_path): + # load activations from file with tf.io.gfile.GFile(acts_path, 'rb') as f: - acts[concept][bottleneck_name] = np.load( - f, allow_pickle=True).squeeze() - tf.compat.v1.logging.info('Loaded {} shape {}'.format( - acts_path, acts[concept][bottleneck_name].shape)) + acts[concept][bottleneck_name] = np.load(f, allow_pickle=True).squeeze() + tf.compat.v1.logging.info(f'Loaded {acts_path} shape {acts[concept][bottleneck_name].shape}') else: - acts[concept][bottleneck_name] = self.get_activations_for_concept( - concept, bottleneck_name) + # compute and save activations + acts[concept][bottleneck_name] = self.get_activations_for_concept(concept, bottleneck_name) + if acts_path: - tf.compat.v1.logging.info( - '{} does not exist, Making one...'.format(acts_path)) + tf.compat.v1.logging.info(f'{acts_path} does not exist, Making one...') tf.io.gfile.mkdir(os.path.dirname(acts_path)) with tf.io.gfile.GFile(acts_path, 'w') as f: np.save(f, acts[concept][bottleneck_name], allow_pickle=False) + return acts diff --git a/tcav/tcav.py b/tcav/tcav.py index 6fb2cf2cde..6d0183b3d9 100644 --- a/tcav/tcav.py +++ b/tcav/tcav.py @@ -210,22 +210,20 @@ def run(self, num_workers=10, run_parallel=False, overwrite=False, return_proto= # pool worker 50 seems to work. tf.compat.v1.logging.info('running %s params' % len(self.params)) results = [] - now = time.time() + begin = time.time() if run_parallel: pool = multiprocessing.Pool(num_workers) - for i, res in enumerate(pool.imap( - lambda p: self._run_single_set( - p, overwrite=overwrite, run_parallel=run_parallel), - self.params), 1): + results = pool.imap(lambda p: self._run_single_set(p, overwrite=overwrite, run_parallel=run_parallel), self.params) + for i, res in enumerate(results, 1): tf.compat.v1.logging.info('Finished running param %s of %s' % (i, len(self.params))) - results.append(res) pool.close() else: - for i, param in enumerate(self.params): + for i, param in enumerate(self.params, 1): tf.compat.v1.logging.info('Running param %s of %s' % (i, len(self.params))) results.append(self._run_single_set(param, overwrite=overwrite, run_parallel=run_parallel)) - tf.compat.v1.logging.info('Done running %s params. Took %s seconds...' % (len( - self.params), time.time() - now)) + + tf.compat.v1.logging.info('Done running %s params. Took %s seconds...' % (len(self.params), time.time() - begin)) + if return_proto: return utils.results_to_proto(results) else: @@ -336,14 +334,14 @@ def _process_what_to_run_expand(self, num_random_exp=100, random_concepts=None): # take away 1 random experiment if the random counterpart already in random concepts # take away 1 random experiment if computing Relative TCAV - all_concepts_concepts, pairs_to_run_concepts = ( - utils.process_what_to_run_expand( - utils.process_what_to_run_concepts(target_concept_pairs), - self.random_counterpart, - num_random_exp=num_random_exp - - (1 if random_concepts and self.random_counterpart in random_concepts - else 0) - (1 if self.relative_tcav else 0), - random_concepts=random_concepts)) + + random_exp_count = num_random_exp - (1 if random_concepts and self.random_counterpart in random_concepts else 0) - (1 if self.relative_tcav else 0) + + all_concepts_concepts, pairs_to_run_concepts = utils.process_what_to_run_expand( + utils.process_what_to_run_concepts(target_concept_pairs), + self.random_counterpart, + num_random_exp=random_exp_count, + random_concepts=random_concepts) pairs_to_run_randoms = [] all_concepts_randoms = [] From 48fac0001d6270cc4aacfed4601db4e4ffd16701 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 8 May 2021 20:10:31 +0200 Subject: [PATCH 05/12] fix: relative tcav computation and plot - the paper states in section 3.6 for relative tcav, construct a positive and negative set, e.g., for concepts P_dot, P_stripe, P_mesh this would look like, P_stripe (positive) and {P_dot U p_mesh} (negative). However, the code was giving sets like, P_stripe (pos), P_dot (neg). Now, the complement set is used for relative tcav as described in the paper. - when computing relative tcav, plot function failed because concepts are also 'random' concepts and is_random_concept(concept) is always true, therefore. Now, plot for relative tcav handles 'random' concepts correctly. --- tcav/activation_generator.py | 33 ++++++++++++++++-------- tcav/tcav.py | 16 ++++++++---- tcav/tcav_test.py | 9 +++---- tcav/utils.py | 36 ++++++++++++++++++++------ tcav/utils_plot.py | 49 ++++++++++++++++++++++++++++++------ 5 files changed, 105 insertions(+), 38 deletions(-) diff --git a/tcav/activation_generator.py b/tcav/activation_generator.py index 541c4cd5f2..4e9119acb1 100644 --- a/tcav/activation_generator.py +++ b/tcav/activation_generator.py @@ -25,12 +25,13 @@ import PIL.Image import six import tensorflow as tf +from .utils import CONCEPT_SEPARATOR class ActivationGeneratorInterface(six.with_metaclass(ABCMeta, object)): """Interface for an activation generator for a model""" @abstractmethod - def process_and_load_activations(self, bottleneck_names, concepts): + def process_and_load_activations(self, bottleneck_names, concepts, relative_tcav=False): pass @abstractmethod @@ -53,15 +54,15 @@ def get_model(self): def get_examples_for_concept(self, concept): pass - def get_activations_for_concept(self, concept, bottleneck): - examples = self.get_examples_for_concept(concept) + def get_activations_for_concept(self, concept, bottleneck, relative_tcav=False): + examples = self.get_examples_for_concept(concept, relative_tcav) return self.get_activations_for_examples(examples, bottleneck) def get_activations_for_examples(self, examples, bottleneck): acts = self.model.run_examples(examples, bottleneck) return self.model.reshape_activations(acts).squeeze() - def process_and_load_activations(self, bottleneck_names, concepts): + def process_and_load_activations(self, bottleneck_names, concepts, relative_tcav=False): acts = {} if self.acts_dir and not tf.io.gfile.exists(self.acts_dir): tf.io.gfile.makedirs(self.acts_dir) @@ -70,8 +71,10 @@ def process_and_load_activations(self, bottleneck_names, concepts): if concept not in acts: acts[concept] = {} + # convert concept name to filesystem friendly version + concept_filename = concept.replace(CONCEPT_SEPARATOR, "-") for bottleneck_name in bottleneck_names: - acts_path = os.path.join(self.acts_dir, f'acts_{concept}_{bottleneck_name}') if self.acts_dir else None + acts_path = os.path.join(self.acts_dir, f'acts_{concept_filename}_{bottleneck_name}') if self.acts_dir else None if acts_path and tf.io.gfile.exists(acts_path): # load activations from file @@ -80,7 +83,7 @@ def process_and_load_activations(self, bottleneck_names, concepts): tf.compat.v1.logging.info(f'Loaded {acts_path} shape {acts[concept][bottleneck_name].shape}') else: # compute and save activations - acts[concept][bottleneck_name] = self.get_activations_for_concept(concept, bottleneck_name) + acts[concept][bottleneck_name] = self.get_activations_for_concept(concept, bottleneck_name, relative_tcav) if acts_path: tf.compat.v1.logging.info(f'{acts_path} does not exist, Making one...') @@ -111,11 +114,19 @@ def __init__(self, super(ImageActivationGenerator, self).__init__(model, acts_dir, max_examples) - def get_examples_for_concept(self, concept): - concept_dir = os.path.join(self.source_dir, concept) - img_paths = [ - os.path.join(concept_dir, d) for d in tf.io.gfile.listdir(concept_dir) - ] + def get_examples_for_concept(self, concept_names, relative_cav=False): + if relative_cav: + concepts = concept_names.split(CONCEPT_SEPARATOR) + else: + concepts = [concept_names] + + img_paths = [] + for concept in concepts: + concept_dir = os.path.join(self.source_dir, concept) + img_paths.append( + [os.path.join(concept_dir, d) for d in tf.io.gfile.listdir(concept_dir)] + ) + img_paths = np.asarray(img_paths).flatten() imgs = self.load_images_from_files( img_paths, self.max_examples, shape=self.model.get_image_shape()[:2]) return imgs diff --git a/tcav/tcav.py b/tcav/tcav.py index 6d0183b3d9..6ed947c05f 100644 --- a/tcav/tcav.py +++ b/tcav/tcav.py @@ -253,7 +253,7 @@ def _run_single_set(self, param, overwrite=False, run_parallel=False): # Get acts acts = activation_generator.process_and_load_activations( - [bottleneck], concepts + [target_class]) + [bottleneck], concepts + [target_class], self.relative_tcav) # Get CAVs cav_hparams = CAV.default_hparams() cav_hparams['alpha'] = alpha @@ -338,10 +338,12 @@ def _process_what_to_run_expand(self, num_random_exp=100, random_concepts=None): random_exp_count = num_random_exp - (1 if random_concepts and self.random_counterpart in random_concepts else 0) - (1 if self.relative_tcav else 0) all_concepts_concepts, pairs_to_run_concepts = utils.process_what_to_run_expand( - utils.process_what_to_run_concepts(target_concept_pairs), + utils.process_what_to_run_concepts(target_concept_pairs, self.relative_tcav), self.random_counterpart, num_random_exp=random_exp_count, - random_concepts=random_concepts) + random_concepts=random_concepts, + relative_tcav=self.relative_tcav, + ) pairs_to_run_randoms = [] all_concepts_randoms = [] @@ -351,7 +353,7 @@ def get_random_concept(i): return (random_concepts[i] if random_concepts else 'random500_{}'.format(i)) - if self.random_counterpart is None: + if self.random_counterpart is None and not self.relative_tcav: # TODO random500_1 vs random500_0 is the same as 1 - (random500_0 vs random500_1) for i in range(num_random_exp): all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = ( @@ -364,7 +366,7 @@ def get_random_concept(i): pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) all_concepts_randoms.extend(all_concepts_randoms_tmp) - else: + elif not self.relative_tcav: # run only random_counterpart as the positve set for random experiments all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = ( utils.process_what_to_run_expand( @@ -378,6 +380,10 @@ def get_random_concept(i): pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) all_concepts_randoms.extend(all_concepts_randoms_tmp) + else: + # relative cav, no random concepts + pass + self.all_concepts = list(set(all_concepts_concepts + all_concepts_randoms)) self.pairs_to_test = pairs_to_run_concepts if self.relative_tcav else pairs_to_run_concepts + pairs_to_run_randoms diff --git a/tcav/tcav_test.py b/tcav/tcav_test.py index 7945c2f2bf..3d6929b29d 100644 --- a/tcav/tcav_test.py +++ b/tcav/tcav_test.py @@ -225,12 +225,9 @@ def test__process_what_to_run_expand_relative_tcav(self): self.assertEqual(sorted(my_relative_tcav.all_concepts), sorted(['t1', 'c1', 'c2', 'c3'])) self.assertEqual(sorted(my_relative_tcav.pairs_to_test), - sorted([('t1',['c1', 'c2']), - ('t1',['c1', 'c3']), - ('t1',['c2', 'c1']), - ('t1',['c2', 'c3']), - ('t1',['c3', 'c1']), - ('t1',['c3', 'c2']), + sorted([('t1',['c1', 'c2:c3']), + ('t1',['c2', 'c1:c3']), + ('t1',['c3', 'c1:c2']), ])) def test_get_params(self): diff --git a/tcav/utils.py b/tcav/utils.py index 27ebf213e6..5c2da8cf27 100644 --- a/tcav/utils.py +++ b/tcav/utils.py @@ -26,6 +26,7 @@ ] MAX_RANDOM_EXPERIMENTS = 100 +CONCEPT_SEPARATOR = ':' def create_session(timeout=10000, interactive=True): @@ -56,7 +57,8 @@ def flatten(nested_list): def process_what_to_run_expand(pairs_to_test, random_counterpart=None, num_random_exp=100, - random_concepts=None): + random_concepts=None, + relative_tcav=False): """Get concept vs. random or random vs. random pairs to run. Given set of target, list of concept pairs, expand them to include @@ -95,17 +97,30 @@ def get_random_concept(i): new_pairs_to_test_t.append((target, [concept_set[0], get_random_concept(i)])) i += 1 elif len(concept_set) > 1: - new_pairs_to_test_t.append((target, concept_set)) + if relative_tcav: + for concept in concept_set: + negative_set = CONCEPT_SEPARATOR.join(np.setdiff1d(concept_set, concept)) # c2:c3:... excluding c_i where i=j + new_pairs_to_test_t.append((target, [concept, negative_set])) + else: + new_pairs_to_test_t.append((target, concept_set)) else: tf.compat.v1.logging.info('PAIR NOT PROCESSED') new_pairs_to_test.extend(new_pairs_to_test_t) - all_concepts = list(set(flatten([cs + [tc] for tc, cs in new_pairs_to_test]))) + if relative_tcav: + all_concepts = [] + for tc, cs in new_pairs_to_test: + all_concepts.append([tc]) + for c in cs: + all_concepts.append(c.split(':')) + all_concepts = list(set(flatten(all_concepts))) + else: + all_concepts = list(set(flatten([cs + [tc] for tc, cs in new_pairs_to_test]))) return all_concepts, new_pairs_to_test -def process_what_to_run_concepts(pairs_to_test): +def process_what_to_run_concepts(pairs_to_test, relative_tcav=False): """Process concepts and pairs to test. Args: @@ -124,10 +139,15 @@ def process_what_to_run_concepts(pairs_to_test): """ pairs_for_sstesting = [] - # prepare pairs for concpet vs random. - for pair in pairs_to_test: - for concept in pair[1]: - pairs_for_sstesting.append([pair[0], [concept]]) + if relative_tcav: + # keep things as is + return pairs_to_test + else: + # prepare pairs for concpet vs random. + for target, concept_set in pairs_to_test: + for concept in concept_set: + pairs_for_sstesting.append([target, [concept]]) + return pairs_for_sstesting diff --git a/tcav/utils_plot.py b/tcav/utils_plot.py index 2f739a827f..e6f4b78c0e 100644 --- a/tcav/utils_plot.py +++ b/tcav/utils_plot.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import sys from scipy.stats import ttest_ind import numpy as np import matplotlib.pyplot as plt @@ -24,7 +25,7 @@ # helper function to output plot and write summary data def plot_results(results, random_counterpart=None, random_concepts=None, num_random_exp=100, - min_p_val=0.05): + min_p_val=0.05, fig_path=None, result_path=None, relative_tcav=False): """Helper function to organize results. When run in a notebook, outputs a matplotlib bar plot of the TCAV scores for all bottlenecks for each concept, replacing the @@ -51,8 +52,14 @@ def is_random_concept(concept): else: return 'random500_' in concept + result_outfile = None + try: + result_outfile = open(result_path, 'w') + except OSError: + result_outfile = sys.stdout + # print class, it will be the same for all - print("Class =", results[0]['target_class']) + print("Class =", results[0]['target_class'], file=result_outfile) # prepare data # dict with keys of concepts containing dict with bottlenecks @@ -86,7 +93,7 @@ def is_random_concept(concept): # if not random if not is_random_concept(concept): - print(" ", "Concept =", concept) + print(" ", "Concept =", concept, file=result_outfile) plot_concepts.append(concept) for bottleneck in result_summary[concept]: @@ -103,7 +110,6 @@ def is_random_concept(concept): plot_data[bottleneck]['bn_vals'].append(0.01) plot_data[bottleneck]['bn_stds'].append(0) plot_data[bottleneck]['significant'].append(False) - else: plot_data[bottleneck]['bn_vals'].append(np.mean(i_ups)) plot_data[bottleneck]['bn_stds'].append(np.std(i_ups)) @@ -114,10 +120,30 @@ def is_random_concept(concept): bottleneck, np.mean(i_ups), np.std(i_ups), np.mean(random_i_ups[bottleneck]), np.std(random_i_ups[bottleneck]), p_val, - "not significant" if p_val > min_p_val else "significant")) + "not significant" if p_val > min_p_val else "significant"), file=result_outfile) + # if relative tcav + elif relative_tcav: + # relative TCAV; c_1 -> c_2 U c_3; c_2 -> c_1 U c_3, ... + print(" ", "Concept =", concept, file=result_outfile) + plot_concepts.append(concept) + + for bottleneck in result_summary[concept]: + i_ups = [item['i_up'] for item in result_summary[concept][bottleneck]] + + if bottleneck not in plot_data: + plot_data[bottleneck] = {'bn_vals': [], 'bn_stds': [], 'significant': []} + + plot_data[bottleneck]['bn_vals'].append(np.mean(i_ups)) + plot_data[bottleneck]['bn_stds'].append(np.std(i_ups)) + plot_data[bottleneck]['significant'].append(True) + + print(3 * " ", "Bottleneck =", "%s. TCAV Score = %.2f (+- %.2f)" % ( + bottleneck, np.mean(i_ups), np.std(i_ups)), file=result_outfile) # subtract number of random experiments - if random_counterpart: + if relative_tcav: + num_concepts = len(result_summary) + elif random_counterpart: num_concepts = len(result_summary) - 1 elif random_concepts: num_concepts = len(result_summary) - len(random_concepts) @@ -146,7 +172,9 @@ def is_random_concept(concept): ax.text(index[j] + i * bar_width - 0.1, 0.01, "*", fontdict = {'weight': 'bold', 'size': 16, 'color': bar.patches[0].get_facecolor()}) - print (plot_data) + + print(plot_data, file=result_outfile) + # set properties ax.set_title('TCAV Scores for each concept and bottleneck') ax.set_ylabel('TCAV Score') @@ -154,4 +182,9 @@ def is_random_concept(concept): ax.set_xticklabels(plot_concepts) ax.legend() fig.tight_layout() - plt.show() + + # show or save + if fig_path is None: + plt.show() + else: + plt.savefig(fig_path, dpi=200) From a5cebc74588375837102be0555b13806c53812b2 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Fri, 14 May 2021 09:18:55 +0200 Subject: [PATCH 06/12] refactor: rename 'relative_tcav' to 'is_relative_tcav' --- tcav/activation_generator.py | 14 +++++++------- tcav/tcav.py | 16 ++++++++-------- tcav/utils.py | 10 +++++----- tcav/utils_plot.py | 6 +++--- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tcav/activation_generator.py b/tcav/activation_generator.py index 4e9119acb1..9928d52f51 100644 --- a/tcav/activation_generator.py +++ b/tcav/activation_generator.py @@ -31,7 +31,7 @@ class ActivationGeneratorInterface(six.with_metaclass(ABCMeta, object)): """Interface for an activation generator for a model""" @abstractmethod - def process_and_load_activations(self, bottleneck_names, concepts, relative_tcav=False): + def process_and_load_activations(self, bottleneck_names, concepts, is_relative_tcav=False): pass @abstractmethod @@ -54,15 +54,15 @@ def get_model(self): def get_examples_for_concept(self, concept): pass - def get_activations_for_concept(self, concept, bottleneck, relative_tcav=False): - examples = self.get_examples_for_concept(concept, relative_tcav) + def get_activations_for_concept(self, concept, bottleneck, is_relative_tcav=False): + examples = self.get_examples_for_concept(concept, is_relative_tcav) return self.get_activations_for_examples(examples, bottleneck) def get_activations_for_examples(self, examples, bottleneck): acts = self.model.run_examples(examples, bottleneck) return self.model.reshape_activations(acts).squeeze() - def process_and_load_activations(self, bottleneck_names, concepts, relative_tcav=False): + def process_and_load_activations(self, bottleneck_names, concepts, is_relative_tcav=False): acts = {} if self.acts_dir and not tf.io.gfile.exists(self.acts_dir): tf.io.gfile.makedirs(self.acts_dir) @@ -83,7 +83,7 @@ def process_and_load_activations(self, bottleneck_names, concepts, relative_tcav tf.compat.v1.logging.info(f'Loaded {acts_path} shape {acts[concept][bottleneck_name].shape}') else: # compute and save activations - acts[concept][bottleneck_name] = self.get_activations_for_concept(concept, bottleneck_name, relative_tcav) + acts[concept][bottleneck_name] = self.get_activations_for_concept(concept, bottleneck_name, is_relative_tcav) if acts_path: tf.compat.v1.logging.info(f'{acts_path} does not exist, Making one...') @@ -114,8 +114,8 @@ def __init__(self, super(ImageActivationGenerator, self).__init__(model, acts_dir, max_examples) - def get_examples_for_concept(self, concept_names, relative_cav=False): - if relative_cav: + def get_examples_for_concept(self, concept_names, is_relative_tcav=False): + if is_relative_tcav: concepts = concept_names.split(CONCEPT_SEPARATOR) else: concepts = [concept_names] diff --git a/tcav/tcav.py b/tcav/tcav.py index 6ed947c05f..10e9c64c71 100644 --- a/tcav/tcav.py +++ b/tcav/tcav.py @@ -178,7 +178,7 @@ def __init__(self, self.model_to_run = self.mymodel.model_name self.sess = sess self.random_counterpart = random_counterpart - self.relative_tcav = (random_concepts is not None) and (set(concepts) == set(random_concepts)) + self.is_relative_tcav = (random_concepts is not None) and (set(concepts) == set(random_concepts)) if num_random_exp < 2: tf.compat.v1.logging.error('the number of random concepts has to be at least 2') @@ -253,7 +253,7 @@ def _run_single_set(self, param, overwrite=False, run_parallel=False): # Get acts acts = activation_generator.process_and_load_activations( - [bottleneck], concepts + [target_class], self.relative_tcav) + [bottleneck], concepts + [target_class], self.is_relative_tcav) # Get CAVs cav_hparams = CAV.default_hparams() cav_hparams['alpha'] = alpha @@ -335,14 +335,14 @@ def _process_what_to_run_expand(self, num_random_exp=100, random_concepts=None): # take away 1 random experiment if the random counterpart already in random concepts # take away 1 random experiment if computing Relative TCAV - random_exp_count = num_random_exp - (1 if random_concepts and self.random_counterpart in random_concepts else 0) - (1 if self.relative_tcav else 0) + random_exp_count = num_random_exp - (1 if random_concepts and self.random_counterpart in random_concepts else 0) - (1 if self.is_relative_tcav else 0) all_concepts_concepts, pairs_to_run_concepts = utils.process_what_to_run_expand( - utils.process_what_to_run_concepts(target_concept_pairs, self.relative_tcav), + utils.process_what_to_run_concepts(target_concept_pairs, self.is_relative_tcav), self.random_counterpart, num_random_exp=random_exp_count, random_concepts=random_concepts, - relative_tcav=self.relative_tcav, + is_relative_tcav=self.is_relative_tcav, ) pairs_to_run_randoms = [] @@ -353,7 +353,7 @@ def get_random_concept(i): return (random_concepts[i] if random_concepts else 'random500_{}'.format(i)) - if self.random_counterpart is None and not self.relative_tcav: + if self.random_counterpart is None and not self.is_relative_tcav: # TODO random500_1 vs random500_0 is the same as 1 - (random500_0 vs random500_1) for i in range(num_random_exp): all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = ( @@ -366,7 +366,7 @@ def get_random_concept(i): pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) all_concepts_randoms.extend(all_concepts_randoms_tmp) - elif not self.relative_tcav: + elif not self.is_relative_tcav: # run only random_counterpart as the positve set for random experiments all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = ( utils.process_what_to_run_expand( @@ -385,7 +385,7 @@ def get_random_concept(i): pass self.all_concepts = list(set(all_concepts_concepts + all_concepts_randoms)) - self.pairs_to_test = pairs_to_run_concepts if self.relative_tcav else pairs_to_run_concepts + pairs_to_run_randoms + self.pairs_to_test = pairs_to_run_concepts if self.is_relative_tcav else pairs_to_run_concepts + pairs_to_run_randoms def get_params(self): """Enumerate parameters for the run function. diff --git a/tcav/utils.py b/tcav/utils.py index 5c2da8cf27..2a7414f464 100644 --- a/tcav/utils.py +++ b/tcav/utils.py @@ -58,7 +58,7 @@ def process_what_to_run_expand(pairs_to_test, random_counterpart=None, num_random_exp=100, random_concepts=None, - relative_tcav=False): + is_relative_tcav=False): """Get concept vs. random or random vs. random pairs to run. Given set of target, list of concept pairs, expand them to include @@ -97,7 +97,7 @@ def get_random_concept(i): new_pairs_to_test_t.append((target, [concept_set[0], get_random_concept(i)])) i += 1 elif len(concept_set) > 1: - if relative_tcav: + if is_relative_tcav: for concept in concept_set: negative_set = CONCEPT_SEPARATOR.join(np.setdiff1d(concept_set, concept)) # c2:c3:... excluding c_i where i=j new_pairs_to_test_t.append((target, [concept, negative_set])) @@ -107,7 +107,7 @@ def get_random_concept(i): tf.compat.v1.logging.info('PAIR NOT PROCESSED') new_pairs_to_test.extend(new_pairs_to_test_t) - if relative_tcav: + if is_relative_tcav: all_concepts = [] for tc, cs in new_pairs_to_test: all_concepts.append([tc]) @@ -120,7 +120,7 @@ def get_random_concept(i): return all_concepts, new_pairs_to_test -def process_what_to_run_concepts(pairs_to_test, relative_tcav=False): +def process_what_to_run_concepts(pairs_to_test, is_relative_tcav=False): """Process concepts and pairs to test. Args: @@ -139,7 +139,7 @@ def process_what_to_run_concepts(pairs_to_test, relative_tcav=False): """ pairs_for_sstesting = [] - if relative_tcav: + if is_relative_tcav: # keep things as is return pairs_to_test else: diff --git a/tcav/utils_plot.py b/tcav/utils_plot.py index e6f4b78c0e..188b9f4cec 100644 --- a/tcav/utils_plot.py +++ b/tcav/utils_plot.py @@ -25,7 +25,7 @@ # helper function to output plot and write summary data def plot_results(results, random_counterpart=None, random_concepts=None, num_random_exp=100, - min_p_val=0.05, fig_path=None, result_path=None, relative_tcav=False): + min_p_val=0.05, fig_path=None, result_path=None, is_relative_tcav=False): """Helper function to organize results. When run in a notebook, outputs a matplotlib bar plot of the TCAV scores for all bottlenecks for each concept, replacing the @@ -122,7 +122,7 @@ def is_random_concept(concept): np.std(random_i_ups[bottleneck]), p_val, "not significant" if p_val > min_p_val else "significant"), file=result_outfile) # if relative tcav - elif relative_tcav: + elif is_relative_tcav: # relative TCAV; c_1 -> c_2 U c_3; c_2 -> c_1 U c_3, ... print(" ", "Concept =", concept, file=result_outfile) plot_concepts.append(concept) @@ -141,7 +141,7 @@ def is_random_concept(concept): bottleneck, np.mean(i_ups), np.std(i_ups)), file=result_outfile) # subtract number of random experiments - if relative_tcav: + if is_relative_tcav: num_concepts = len(result_summary) elif random_counterpart: num_concepts = len(result_summary) - 1 From f37780fc01f57b4e327a76f24d85c0c9354841e6 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Fri, 14 May 2021 16:01:10 +0200 Subject: [PATCH 07/12] fix: missing usage of separator constant --- tcav/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcav/utils.py b/tcav/utils.py index 2a7414f464..86d3809a5d 100644 --- a/tcav/utils.py +++ b/tcav/utils.py @@ -112,7 +112,7 @@ def get_random_concept(i): for tc, cs in new_pairs_to_test: all_concepts.append([tc]) for c in cs: - all_concepts.append(c.split(':')) + all_concepts.append(c.split(CONCEPT_SEPARATOR)) all_concepts = list(set(flatten(all_concepts))) else: all_concepts = list(set(flatten([cs + [tc] for tc, cs in new_pairs_to_test]))) From b19151e93c37aa6999c6fb5408d18d1ca5b99d24 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Fri, 14 May 2021 16:03:30 +0200 Subject: [PATCH 08/12] update: use filesystem friendly separator character --- tcav/activation_generator.py | 4 +--- tcav/tcav_test.py | 6 +++--- tcav/utils.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tcav/activation_generator.py b/tcav/activation_generator.py index 9928d52f51..edd82d8b3d 100644 --- a/tcav/activation_generator.py +++ b/tcav/activation_generator.py @@ -71,10 +71,8 @@ def process_and_load_activations(self, bottleneck_names, concepts, is_relative_t if concept not in acts: acts[concept] = {} - # convert concept name to filesystem friendly version - concept_filename = concept.replace(CONCEPT_SEPARATOR, "-") for bottleneck_name in bottleneck_names: - acts_path = os.path.join(self.acts_dir, f'acts_{concept_filename}_{bottleneck_name}') if self.acts_dir else None + acts_path = os.path.join(self.acts_dir, f'acts_{concept}_{bottleneck_name}') if self.acts_dir else None if acts_path and tf.io.gfile.exists(acts_path): # load activations from file diff --git a/tcav/tcav_test.py b/tcav/tcav_test.py index 3d6929b29d..f019e86b87 100644 --- a/tcav/tcav_test.py +++ b/tcav/tcav_test.py @@ -225,9 +225,9 @@ def test__process_what_to_run_expand_relative_tcav(self): self.assertEqual(sorted(my_relative_tcav.all_concepts), sorted(['t1', 'c1', 'c2', 'c3'])) self.assertEqual(sorted(my_relative_tcav.pairs_to_test), - sorted([('t1',['c1', 'c2:c3']), - ('t1',['c2', 'c1:c3']), - ('t1',['c3', 'c1:c2']), + sorted([('t1',['c1', 'c2+c3']), + ('t1',['c2', 'c1+c3']), + ('t1',['c3', 'c1+c2']), ])) def test_get_params(self): diff --git a/tcav/utils.py b/tcav/utils.py index 86d3809a5d..e28f022e19 100644 --- a/tcav/utils.py +++ b/tcav/utils.py @@ -26,7 +26,7 @@ ] MAX_RANDOM_EXPERIMENTS = 100 -CONCEPT_SEPARATOR = ':' +CONCEPT_SEPARATOR = '+' def create_session(timeout=10000, interactive=True): From 3da2bda2a7135f9f5ac73efce0034e7ddc9a0a4d Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 15 May 2021 10:46:28 +0200 Subject: [PATCH 09/12] update: comply with abstract method definition --- tcav/activation_generator.py | 11 ++++++++--- tcav/tcav_test.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tcav/activation_generator.py b/tcav/activation_generator.py index edd82d8b3d..6ba034a334 100644 --- a/tcav/activation_generator.py +++ b/tcav/activation_generator.py @@ -51,7 +51,7 @@ def get_model(self): return self.model @abstractmethod - def get_examples_for_concept(self, concept): + def get_examples_for_concept(self, concept, is_relative_tcav=False): pass def get_activations_for_concept(self, concept, bottleneck, is_relative_tcav=False): @@ -125,8 +125,9 @@ def get_examples_for_concept(self, concept_names, is_relative_tcav=False): [os.path.join(concept_dir, d) for d in tf.io.gfile.listdir(concept_dir)] ) img_paths = np.asarray(img_paths).flatten() + image_shape = self.model.get_image_shape()[:2] imgs = self.load_images_from_files( - img_paths, self.max_examples, shape=self.model.get_image_shape()[:2]) + img_paths, self.max_examples, shape=image_shape) return imgs def load_image_from_file(self, filename, shape): @@ -229,7 +230,7 @@ def __init__(self, model, source_dir, acts_dir, max_examples): super(DiscreteActivationGeneratorBase, self).__init__( model=model, acts_dir=acts_dir, max_examples=max_examples) - def get_examples_for_concept(self, concept): + def get_examples_for_concept(self, concept, is_relative_tcav=False): """Extracts examples for a concept and transforms them to the desired format. Args: @@ -243,6 +244,10 @@ def get_examples_for_concept(self, concept): load_data() and transform_data() functions. """ + if is_relative_tcav: + # Needs to be implemented + raise NotImplementedError() + data = self.load_data(concept) data_parsed = self.transform_data(data) return data_parsed diff --git a/tcav/tcav_test.py b/tcav/tcav_test.py index f019e86b87..e72b87321d 100644 --- a/tcav/tcav_test.py +++ b/tcav/tcav_test.py @@ -50,7 +50,7 @@ class TcavTest_ActGen(ActivationGeneratorBase): def __init__(self, model): super(TcavTest_ActGen, self).__init__(model, None, 10) - def get_examples_for_concept(self, concept): + def get_examples_for_concept(self, concept, is_relative_tcav=False): return [] class TcavTest(googletest.TestCase): From 7a02cdf434e93bf8a18abeaa78b028394af7a519 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 15 May 2021 10:47:27 +0200 Subject: [PATCH 10/12] add: tests for relative tcav --- tcav/activation_generator_test.py | 241 ++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 tcav/activation_generator_test.py diff --git a/tcav/activation_generator_test.py b/tcav/activation_generator_test.py new file mode 100644 index 0000000000..9c0aa90054 --- /dev/null +++ b/tcav/activation_generator_test.py @@ -0,0 +1,241 @@ +import os +import tempfile +import numpy as np +from tensorflow.python.platform import googletest +from PIL import Image +import weakref as _weakref +import sys + +from tcav.activation_generator import ImageActivationGenerator +from tcav.tcav_examples.discrete.kdd99_activation_generator import KDD99DiscreteActivationGenerator +from tcav.tcav import TCAV +from tcav.utils import CONCEPT_SEPARATOR + +IMG_SHAPE = (28, 28, 3) +MAX_TEST_IMAGES = 255 + + +class TemporaryDirectory(tempfile.TemporaryDirectory): + """ + Create temporary directories as defined int tempfile.TemporaryDirectory but when + prefix starts with 'random500' do not add unique directory name, but keep directory + name as is, defined by prefix+suffix. + """ + def __init__(self, suffix=None, prefix=None, dir=None): + prefix, suffix, dir, output_type = tempfile._sanitize_params(prefix, suffix, dir) + if prefix.startswith('random500_'): + self.name = os.path.join(dir, prefix + suffix) + sys.audit("tempfile.mkstemp", self.name) + os.mkdir(self.name, 0o700) + else: + self.name = tempfile.mkdtemp(suffix, prefix, dir) + self._finalizer = _weakref.finalize( + self, self._cleanup, self.name, + warn_message="Implicitly cleaning up {!r}".format(self)) + + +def _create_random_test_image_files(dst_dir, img_shape, count, start_offset, prefix): + if (count + start_offset) > MAX_TEST_IMAGES: + raise Exception(f"Cannot create more than '{MAX_TEST_IMAGES}' in one directory") + + count_magnitude = int(np.floor(np.log10(count))) + test_file_paths = [] + for i in range(count): + # create image with recognizable image values + # image value corresponds to image filename, + # zero values for img_0.jpg, ones for img_1.jpg, ... + values = np.ones(shape=img_shape, dtype=np.uint8) * (i + start_offset) # because of this, max_test_images=255 + img = Image.fromarray(values) + # save to disk + test_file = os.path.join(dst_dir, f"{prefix}_testfile_{i:0>{count_magnitude}d}.jpg") + img.save(test_file) + + test_file_paths.append(test_file) + + return test_file_paths + + +def _create_concept_dirs(dst_dir, concept_settings, image_shape, test_file_count, test_file_prefix): + concept_dirs = [] + for concept in concept_settings.keys(): + # create concept dir + concept_dir = TemporaryDirectory(prefix=concept, dir=dst_dir) + + # fill concept dir with test files + start_offset = concept_settings[concept].get('start_offset') + _ = _create_random_test_image_files(concept_dir.name, image_shape, test_file_count, start_offset, test_file_prefix) + + concept_dirs.append(concept_dir) + + return concept_dirs + + +def _get_image_value(image): + # return first pixel value + return image[0, 0, 0] # this works only for image len(shape)==3, e.g., for HxWxC + + +class MockTestModel: + """ + A mock model of model class. + """ + def __init__(self, image_shape): + self.model_name = 'test_model' + self.image_shape = image_shape + + def get_image_shape(self): + return self.image_shape + + +class ActivationGeneratorTest(googletest.TestCase): + def setUp(self): + self.concept_image_dir = tempfile.TemporaryDirectory() + + self.image_test_file_prefix = 'img' + self.image_test_file_count = 7 + + self.concept_settings = { + 'concept-1': {'start_offset': 0}, + 'concept-2': {'start_offset': self.image_test_file_count}, + 'concept-3': {'start_offset': self.image_test_file_count*2}, + 'random500_0': {'start_offset': self.image_test_file_count*3}, + 'random500_1': {'start_offset': self.image_test_file_count*4}, + 'random500_2': {'start_offset': self.image_test_file_count*5}, + } + self.concept_dirs = _create_concept_dirs(self.concept_image_dir.name, self.concept_settings, IMG_SHAPE, self.image_test_file_count, self.image_test_file_prefix) + self.concepts = [os.path.basename(concept_dir.name) for concept_dir in self.concept_dirs] + + self.model = MockTestModel(IMG_SHAPE) + self.max_examples = 1000 + self.img_act_gen = ImageActivationGenerator(model=self.model, source_dir=self.concept_image_dir.name, acts_dir=None, max_examples=self.max_examples, normalize_image=False) + + self.target = 't0' + self.bottleneck = 'bn' + self.hparams = {'model_type': 'linear', 'alpha': .01} + self.num_random_exp = 2 + self.normal_tcav = TCAV(sess=None, + target=self.target, + concepts=self.concepts, + bottlenecks=[self.bottleneck], + activation_generator=self.img_act_gen, + alphas=[self.hparams['alpha']], + num_random_exp=self.num_random_exp) + self.relative_tcav = TCAV(sess=None, + target=self.target, + concepts=self.concepts, + bottlenecks=[self.bottleneck], + activation_generator=self.img_act_gen, + alphas=[self.hparams['alpha']], + random_concepts=self.concepts) # this makes it relative_tcav + + def tearDown(self): + self.concept_image_dir.cleanup() + + for concept_dir in self.concept_dirs: + concept_dir.cleanup() + + def _get_concept_setting(self, concept_dir_name): + for concept in self.concept_settings.keys(): + if concept_dir_name.startswith(concept): + return self.concept_settings[concept] + return None + + def _get_expected_values_by_concept(self, concept_name, is_relative_tcav=False): + if is_relative_tcav: + concepts = concept_name.split(CONCEPT_SEPARATOR) + else: + concepts = [concept_name] + + expected_set_image_values = set() + for concept in concepts: + concept_settings = self._get_concept_setting(concept) + self.assertIsNotNone(concept_settings) + + start = concept_settings.get('start_offset') + end = start + self.image_test_file_count + expected_set_image_values.update(np.arange(start, end, dtype=np.float32)) + + return expected_set_image_values + + def test_get_examples_for_concept(self): + # (target, [positive-concept, negative-concept]) + concept_pairs = sorted(self.normal_tcav.pairs_to_test) + + # extract concepts + concept_pair_id = 0 + target, (pos_concept, neg_concept) = concept_pairs[concept_pair_id] + + # get positive concept image values + pos_set_images = self.img_act_gen.get_examples_for_concept(pos_concept) + actual_pos_set_image_values = [_get_image_value(img) for img in pos_set_images] + + # compute expected values for positive concept + expected_pos_set_image_values = self._get_expected_values_by_concept(pos_concept) + + # test whether correct positive set images were loaded + self.assertEqual(len(expected_pos_set_image_values), len(actual_pos_set_image_values)) + actual_pos_set_image_values = set(actual_pos_set_image_values) + self.assertEqual(expected_pos_set_image_values, actual_pos_set_image_values) + + # get negative concept image values + neg_set_images = self.img_act_gen.get_examples_for_concept(neg_concept) + actual_neg_set_image_values = [_get_image_value(img) for img in neg_set_images] + + # compute expected values for negative concept + expected_neg_set_image_values = self._get_expected_values_by_concept(neg_concept) + + # test whether correct negative set images were loaded + self.assertEqual(len(expected_neg_set_image_values), len(actual_neg_set_image_values)) + actual_neg_set_image_values = set(actual_neg_set_image_values) + self.assertEqual(expected_neg_set_image_values, actual_neg_set_image_values) + + def test_get_examples_for_concept_relative_tcav(self): + # (target, [positive-concept, negative-concept1+negative-concept2+...]) + concept_pairs = sorted(self.relative_tcav.pairs_to_test) + + # test if tcav object is relative tcav + is_relative_tcav = self.relative_tcav.is_relative_tcav + self.assertTrue(is_relative_tcav) + + # extract concepts + concept_pair_id = 0 + target, (pos_concept, neg_concept) = concept_pairs[concept_pair_id] + + # get positive concept image values + pos_set_images = self.img_act_gen.get_examples_for_concept(pos_concept, is_relative_tcav) + actual_pos_set_image_values = [_get_image_value(img) for img in pos_set_images] + + # compute expected values for positive concept + expected_pos_set_image_values = self._get_expected_values_by_concept(pos_concept, is_relative_tcav) + + # test whether correct positive set images were loaded + self.assertEqual(len(expected_pos_set_image_values), len(actual_pos_set_image_values)) + actual_pos_set_image_values = set(actual_pos_set_image_values) + self.assertEqual(expected_pos_set_image_values, actual_pos_set_image_values) + + # get negative concept image values + neg_set_images = self.img_act_gen.get_examples_for_concept(neg_concept, is_relative_tcav) + actual_neg_set_image_values = [_get_image_value(img) for img in neg_set_images] + + # compute expected values for negative concept + expected_neg_set_image_values = self._get_expected_values_by_concept(neg_concept, is_relative_tcav) + + # test whether correct negative set images were loaded + self.assertEqual(len(expected_neg_set_image_values), len(actual_neg_set_image_values)) + actual_neg_set_image_values = set(actual_neg_set_image_values) + self.assertEqual(expected_neg_set_image_values, actual_neg_set_image_values) + + def test_get_examples_for_concept_discrete(self): + discrete_act_gen = KDD99DiscreteActivationGenerator( + model=None, + source_dir=None, + acts_dir=None, + max_examples=0 + ) + concept = None + is_relative = True + self.assertRaises(NotImplementedError, discrete_act_gen.get_examples_for_concept, concept, is_relative) + + +if __name__ == '__main__': + googletest.main() From a04a54a9a23ce5bad56c37deb8ba0f11f2e05dd7 Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Sat, 4 Sep 2021 13:37:51 +0200 Subject: [PATCH 11/12] fix: plot resources not released on save figure --- tcav/utils_plot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tcav/utils_plot.py b/tcav/utils_plot.py index 188b9f4cec..58f15df959 100644 --- a/tcav/utils_plot.py +++ b/tcav/utils_plot.py @@ -188,3 +188,6 @@ def is_random_concept(concept): plt.show() else: plt.savefig(fig_path, dpi=200) + + # release resources + plt.close() \ No newline at end of file From 782cd1541753dce82f790bb8b96be57609e323ca Mon Sep 17 00:00:00 2001 From: Markus Monz Date: Fri, 22 Apr 2022 15:52:01 +0200 Subject: [PATCH 12/12] add: target class name to tcav plot --- tcav/utils_plot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tcav/utils_plot.py b/tcav/utils_plot.py index 58f15df959..bc241f3894 100644 --- a/tcav/utils_plot.py +++ b/tcav/utils_plot.py @@ -59,7 +59,8 @@ def is_random_concept(concept): result_outfile = sys.stdout # print class, it will be the same for all - print("Class =", results[0]['target_class'], file=result_outfile) + target_class = results[0]['target_class'] + print("Class =", target_class, file=result_outfile) # prepare data # dict with keys of concepts containing dict with bottlenecks @@ -176,7 +177,7 @@ def is_random_concept(concept): print(plot_data, file=result_outfile) # set properties - ax.set_title('TCAV Scores for each concept and bottleneck') + ax.set_title(f"TCAV Scores for each concept and bottleneck, target: '{target_class}'") ax.set_ylabel('TCAV Score') ax.set_xticks(index + num_bottlenecks * bar_width / 2) ax.set_xticklabels(plot_concepts) @@ -190,4 +191,4 @@ def is_random_concept(concept): plt.savefig(fig_path, dpi=200) # release resources - plt.close() \ No newline at end of file + plt.close()