diff --git a/docs/api.md b/docs/api.md index a9ffd12..a557871 100644 --- a/docs/api.md +++ b/docs/api.md @@ -11,6 +11,17 @@ show_root_toc_entry: false show_source: false +::: unpackqa.pack + selection: + members: + - pack_from_array + - pack_from_dict + docstring_style: numpy + rendering: + show_root_heading: false + show_root_toc_entry: false + show_source: false + ::: unpackqa.product_loader selection: members: diff --git a/test/test_core_unpacking.py b/test/test_core_unpacking.py new file mode 100644 index 0000000..7c12b44 --- /dev/null +++ b/test/test_core_unpacking.py @@ -0,0 +1,59 @@ +import numpy as np +import pytest + +from unpackqa.tools.unpackbits import unpackbits, int16_to_bits, packbits + + +def test_compare_unpack_methods(): + """ + Core unpacking function should have the exact results as a slower + string based method + """ + arr = np.arange(2046) + + method1 = np.array([int16_to_bits(int(x)) for x in arr], dtype=np.uint8) + method2 = unpackbits(arr, num_bits=16) + + # bit axis at the end to match method1 + method2 = np.moveaxis(method2, source = 0, destination = -1) + + assert (method1 == method2).all() + +@pytest.mark.parametrize('num_bits', [8,16,32]) +def test_unpackbits_shape_retention(num_bits): + """ + Core unpacking function should take an arbitrary shape and return the + same with 1 new axis of length num_bits at axis position 0. + """ + high_range = (2**num_bits)-1 + arr1 = np.random.randint(low=0, high=high_range, size=64) + arr2 = np.random.randint(low=0, high=high_range, size=64**2).reshape((64,64)) + arr3 = np.random.randint(low=0, high=high_range, size=64**3).reshape((64,64,64)) + arr4 = np.random.randint(low=0, high=high_range, size=64*21*42).reshape((64,21,42)) + + test_result = [ + unpackbits(arr1, num_bits=num_bits).shape == (num_bits,64), + unpackbits(arr2, num_bits=num_bits).shape == (num_bits,64,64), + unpackbits(arr3, num_bits=num_bits).shape == (num_bits,64,64,64), + unpackbits(arr4, num_bits=num_bits).shape == (num_bits,64,21,42), + ] + + assert all(test_result) + +@pytest.mark.parametrize('num_bits', [8,16,32]) +def test_unpack_to_pack(num_bits): + """packbits should reverse unpackbits""" + high_range = (2**num_bits)-1 + arr1 = np.random.randint(low=0, high=high_range, size=64) + arr2 = np.random.randint(low=0, high=high_range, size=64**2).reshape((64,64)) + arr3 = np.random.randint(low=0, high=high_range, size=64**3).reshape((64,64,64)) + arr4 = np.random.randint(low=0, high=high_range, size=64*21*42).reshape((64,21,42)) + + test_results = [ + (arr1 == packbits(unpackbits(arr1, num_bits=num_bits), num_bits=num_bits)).all(), + (arr2 == packbits(unpackbits(arr2, num_bits=num_bits), num_bits=num_bits)).all(), + (arr3 == packbits(unpackbits(arr3, num_bits=num_bits), num_bits=num_bits)).all(), + (arr4 == packbits(unpackbits(arr4, num_bits=num_bits), num_bits=num_bits)).all(), + ] + + assert all(test_results) \ No newline at end of file diff --git a/test/test_output.py b/test/test_output.py deleted file mode 100644 index 0b41bb9..0000000 --- a/test/test_output.py +++ /dev/null @@ -1,115 +0,0 @@ -import pytest -import numpy as np - -from unpackqa import (unpack_to_array, - unpack_to_dict, - list_products, - list_qa_flags - ) - - -""" -Just iterating thru all products as integration tests. -""" - -qa_array = np.array([[8,8,8], - [16,16,16], - [255,255,255]]) - -all_product_identifiers = list_products(sensor='all') - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_unpack_array_shape(product): - """ - With >1 flag unpack_to_array result should match the input shape and have - a new axis at position (-1) with the same length as number of flags. - """ - n_flags = len(list_qa_flags(product = product)) - target_shape = qa_array.shape + (n_flags,) - - result = unpack_to_array(qa_array, product = product) - assert result.shape == target_shape - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_single_flag_array_shape(product): - """ - A single flag should return an array the same shape as the input and - *not* have an added axis. - """ - flags = list_qa_flags(product = product) - flags = [flags[0]] - result = unpack_to_array(qa_array, product = product, flags=flags) - - assert result.shape == qa_array.shape - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_unpack_dict_shape(product): - """Each dictionary value in unpack_to_dick should match the input qa shape""" - flags = list_qa_flags(product = product) - target_shape = qa_array.shape - - result = unpack_to_dict(qa_array, product = product, flags='all') - - all_shapes_good = [result[flag].shape == target_shape for flag in flags] - assert all(all_shapes_good) - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_unpack_dict_all_flags(product): - """ The all flag indicator returns all available flags""" - flags = list_qa_flags(product = product) - - result = unpack_to_dict(qa_array, product = product, flags='all') - all_flags_in_result = [f in result for f in flags] - all_results_in_flags = [f in flags for f in result.keys()] - assert all(all_flags_in_result) and all(all_results_in_flags) - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_unpack_dict_2_flags1(product): - """ Get the first and last flag only. """ - flags = list_qa_flags(product = product) - flags = [flags[0], flags[-1]] - - result = unpack_to_dict(qa_array, product = product, flags=flags) - all_flags_in_result = [f in result for f in flags] - all_results_in_flags = [f in flags for f in result.keys()] - assert all(all_flags_in_result) and all(all_results_in_flags) - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_unpack_dict_2_flags2(product): - """ Get the first flag only. """ - flags = list_qa_flags(product = product) - flags = [flags[0]] - - result = unpack_to_dict(qa_array, product = product, flags=flags) - all_flags_in_result = [f in result for f in flags] - all_results_in_flags = [f in flags for f in result.keys()] - assert all(all_flags_in_result) and all(all_results_in_flags) - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_flag_wrong_axis_ordering(product): - """ - Ordering of the flag axis is the same as the flag list. - When the flag list is reversed, the resulting arrays should not match. - """ - flags = list_qa_flags(product = product) - - mask1 = unpack_to_array(qa_array, product=product, flags=flags) - flags.reverse() - mask2 = unpack_to_array(qa_array, product=product, flags=flags) - - assert not (mask1 == mask2).all() - -@pytest.mark.parametrize('product', all_product_identifiers) -def test_flag_correct_axis_ordering(product): - """ - Ordering of the flag axis is the same as the flag list. - When the flag list is reversed, the resulting arrays should not match. - But should match when the axis is flipped back again. - """ - flags = list_qa_flags(product = product) - - mask1 = unpack_to_array(qa_array, product=product, flags=flags) - flags.reverse() - mask2 = unpack_to_array(qa_array, product=product, flags=flags) - - assert (mask1 == np.flip(mask2, axis=-1)).all() \ No newline at end of file diff --git a/test/test_packing.py b/test/test_packing.py new file mode 100644 index 0000000..6dc57c2 --- /dev/null +++ b/test/test_packing.py @@ -0,0 +1,68 @@ +import pytest +import numpy as np + +from unpackqa import (unpack_to_array, + unpack_to_dict, + pack_from_array, + pack_from_dict, + ) + + + +custom_spec1 = {'flag_info':{ + 'flag1_description':[0], + 'flag2_description':[1], + 'flag3_description':[2], + 'flag4_description':[3], + 'flag5_description':[4], + 'flag6_description':[5,6,7], + + }, + 'max_value' : 255, + 'num_bits' : 8} + +custom_spec2 = {'flag_info':{ + 'flag1_description':[0], + 'flag2_description':[1], + 'flag3_description':[2], + 'flag4_description':[3], + 'flag5_description':[4,5], + 'flag6_description':[6], + 'flag7_description':[7,8,9], + 'flag8_description':[10,11], + 'flag10_description':[12], + 'flag11_description':[13], + 'flag12_description':[14], + 'flag13_description':[15], + }, + 'max_value' : 65535, + 'num_bits' : 16} + +product_test_cases = [custom_spec1, custom_spec2] + +@pytest.mark.parametrize('product_spec', product_test_cases) +def test_array_packing(product_spec): + high_range = (2**product_spec['num_bits'])-1 + qa_array = np.random.randint(low=0, high=high_range, size=32**3).reshape((32,32,32)) + + flag_array = unpack_to_array(qa_array, product=product_spec, flags='all') + packed_array = pack_from_array(flag_array, product=product_spec, flags='all') + + assert (qa_array == packed_array).all() + +@pytest.mark.parametrize('product_spec', product_test_cases) +def test_dict_packing(product_spec): + high_range = (2**product_spec['num_bits'])-1 + qa_array = np.random.randint(low=0, high=high_range, size=32**3).reshape((32,32,32)) + + flag_array = unpack_to_dict(qa_array, product=product_spec, flags='all') + packed_array = pack_from_dict(flag_array, product=product_spec, flags='all') + + assert (qa_array == packed_array).all() + + + + + + + diff --git a/test/test_qa_values.py b/test/test_qa_values.py index 83338ab..809e8f1 100644 --- a/test/test_qa_values.py +++ b/test/test_qa_values.py @@ -2,10 +2,14 @@ import numpy as np from unpackqa import (unpack_to_array, - unpack_to_dict, - list_products, - list_qa_flags - ) + unpack_to_dict, + pack_from_array, + pack_from_dict, + list_products, + list_qa_flags + ) + +from unpackqa.product_loader import all_products # Given a list of actual QA values, ensure the flags are as expected. # These values and resulting flags are @@ -69,3 +73,12 @@ def test_confirm_wrong_flag_values(product, test_qa_value, expected_output): first_key = list(output.keys())[0] output[first_key] += 1 assert output != expected_output + +@pytest.mark.parametrize('product, test_qa_value, expected_output', qa_test_cases) +def test_confirm_repacked_qa_values(product, test_qa_value, expected_output): + """unpack -> pack should have the same result""" + unpacked_array = unpack_to_array(test_qa_value, product=product, flags='all') + # pack_to_ functions do not currently take a preconfigured product. + product_spec = all_products[product] + repacked_qa = pack_from_array(unpacked_array, product=product_spec, flags='all') + assert (test_qa_value == repacked_qa).all() \ No newline at end of file diff --git a/test/test_unpacking.py b/test/test_unpacking.py index 7c12b44..0b41bb9 100644 --- a/test/test_unpacking.py +++ b/test/test_unpacking.py @@ -1,59 +1,115 @@ -import numpy as np import pytest +import numpy as np + +from unpackqa import (unpack_to_array, + unpack_to_dict, + list_products, + list_qa_flags + ) -from unpackqa.tools.unpackbits import unpackbits, int16_to_bits, packbits +""" +Just iterating thru all products as integration tests. +""" -def test_compare_unpack_methods(): +qa_array = np.array([[8,8,8], + [16,16,16], + [255,255,255]]) + +all_product_identifiers = list_products(sensor='all') + +@pytest.mark.parametrize('product', all_product_identifiers) +def test_unpack_array_shape(product): + """ + With >1 flag unpack_to_array result should match the input shape and have + a new axis at position (-1) with the same length as number of flags. """ - Core unpacking function should have the exact results as a slower - string based method + n_flags = len(list_qa_flags(product = product)) + target_shape = qa_array.shape + (n_flags,) + + result = unpack_to_array(qa_array, product = product) + assert result.shape == target_shape + +@pytest.mark.parametrize('product', all_product_identifiers) +def test_single_flag_array_shape(product): + """ + A single flag should return an array the same shape as the input and + *not* have an added axis. """ - arr = np.arange(2046) + flags = list_qa_flags(product = product) + flags = [flags[0]] + result = unpack_to_array(qa_array, product = product, flags=flags) + + assert result.shape == qa_array.shape + +@pytest.mark.parametrize('product', all_product_identifiers) +def test_unpack_dict_shape(product): + """Each dictionary value in unpack_to_dick should match the input qa shape""" + flags = list_qa_flags(product = product) + target_shape = qa_array.shape + + result = unpack_to_dict(qa_array, product = product, flags='all') - method1 = np.array([int16_to_bits(int(x)) for x in arr], dtype=np.uint8) - method2 = unpackbits(arr, num_bits=16) + all_shapes_good = [result[flag].shape == target_shape for flag in flags] + assert all(all_shapes_good) + +@pytest.mark.parametrize('product', all_product_identifiers) +def test_unpack_dict_all_flags(product): + """ The all flag indicator returns all available flags""" + flags = list_qa_flags(product = product) + + result = unpack_to_dict(qa_array, product = product, flags='all') + all_flags_in_result = [f in result for f in flags] + all_results_in_flags = [f in flags for f in result.keys()] + assert all(all_flags_in_result) and all(all_results_in_flags) + +@pytest.mark.parametrize('product', all_product_identifiers) +def test_unpack_dict_2_flags1(product): + """ Get the first and last flag only. """ + flags = list_qa_flags(product = product) + flags = [flags[0], flags[-1]] - # bit axis at the end to match method1 - method2 = np.moveaxis(method2, source = 0, destination = -1) + result = unpack_to_dict(qa_array, product = product, flags=flags) + all_flags_in_result = [f in result for f in flags] + all_results_in_flags = [f in flags for f in result.keys()] + assert all(all_flags_in_result) and all(all_results_in_flags) - assert (method1 == method2).all() +@pytest.mark.parametrize('product', all_product_identifiers) +def test_unpack_dict_2_flags2(product): + """ Get the first flag only. """ + flags = list_qa_flags(product = product) + flags = [flags[0]] + + result = unpack_to_dict(qa_array, product = product, flags=flags) + all_flags_in_result = [f in result for f in flags] + all_results_in_flags = [f in flags for f in result.keys()] + assert all(all_flags_in_result) and all(all_results_in_flags) -@pytest.mark.parametrize('num_bits', [8,16,32]) -def test_unpackbits_shape_retention(num_bits): +@pytest.mark.parametrize('product', all_product_identifiers) +def test_flag_wrong_axis_ordering(product): """ - Core unpacking function should take an arbitrary shape and return the - same with 1 new axis of length num_bits at axis position 0. + Ordering of the flag axis is the same as the flag list. + When the flag list is reversed, the resulting arrays should not match. """ - high_range = (2**num_bits)-1 - arr1 = np.random.randint(low=0, high=high_range, size=64) - arr2 = np.random.randint(low=0, high=high_range, size=64**2).reshape((64,64)) - arr3 = np.random.randint(low=0, high=high_range, size=64**3).reshape((64,64,64)) - arr4 = np.random.randint(low=0, high=high_range, size=64*21*42).reshape((64,21,42)) - - test_result = [ - unpackbits(arr1, num_bits=num_bits).shape == (num_bits,64), - unpackbits(arr2, num_bits=num_bits).shape == (num_bits,64,64), - unpackbits(arr3, num_bits=num_bits).shape == (num_bits,64,64,64), - unpackbits(arr4, num_bits=num_bits).shape == (num_bits,64,21,42), - ] - - assert all(test_result) - -@pytest.mark.parametrize('num_bits', [8,16,32]) -def test_unpack_to_pack(num_bits): - """packbits should reverse unpackbits""" - high_range = (2**num_bits)-1 - arr1 = np.random.randint(low=0, high=high_range, size=64) - arr2 = np.random.randint(low=0, high=high_range, size=64**2).reshape((64,64)) - arr3 = np.random.randint(low=0, high=high_range, size=64**3).reshape((64,64,64)) - arr4 = np.random.randint(low=0, high=high_range, size=64*21*42).reshape((64,21,42)) - - test_results = [ - (arr1 == packbits(unpackbits(arr1, num_bits=num_bits), num_bits=num_bits)).all(), - (arr2 == packbits(unpackbits(arr2, num_bits=num_bits), num_bits=num_bits)).all(), - (arr3 == packbits(unpackbits(arr3, num_bits=num_bits), num_bits=num_bits)).all(), - (arr4 == packbits(unpackbits(arr4, num_bits=num_bits), num_bits=num_bits)).all(), - ] - - assert all(test_results) \ No newline at end of file + flags = list_qa_flags(product = product) + + mask1 = unpack_to_array(qa_array, product=product, flags=flags) + flags.reverse() + mask2 = unpack_to_array(qa_array, product=product, flags=flags) + + assert not (mask1 == mask2).all() + +@pytest.mark.parametrize('product', all_product_identifiers) +def test_flag_correct_axis_ordering(product): + """ + Ordering of the flag axis is the same as the flag list. + When the flag list is reversed, the resulting arrays should not match. + But should match when the axis is flipped back again. + """ + flags = list_qa_flags(product = product) + + mask1 = unpack_to_array(qa_array, product=product, flags=flags) + flags.reverse() + mask2 = unpack_to_array(qa_array, product=product, flags=flags) + + assert (mask1 == np.flip(mask2, axis=-1)).all() \ No newline at end of file diff --git a/unpackqa/__init__.py b/unpackqa/__init__.py index 25abdac..4ec0311 100644 --- a/unpackqa/__init__.py +++ b/unpackqa/__init__.py @@ -9,6 +9,10 @@ unpack_to_dict, ) +from .pack import (pack_from_array, + pack_from_dict, + ) + # Make this available to catch exceptions from .tools.validation import InvalidProductSpec @@ -17,5 +21,7 @@ 'list_qa_flags', 'unpack_to_array', 'unpack_to_dict', + 'pack_from_array', + 'pack_from_dict', 'InvalidProductSpec', ] \ No newline at end of file diff --git a/unpackqa/base.py b/unpackqa/base.py index 3b8190b..f389bfd 100644 --- a/unpackqa/base.py +++ b/unpackqa/base.py @@ -1,7 +1,7 @@ from collections import OrderedDict import numpy as np -from .tools.unpackbits import unpackbits +from .tools.unpackbits import unpackbits, packbits class UnpackQABase: """ @@ -186,3 +186,182 @@ def _unpack_to_dict(self, qa, flags='all'): flags = self._parse_flag_args(flags) flag_array = self._unpack_array_core(qa = qa, flags = flags) return {flag:flag_array[flag_i] for flag_i, flag in enumerate(flags)} + + +class PackQABase: + """ + Generalized bit packing methods. + """ + def __init__(self, product_info, validate): + self.flag_info = product_info['flag_info'] + self.num_bits = product_info['num_bits'] + self.max_value = product_info['max_value'] + self.validate = validate + + def _parse_flag_args(self, passed_flags): + if passed_flags == 'all': + passed_flags = self._available_qa_flags() + elif isinstance(passed_flags, list) and len(passed_flags) > 0: + valid_flags = self._available_qa_flags() + if not all([f in valid_flags for f in passed_flags]): + raise ValueError('Invalid flag name passed') + else: + error_message = "flags should be a list of strings or 'all'" + raise ValueError(error_message) + + return passed_flags + + def _validate_flag_array(self, flag_array): + n_flags = len(self.flag_info) + if flag_array.shape[0] != n_flags: + msg = 'flag_array axis length ({}) must match number of flags ({})' + raise ValueError(error_message.forma(flag_array.shape[0], n_flags)) + + if flag_array.min() < 0: + m = flag_array.min() + msg = 'flag_array values smaller than the specified range for this product. ' + \ + 'min value was {} and the valid range is 0-{}' + raise ValueError(error_message.format(m, self.max_value)) + + for flag_i, (flag_name, flag_bit_locs) in enumerate(self.flag_info.items()): + max_value = (2**len(flag_bit_locs)) - 1 + if flag_array[flag_i].max() > max_value: + m = flag_array[flag_i].max() + n_bits = len(flag_bit_locs) + msg = 'axis for "{}" has values larger than larger than specified ' + \ + 'bit allows. max_value: {} for {} bits.' + raise ValueError(msg.format(flag_name, m, n_bits)) + + #TODO: if some bits are not specified in product, ensure they equal 0 + + def _available_qa_flags(self): + """ + A list of available QA flags for this product. + + Returns + ------- + list + List of strings + + """ + return list(self.flag_info.keys()) + + def _pack_array_core(self, flag_array, flags): + """ + Core function for bit packing. + + Primary role here is converting flag values (which are generally 0-1, but can also be 0-16), + into the binary equivilant and in the correct bit location. + + Parameters + ---------- + flag_array : np.array + An array where the axis at location 0 is the flag axis, and has the same + length as `flags`. + flags : TYPE + list of flag names represented in flag_array. Should match those in the + prodcuct specification. + + Returns + ------- + np.array + A numpy array with shape flag_array.shape[1:], and dtype of either + 8, 16, or 32 bit np.uint matching `num_bits`. + + """ + if self.validate: + self._validate_flag_array(flag_array) + #--- + # First convert the flag values, which are usually 0-1 but can also + # be 0-3 (2 bits) or even 0-15 (4 bits), to the bit (0/1) values. + # These are ordered by the bit location defined in the product_spec + #--- + # bit_array shape is the same, but axis 0 is now the bit array with + # length num_bits + # eg. (12,1024,1024) is a 1024x1024 array with 12 flags. 16 bits are needed + # to store that many, so it ends up as (16,1024,1024) + bit_shape = (self.num_bits,) + flag_array.shape[1:] + bit_array = np.zeros(bit_shape, dtype=np.uint8) + + for flag_i, flag_name in enumerate(flags): + flag_bit_locs = self.flag_info[flag_name] + if len(flag_bit_locs) == 1: + bit_array[flag_bit_locs[0]] = flag_array[flag_i] + else: + # > 1 bit flag means the flag values need conversion back + # to 0/1. eg. [1,2,3] -> [[0,1],[1,0],[1,1]] + bit_array[flag_bit_locs] = unpackbits( + flag_array[flag_i], + num_bits = len(flag_bit_locs) + ) + + return packbits(bit_array, num_bits=self.num_bits) + + def _pack_from_array(self, flag_values, flag_axis, flags='all'): + """ + Get a qa array from a flag array. + + Parameters + ---------- + flag_values : np.array + numpy array of flag values. + flags : list or str + List of flags to pack, or `all` (the default) to pack all flags + specified in `product` + flag_axis: int + The location of the flag axis in flag_values. + + Returns + ------- + np.array + An array of qa values derived from bit packing individual flags. + Shape is the same as the arrays in flag_values.. + + """ + if flag_axis != 0: + flag_values = np.moveaxis( + flag_values, + source=flag_axis, + destination=0) + + flags = self._parse_flag_args(flags) + return self._pack_array_core(flag_array = flag_values, flags = flags) + + def _pack_from_dict(self, flag_values, flags='all'): + """ + Get a qa array from flags in a dictionary + + Parameters + ---------- + flag_values : dict + A dictionary where the flag names are keys and values are np.arrays + of the flag mask with shape qa.shape. All entries must have the + same shape. + flags : list of strings or 'all', optional + List of flags to return. If 'all', the default, then all available + flags are returned in the array. See available flags for each + product with `list_qa_flags()`. + + Returns + ------- + np.array + An array of qa values derived from bit packing individual flags. + Shape is the same as the arrays in flag_values. + + """ + flags = self._parse_flag_args(flags) + flags_present = [f in flag_values for f in flags] + if not all(flags_present): + raise ValueError('flag entries missing from flag_values dict') + + # stack the dictionary into a single array. ordered by product spec + n_flags = len(flags) + flag_value_shape = (n_flags,) + flag_values[flags[0]].shape + flag_value_dtype = flag_values[flags[0]].dtype + + flag_array = np.zeros(flag_value_shape, dtype=flag_value_dtype) + + for flag_i, flag_name in enumerate(flags): + flag_array[flag_i] = flag_values[flag_name] + + return self._pack_array_core(flag_array = flag_array, flags = flags) diff --git a/unpackqa/pack.py b/unpackqa/pack.py new file mode 100644 index 0000000..9a16115 --- /dev/null +++ b/unpackqa/pack.py @@ -0,0 +1,136 @@ +from .base import PackQABase +from .product_loader import all_products +from .tools.validation import validate_product_spec + + + +def pack_from_array(flag_values, product, flags='all', flag_axis=-1, validate=True): + """ + Get a bitpacked QA array from an array of flag values. + + This is currently experimental. Tests for it work, but some use cases would be useful to validate it. + Open an issue on the github page if you have an interesting use case and would like to contribute. + + `flag_values` should be a np.array where either the first (0) or last (-1) axis + is the flag values. The resulting array will match the array shape minus + the flag axis. + + This reverses `unpack_to_array()` in that: + `qa_values == pack_from_array(unpack_to_array(qa_values))` is True + + All flags specified in `product` should have a corresponding location in + flag_values. + + Parameters + ---------- + flag_values : np.array + A numpy array of flag values. The flag axis (specifed via `flag_axis`) + needs to match the number of flags specified in `product`. + eg. `flag_values.shape[flag_axis] == len(product_info['flag_info'])` must + be true. + + product : dict + A custom bit flag specification may also be used via a dictionary. + For example, below is an 8 bit qa flag where the 1st and 2nd flags are bits 0 and 1, + and the 3rd flag is spread across bits 4 and 5. Note bit 3 is not specified + so is ignored. `max_value` is generally set to the maximum possible + value given the bit size. If this option is used then the `flags` + argument can be left as the default `all`. + Flags should be ordered from lowest to highest bits. The order of the mask axis + in the returned array will be the same as the flag order specifed here. + + ``` + product_info = {'flag_info':{'flag1_description':[0], + 'flag2_description':[1], + 'flag3_description':[4,5]}, + 'max_value' : 255, + 'num_bits' : 8} + ``` + flags : list or str + List of flags to pack, or `all` (the default) to pack all flags + specified in `product` + flag_axis : int, optional + The location of the flag axis within the flag_values array. This defaults + to -1 (the last position), which matches the output of unpack_to_array(). + validate : bool, optional + Perform checks before bitpacking. Setting to false can speed this + up slightly. + + Returns + ------- + np.array + A QA array with the same shape as `flag_values` minus the flag axis, and + with the dtype matching `num_bits` in `product` (either np.uint 8, 16, or 32). + + """ + if isinstance(product, dict): + validate_product_spec(product) + base = PackQABase(product, validate=validate) + return base._pack_from_array(flag_values, flags=flags, flag_axis=flag_axis) + else: + error_message = 'product should be a dictionary.' + raise TypeError(error_message) + +def pack_from_dict(flag_values, product, flags='all', validate=True): + """ + Get a bitpacked QA array from an array of dictionary values. + + This is currently experimental. Tests for it work, but some use cases would be useful to validate it. + Open an issue on the github page if you have an interesting use case and would like to contribute. + + flag_values should be a dict where the keys are flag names and match the + flag names in `product`. The dictionary values should be corresponding + flag array, and all arrays must have the same shape. The resulting + bitpacked QA array will have the same shape. + + This reverses `unpack_to_dict()` in that: + `qa_values == pack_from_dict(unpack_to_dict(qa_values))` is True + + All flags specified in `product` should have a corresponding location in + flag_values. + + + Parameters + ---------- + flag_values : dict + A dictionary where the flag names are keys and values are np.arrays + of the flag values. All entries must have the same shape. + product : dict + A custom bit flag specification may also be used via a dictionary. + For example, below is an 8 bit qa flag where the 1st and 2nd flags are bits 0 and 1, + and the 3rd flag is spread across bits 4 and 5. Note bit 3 is not specified + so is ignored. `max_value` is generally set to the maximum possible + value given the bit size. If this option is used then the `flags` + argument can be left as the default `all`. + Flags should be ordered from lowest to highest bits. The order of the mask axis + in the returned array will be the same as the flag order specifed here. + + ``` + product_info = {'flag_info':{'flag1_description':[0], + 'flag2_description':[1], + 'flag3_description':[4,5]}, + 'max_value' : 255, + 'num_bits' : 8} + ``` + flags : list or str + List of flags to pack, or `all` (the default) to pack all flags + specified in `product` + validate : bool, optional + Perform checks before bitpacking. Setting to false can speed this + up slightly. + + + Returns + ------- + np.array + A QA array with the same shape as the arrays in `flag_values`, and + with dtype matching `num_bits` in `product` (either np.uint 8, 16, or 32). + + """ + if isinstance(product, dict): + validate_product_spec(product) + base = PackQABase(product, validate=validate) + return base._pack_from_dict(flag_values, flags=flags) + else: + error_message = 'product should be a dictionary.' + raise TypeError(error_message)