From 2cbb6ae973189561be09fc63d71eb918a062856a Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Tue, 9 Aug 2022 23:50:21 -0700 Subject: [PATCH 01/10] update test --- .../backend/dashboard_aws_helper.py | 7 +- .../tests/dashboard_aws_helper_test.py | 76 +++++++++++-------- .../backend/tests/s3_client_stubber.py | 38 ++++++++++ 3 files changed, 89 insertions(+), 32 deletions(-) create mode 100644 droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py index f813998f88..11dbbda6bb 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py +++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py @@ -75,11 +75,14 @@ def get_job_list(): res = s3.meta.client.get_paginator("list_objects").paginate( Bucket=S3_BUCKET_NAME, Delimiter="/" ) + print(res) + print(s3.meta.client.get_paginator("list_objects")) # pattern of YYYYMMDDHHMMSS (batch id pattern) pattern = r"([0-9]{4})(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])" - + for prefix in res.search("CommonPrefixes"): - if re.match(pattern, prefix.get("Prefix")): + print(prefix) + if prefix is not None and re.match(pattern, prefix.get("Prefix")): job_list.append(int(prefix.get("Prefix")[:-1])) return job_list diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py index 6df072e91b..6e7746c76c 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py @@ -1,13 +1,12 @@ import unittest import os +from unittest.mock import patch import boto3 +from moto import mock_s3 +import botocore.session import json -from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( - get_job_list, - get_run_info_by_id, - get_traceback_by_id, -) +from droidlet.tools.hitl.dashboard_app.backend.tests.s3_client_stubber import OS_ENV_DICT, MockListObjResult, S3ClientMock, mock_make_api_call S3_BUCKET_NAME = "droidlet-hitl" S3_ROOT = "s3://droidlet-hitl" @@ -15,65 +14,82 @@ VALID_ID = 222222222222222222 INVALID_ID = 11111111111111111 -AWS_ACCESS_KEY_ID = os.environ["AWS_ACCESS_KEY_ID"] -AWS_SECRET_ACCESS_KEY = os.environ["AWS_SECRET_ACCESS_KEY"] -AWS_DEFAULT_REGION = os.environ["AWS_DEFAULT_REGION"] - HITL_TMP_DIR = ( os.environ["HITL_TMP_DIR"] if os.getenv("HITL_TMP_DIR") else f"{os.path.expanduser('~')}/.hitl" ) -s3 = boto3.resource( - "s3", - region_name=AWS_DEFAULT_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, -) - - +@mock_s3 class TestAWSHelper(unittest.TestCase): def setUp(self): + conn = boto3.resource('s3', region_name='us-east-1') + # We need to create the bucket since this is all in Moto's 'virtual' AWS account + conn.create_bucket(Bucket='droidlet-hitl') self._info_fname = f"job_management_records/{VALID_ID}.json" self._traceback_fname = f"{VALID_ID}/log_traceback.csv" some_dict = {"msg": "hello"} json_content = json.dumps(some_dict) + s3 = boto3.client('s3', region_name='us-east-1') + s3.put_object(Bucket='droidlet-hitl', Key=self._info_fname, Body=json_content) + s3.put_object(Bucket='droidlet-hitl', Key=self._traceback_fname, Body="1, 2 \n1, 2") - s3.Object(S3_BUCKET_NAME, self._info_fname).put(Body=json_content) - s3.Object(S3_BUCKET_NAME, self._traceback_fname).put(Body="1, 2 \n1, 2") - + @patch.dict('os.environ', OS_ENV_DICT) def test_get_job_list(self): + s3 = boto3.client('s3', region_name='us-east-1') + s3.put_object(Bucket='droidlet-hitl', Key="20220224132033/", Body="1, 2 \n1, 2") + from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( + get_job_list + ) res = get_job_list() self.assertGreater(len(res), 0) + @patch.dict('os.environ', OS_ENV_DICT) def test_get_traceback_by_id_valid(self): + from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( + get_traceback_by_id + ) res, _ = get_traceback_by_id(VALID_ID) self.assertIsNotNone(res) self.assertNotEqual(res, f"cannot find traceback with id {VALID_ID}") + + @patch.dict('os.environ', OS_ENV_DICT) def test_get_traceback_by_id_invalid(self): + from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( + get_traceback_by_id + ) res, _ = get_traceback_by_id(INVALID_ID) self.assertEqual(res, f"cannot find traceback with id {INVALID_ID}") + @patch.dict('os.environ', OS_ENV_DICT) def test_get_run_info_by_id_valid(self): + from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( + get_run_info_by_id + ) res, _ = get_run_info_by_id(VALID_ID) self.assertIsNotNone(res) self.assertNotEqual(res, f"cannot find run info with id {VALID_ID}") + @patch.dict('os.environ', OS_ENV_DICT) + @mock_s3 def test_get_run_info_by_id_inalid(self): + from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( + get_run_info_by_id + ) res, _ = get_run_info_by_id(INVALID_ID) self.assertEqual(res, f"cannot find run info with id {INVALID_ID}") def tearDown(self): - s3.Object(S3_BUCKET_NAME, self._info_fname).delete() - s3.Object(S3_BUCKET_NAME, self._traceback_fname).delete() - - # remove from local temp directory as well - local_info_fname = os.path.join(HITL_TMP_DIR, self._info_fname) - local_traceback_fname = os.path.join(HITL_TMP_DIR, self._traceback_fname) - if os.path.exists(local_info_fname): - os.remove(local_info_fname) - if os.path.exists(local_traceback_fname): - os.remove(local_traceback_fname) + # s3.Object(S3_BUCKET_NAME, self._info_fname).delete() + # s3.Object(S3_BUCKET_NAME, self._traceback_fname).delete() + + # # remove from local temp directory as well + # local_info_fname = os.path.join(HITL_TMP_DIR, self._info_fname) + # local_traceback_fname = os.path.join(HITL_TMP_DIR, self._traceback_fname) + # if os.path.exists(local_info_fname): + # os.remove(local_info_fname) + # if os.path.exists(local_traceback_fname): + # os.remove(local_traceback_fname) + pass if __name__ == "__main__": diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py new file mode 100644 index 0000000000..ef5f419e44 --- /dev/null +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py @@ -0,0 +1,38 @@ +OS_ENV_DICT = { + "AWS_ACCESS_KEY_ID": "test_key_id", + "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", + "AWS_DEFAULT_REGION": "us-west-1" +} + +class S3ClientMock: + def __init__(self) -> None: + self.method_called = {} + + def _add_method_called_count(self, method_name): + if method_name not in self.method_called: + self.method_called[method_name] = 0 + self.method_called[method_name] += 1 + +class MockListObjResult: + def __init__(self) -> None: + print("init") + pass + + def search(search_param = None): + print(search_param) + return [ + {"Prefix": "20280224132033/"}, + {"Prefix": "20290224132013/"}, + ] + +def mock_make_api_call(caller, method_name, params): + if method_name == "ListObjects": + print("list obj") + # called list_objects + return MockListObjResult() + + + + + + \ No newline at end of file From 7173db75fa7eb2a4c0d51a1c45e0555e4a520332 Mon Sep 17 00:00:00 2001 From: bot Date: Wed, 10 Aug 2022 06:51:10 +0000 Subject: [PATCH 02/10] Automatic style fix for droidlet --- .../backend/dashboard_aws_helper.py | 2 +- .../tests/dashboard_aws_helper_test.py | 50 +++++++++++-------- .../backend/tests/s3_client_stubber.py | 19 +++---- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py index 11dbbda6bb..d40fcebd07 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py +++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py @@ -79,7 +79,7 @@ def get_job_list(): print(s3.meta.client.get_paginator("list_objects")) # pattern of YYYYMMDDHHMMSS (batch id pattern) pattern = r"([0-9]{4})(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])" - + for prefix in res.search("CommonPrefixes"): print(prefix) if prefix is not None and re.match(pattern, prefix.get("Prefix")): diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py index 6e7746c76c..e128203e31 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py @@ -6,7 +6,12 @@ import botocore.session import json -from droidlet.tools.hitl.dashboard_app.backend.tests.s3_client_stubber import OS_ENV_DICT, MockListObjResult, S3ClientMock, mock_make_api_call +from droidlet.tools.hitl.dashboard_app.backend.tests.s3_client_stubber import ( + OS_ENV_DICT, + MockListObjResult, + S3ClientMock, + mock_make_api_call, +) S3_BUCKET_NAME = "droidlet-hitl" S3_ROOT = "s3://droidlet-hitl" @@ -18,63 +23,66 @@ os.environ["HITL_TMP_DIR"] if os.getenv("HITL_TMP_DIR") else f"{os.path.expanduser('~')}/.hitl" ) + @mock_s3 class TestAWSHelper(unittest.TestCase): def setUp(self): - conn = boto3.resource('s3', region_name='us-east-1') + conn = boto3.resource("s3", region_name="us-east-1") # We need to create the bucket since this is all in Moto's 'virtual' AWS account - conn.create_bucket(Bucket='droidlet-hitl') + conn.create_bucket(Bucket="droidlet-hitl") self._info_fname = f"job_management_records/{VALID_ID}.json" self._traceback_fname = f"{VALID_ID}/log_traceback.csv" some_dict = {"msg": "hello"} json_content = json.dumps(some_dict) - s3 = boto3.client('s3', region_name='us-east-1') - s3.put_object(Bucket='droidlet-hitl', Key=self._info_fname, Body=json_content) - s3.put_object(Bucket='droidlet-hitl', Key=self._traceback_fname, Body="1, 2 \n1, 2") + s3 = boto3.client("s3", region_name="us-east-1") + s3.put_object(Bucket="droidlet-hitl", Key=self._info_fname, Body=json_content) + s3.put_object(Bucket="droidlet-hitl", Key=self._traceback_fname, Body="1, 2 \n1, 2") - @patch.dict('os.environ', OS_ENV_DICT) + @patch.dict("os.environ", OS_ENV_DICT) def test_get_job_list(self): - s3 = boto3.client('s3', region_name='us-east-1') - s3.put_object(Bucket='droidlet-hitl', Key="20220224132033/", Body="1, 2 \n1, 2") - from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( - get_job_list - ) + s3 = boto3.client("s3", region_name="us-east-1") + s3.put_object(Bucket="droidlet-hitl", Key="20220224132033/", Body="1, 2 \n1, 2") + from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import get_job_list + res = get_job_list() self.assertGreater(len(res), 0) - @patch.dict('os.environ', OS_ENV_DICT) + @patch.dict("os.environ", OS_ENV_DICT) def test_get_traceback_by_id_valid(self): from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( - get_traceback_by_id + get_traceback_by_id, ) + res, _ = get_traceback_by_id(VALID_ID) self.assertIsNotNone(res) self.assertNotEqual(res, f"cannot find traceback with id {VALID_ID}") - - @patch.dict('os.environ', OS_ENV_DICT) + @patch.dict("os.environ", OS_ENV_DICT) def test_get_traceback_by_id_invalid(self): from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( - get_traceback_by_id + get_traceback_by_id, ) + res, _ = get_traceback_by_id(INVALID_ID) self.assertEqual(res, f"cannot find traceback with id {INVALID_ID}") - @patch.dict('os.environ', OS_ENV_DICT) + @patch.dict("os.environ", OS_ENV_DICT) def test_get_run_info_by_id_valid(self): from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( - get_run_info_by_id + get_run_info_by_id, ) + res, _ = get_run_info_by_id(VALID_ID) self.assertIsNotNone(res) self.assertNotEqual(res, f"cannot find run info with id {VALID_ID}") - @patch.dict('os.environ', OS_ENV_DICT) + @patch.dict("os.environ", OS_ENV_DICT) @mock_s3 def test_get_run_info_by_id_inalid(self): from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import ( - get_run_info_by_id + get_run_info_by_id, ) + res, _ = get_run_info_by_id(INVALID_ID) self.assertEqual(res, f"cannot find run info with id {INVALID_ID}") diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py index ef5f419e44..47eb472a74 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py @@ -1,9 +1,10 @@ OS_ENV_DICT = { "AWS_ACCESS_KEY_ID": "test_key_id", "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", - "AWS_DEFAULT_REGION": "us-west-1" + "AWS_DEFAULT_REGION": "us-west-1", } + class S3ClientMock: def __init__(self) -> None: self.method_called = {} @@ -13,26 +14,22 @@ def _add_method_called_count(self, method_name): self.method_called[method_name] = 0 self.method_called[method_name] += 1 + class MockListObjResult: def __init__(self) -> None: print("init") pass - - def search(search_param = None): + + def search(search_param=None): print(search_param) return [ - {"Prefix": "20280224132033/"}, - {"Prefix": "20290224132013/"}, + {"Prefix": "20280224132033/"}, + {"Prefix": "20290224132013/"}, ] + def mock_make_api_call(caller, method_name, params): if method_name == "ListObjects": print("list obj") # called list_objects return MockListObjResult() - - - - - - \ No newline at end of file From 0dc41c5104812d0a29b231414de9bc23ce112ec1 Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Wed, 10 Aug 2022 11:10:02 -0700 Subject: [PATCH 03/10] clean up --- .../backend/dashboard_aws_helper.py | 3 -- .../tests/dashboard_aws_helper_test.py | 8 +++- .../backend/tests/s3_client_stubber.py | 38 ------------------- 3 files changed, 6 insertions(+), 43 deletions(-) diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py index 11dbbda6bb..638b7e901d 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py +++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py @@ -75,13 +75,10 @@ def get_job_list(): res = s3.meta.client.get_paginator("list_objects").paginate( Bucket=S3_BUCKET_NAME, Delimiter="/" ) - print(res) - print(s3.meta.client.get_paginator("list_objects")) # pattern of YYYYMMDDHHMMSS (batch id pattern) pattern = r"([0-9]{4})(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])" for prefix in res.search("CommonPrefixes"): - print(prefix) if prefix is not None and re.match(pattern, prefix.get("Prefix")): job_list.append(int(prefix.get("Prefix")[:-1])) return job_list diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py index 6e7746c76c..eef8ebb7c4 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py @@ -6,8 +6,6 @@ import botocore.session import json -from droidlet.tools.hitl.dashboard_app.backend.tests.s3_client_stubber import OS_ENV_DICT, MockListObjResult, S3ClientMock, mock_make_api_call - S3_BUCKET_NAME = "droidlet-hitl" S3_ROOT = "s3://droidlet-hitl" @@ -18,6 +16,12 @@ os.environ["HITL_TMP_DIR"] if os.getenv("HITL_TMP_DIR") else f"{os.path.expanduser('~')}/.hitl" ) +OS_ENV_DICT = { + "AWS_ACCESS_KEY_ID": "test_key_id", + "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", + "AWS_DEFAULT_REGION": "us-west-1" +} + @mock_s3 class TestAWSHelper(unittest.TestCase): def setUp(self): diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py index ef5f419e44..e69de29bb2 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py @@ -1,38 +0,0 @@ -OS_ENV_DICT = { - "AWS_ACCESS_KEY_ID": "test_key_id", - "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", - "AWS_DEFAULT_REGION": "us-west-1" -} - -class S3ClientMock: - def __init__(self) -> None: - self.method_called = {} - - def _add_method_called_count(self, method_name): - if method_name not in self.method_called: - self.method_called[method_name] = 0 - self.method_called[method_name] += 1 - -class MockListObjResult: - def __init__(self) -> None: - print("init") - pass - - def search(search_param = None): - print(search_param) - return [ - {"Prefix": "20280224132033/"}, - {"Prefix": "20290224132013/"}, - ] - -def mock_make_api_call(caller, method_name, params): - if method_name == "ListObjects": - print("list obj") - # called list_objects - return MockListObjResult() - - - - - - \ No newline at end of file From a353bbb980accdfd710685d611abfa57b4280b08 Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Wed, 10 Aug 2022 11:11:35 -0700 Subject: [PATCH 04/10] removed redundent file --- .../tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py deleted file mode 100644 index e69de29bb2..0000000000 From a8f828367b3bd522e4845116ce8f401d52d3d116 Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Wed, 10 Aug 2022 11:20:42 -0700 Subject: [PATCH 05/10] update tearDown, refactor names --- .../tests/dashboard_aws_helper_test.py | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py index 1bf461bf2a..0da79feac5 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py @@ -19,27 +19,27 @@ OS_ENV_DICT = { "AWS_ACCESS_KEY_ID": "test_key_id", "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", - "AWS_DEFAULT_REGION": "us-west-1" + "AWS_DEFAULT_REGION": "us-east-1" } @mock_s3 class TestAWSHelper(unittest.TestCase): def setUp(self): - conn = boto3.resource("s3", region_name="us-east-1") + conn = boto3.resource("s3", region_name=OS_ENV_DICT["AWS_DEFAULT_REGION"]) # We need to create the bucket since this is all in Moto's 'virtual' AWS account - conn.create_bucket(Bucket="droidlet-hitl") + conn.create_bucket(Bucket=S3_BUCKET_NAME) self._info_fname = f"job_management_records/{VALID_ID}.json" self._traceback_fname = f"{VALID_ID}/log_traceback.csv" some_dict = {"msg": "hello"} json_content = json.dumps(some_dict) - s3 = boto3.client("s3", region_name="us-east-1") - s3.put_object(Bucket="droidlet-hitl", Key=self._info_fname, Body=json_content) - s3.put_object(Bucket="droidlet-hitl", Key=self._traceback_fname, Body="1, 2 \n1, 2") + s3 = boto3.client("s3", region_name=OS_ENV_DICT["AWS_DEFAULT_REGION"]) + s3.put_object(Bucket=S3_BUCKET_NAME, Key=self._info_fname, Body=json_content) + s3.put_object(Bucket=S3_BUCKET_NAME, Key=self._traceback_fname, Body="1, 2 \n1, 2") @patch.dict("os.environ", OS_ENV_DICT) def test_get_job_list(self): - s3 = boto3.client("s3", region_name="us-east-1") - s3.put_object(Bucket="droidlet-hitl", Key="20220224132033/", Body="1, 2 \n1, 2") + s3 = boto3.client("s3", region_name=OS_ENV_DICT["AWS_DEFAULT_REGION"]) + s3.put_object(Bucket=S3_BUCKET_NAME, Key="20220224132033/", Body="1, 2 \n1, 2") from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import get_job_list res = get_job_list() @@ -85,17 +85,14 @@ def test_get_run_info_by_id_inalid(self): self.assertEqual(res, f"cannot find run info with id {INVALID_ID}") def tearDown(self): - # s3.Object(S3_BUCKET_NAME, self._info_fname).delete() - # s3.Object(S3_BUCKET_NAME, self._traceback_fname).delete() - - # # remove from local temp directory as well - # local_info_fname = os.path.join(HITL_TMP_DIR, self._info_fname) - # local_traceback_fname = os.path.join(HITL_TMP_DIR, self._traceback_fname) - # if os.path.exists(local_info_fname): - # os.remove(local_info_fname) - # if os.path.exists(local_traceback_fname): - # os.remove(local_traceback_fname) - pass + # no need to clean up s3 as using mock s3 client + # remove from local temp directory + local_info_fname = os.path.join(HITL_TMP_DIR, self._info_fname) + local_traceback_fname = os.path.join(HITL_TMP_DIR, self._traceback_fname) + if os.path.exists(local_info_fname): + os.remove(local_info_fname) + if os.path.exists(local_traceback_fname): + os.remove(local_traceback_fname) if __name__ == "__main__": From dd60b4d85dcf84f45f4270b22e1236782c0cb618 Mon Sep 17 00:00:00 2001 From: bot Date: Wed, 10 Aug 2022 18:21:28 +0000 Subject: [PATCH 06/10] Automatic style fix for droidlet --- .../dashboard_app/backend/tests/dashboard_aws_helper_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py index 0da79feac5..57766d9388 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py @@ -19,9 +19,10 @@ OS_ENV_DICT = { "AWS_ACCESS_KEY_ID": "test_key_id", "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", - "AWS_DEFAULT_REGION": "us-east-1" + "AWS_DEFAULT_REGION": "us-east-1", } + @mock_s3 class TestAWSHelper(unittest.TestCase): def setUp(self): @@ -86,7 +87,7 @@ def test_get_run_info_by_id_inalid(self): def tearDown(self): # no need to clean up s3 as using mock s3 client - # remove from local temp directory + # remove from local temp directory local_info_fname = os.path.join(HITL_TMP_DIR, self._info_fname) local_traceback_fname = os.path.join(HITL_TMP_DIR, self._traceback_fname) if os.path.exists(local_info_fname): From c279fcb5eccd4f8654a99905a1d9a2d26f07a3cd Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Wed, 10 Aug 2022 11:24:34 -0700 Subject: [PATCH 07/10] moved unnecessary import --- .../dashboard_app/backend/tests/dashboard_aws_helper_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py index 0da79feac5..ec42212e96 100644 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py +++ b/droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py @@ -3,7 +3,6 @@ from unittest.mock import patch import boto3 from moto import mock_s3 -import botocore.session import json S3_BUCKET_NAME = "droidlet-hitl" From b697ec89517c4efd2224f9db46f33e2caac2add4 Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Wed, 10 Aug 2022 16:03:38 -0700 Subject: [PATCH 08/10] update readme --- droidlet/tools/hitl/dashboard_app/README.MD | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/droidlet/tools/hitl/dashboard_app/README.MD b/droidlet/tools/hitl/dashboard_app/README.MD index c0e705242f..a9c1f5fb00 100644 --- a/droidlet/tools/hitl/dashboard_app/README.MD +++ b/droidlet/tools/hitl/dashboard_app/README.MD @@ -1,9 +1,11 @@ # Dashboard App for HITL -Updated July 13 by Chuxi. +Updated Aug 10 by Chuxi. This is a Dashboard app prototype for the HITL system. ## Update Note +- Aug 10: + - Updated dashboard_aws_helper_test.py to use mock s3. - July 14: - Added model visualization component, user can see text info about a model: - Demo: @@ -53,6 +55,8 @@ Please note this dashboard app is currently only in development build and is not - Dependency required for backend server (in addition to the droidelet setting, please make sure you have the updated version as below): Flask==2.1.2 Flask-SocketIO==5.2.0 +- You need to have the following python package to be able to run the tests: +moto==3.1.17 - Dependency for frontend is specified in the `fairo/droidlet/tools/hitl/dashboard_app/dashboard_frontend/package.json` file. ## How to run From 3d820d4e2a9a67f7e868fc15508ec59777638a6f Mon Sep 17 00:00:00 2001 From: Chuxi Wang <51009396+mialsy@users.noreply.github.com> Date: Wed, 10 Aug 2022 16:04:24 -0700 Subject: [PATCH 09/10] Delete s3_client_stubber.py --- .../backend/tests/s3_client_stubber.py | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py diff --git a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py b/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py deleted file mode 100644 index 47eb472a74..0000000000 --- a/droidlet/tools/hitl/dashboard_app/backend/tests/s3_client_stubber.py +++ /dev/null @@ -1,35 +0,0 @@ -OS_ENV_DICT = { - "AWS_ACCESS_KEY_ID": "test_key_id", - "AWS_SECRET_ACCESS_KEY": "secretkkkkkk", - "AWS_DEFAULT_REGION": "us-west-1", -} - - -class S3ClientMock: - def __init__(self) -> None: - self.method_called = {} - - def _add_method_called_count(self, method_name): - if method_name not in self.method_called: - self.method_called[method_name] = 0 - self.method_called[method_name] += 1 - - -class MockListObjResult: - def __init__(self) -> None: - print("init") - pass - - def search(search_param=None): - print(search_param) - return [ - {"Prefix": "20280224132033/"}, - {"Prefix": "20290224132013/"}, - ] - - -def mock_make_api_call(caller, method_name, params): - if method_name == "ListObjects": - print("list obj") - # called list_objects - return MockListObjResult() From e0348acd8ee70af60a3251cd6adf58a50d8ec68a Mon Sep 17 00:00:00 2001 From: Chuxi Wang Date: Thu, 11 Aug 2022 15:31:09 -0700 Subject: [PATCH 10/10] add moto dependency in the requirements --- agents/craftassist/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agents/craftassist/requirements.txt b/agents/craftassist/requirements.txt index 2693cfeef8..f15f8660e7 100644 --- a/agents/craftassist/requirements.txt +++ b/agents/craftassist/requirements.txt @@ -1,2 +1,3 @@ -r ../../requirements.txt -eventlet \ No newline at end of file +eventlet +moto[ec2,s3,all] \ No newline at end of file