Add regression flag
PiperOrigin-RevId: 585548529
diff --git a/ui_automator/ui_automator.py b/ui_automator/ui_automator.py
index 5a551a7..6670fb1 100644
--- a/ui_automator/ui_automator.py
+++ b/ui_automator/ui_automator.py
@@ -48,17 +48,31 @@
'Use --commission {DeviceName},{PairingCode},{GHARoom} to commission a'
' device to google fabric on GHA.'
)
+_REGRESSION_TESTS_FLAG_USAGE_GUIDE = (
+ 'Use --regtest to run regression tests infinitely. Add --repeat'
+ ' <repeat-times> to stop after repeat-times. `--regtest` must be'
+ ' used with `--commission`.'
+)
_COMMISSION = flags.DEFINE_list(
name='commission',
default=None,
help=_COMMISSIONING_FLAG_USAGE_GUIDE,
)
-
_DECOMMISSION = flags.DEFINE_string(
name='decommission',
default='',
help='Use --decommission {DeviceName} to remove the device on GHA.',
)
+_RUN_REGRESSION_TESTS = flags.DEFINE_boolean(
+ name='regtest',
+ default=False,
+ help=_REGRESSION_TESTS_FLAG_USAGE_GUIDE,
+)
+_REPEAT = flags.DEFINE_integer(
+ name='repeat',
+ default=None,
+ help=_REGRESSION_TESTS_FLAG_USAGE_GUIDE,
+)
def _validate_commissioning_arg(
@@ -294,6 +308,50 @@
f' on device({self._connected_device.device_info["serial"]}).'
) from e
+ def run_regression_tests(
+ self, repeat: int | None, *args: Any,
+ ) -> None:
+ """Executes automated regression tests.
+
+ A single execution of both commissioning and decommissioning constitutes one
+ cycle.
+
+ Args:
+ repeat: The value of flag `repeat`. If the value is None, regression
+ tests will be run repeatedly until keyboard interrupts.
+ *args: Any required value to run regression tests.
+
+ Raises:
+ ValueError: When the value of flag `repeat` is not positive.
+ """
+ if repeat and repeat <= 0:
+ raise ValueError('Number placed after `--repeat` must be positive.')
+
+ failure_count = 0
+ run_count = 0
+ self._logger.info(
+ 'Start running regression tests'
+ f' {str(repeat) + " times" if repeat is not None else "continuously"}.'
+ )
+ while repeat is None or run_count < repeat:
+ try:
+ device_name, _, _ = args
+ self.commission_device(*args)
+ self.decommission_device(device_name)
+ except errors.MoblySnippetError:
+ failure_count += 1
+ except KeyboardInterrupt:
+ self._logger.info('Tests interrupted by keyboard.')
+ break
+ run_count += 1
+
+ self._logger.info(
+ 'Ran %d times. Passed %d times. Failed %d times.',
+ run_count,
+ run_count - failure_count,
+ failure_count,
+ )
+
def _get_mbs_apk_path(self) -> str:
return os.path.join(
os.path.dirname(os.path.abspath(__file__)),
@@ -308,11 +366,14 @@
if _COMMISSION.value:
if len(_COMMISSION.value) != 3:
raise flags.IllegalFlagValueError(_COMMISSIONING_FLAG_USAGE_GUIDE)
-
- ui_automator.commission_device(*_COMMISSION.value)
-
- if _DECOMMISSION.value:
+ if _RUN_REGRESSION_TESTS.value:
+ ui_automator.run_regression_tests(_REPEAT.value, *_COMMISSION.value)
+ else:
+ ui_automator.commission_device(*_COMMISSION.value)
+ elif _DECOMMISSION.value:
ui_automator.decommission_device(_DECOMMISSION.value)
+ elif _RUN_REGRESSION_TESTS.value:
+ raise flags.IllegalFlagValueError(_REGRESSION_TESTS_FLAG_USAGE_GUIDE)
# TODO(b/309745485): Type of argv should be Sequence[str].
diff --git a/ui_automator/ui_automator_test.py b/ui_automator/ui_automator_test.py
index 671d41a..1e44ee9 100644
--- a/ui_automator/ui_automator_test.py
+++ b/ui_automator/ui_automator_test.py
@@ -51,6 +51,20 @@
'--decommission',
'm5stack',
]
+_FAKE_VALID_SYS_ARGV_FOR_REGRESSION_TESTS = [
+ _PYTHON_BIN_PATH + 'ui-automator',
+ '--commission',
+ 'm5stack,34970112332,Office',
+ '--regtest',
+ '--repeat',
+ '5',
+]
+_FAKE_INVALID_SYS_ARGV_FOR_REGRESSION_TESTS = [
+ _PYTHON_BIN_PATH + 'ui-automator',
+ '--regtest',
+ '--repeat',
+ '5',
+]
_FAKE_SYS_ARGV_FOR_COMMISSIONING_WITH_INVALID_LENGTH = [
_PYTHON_BIN_PATH + 'ui-automator',
'--commission',
@@ -457,12 +471,8 @@
)
mock_exit.assert_called_once()
- @flagsaver.flagsaver(
- (ui_automator._COMMISSION, ['m5'])
- )
- def test_commission_with_cmd_invalid_arg_should_raise_an_error(
- self
- ):
+ @flagsaver.flagsaver((ui_automator._COMMISSION, ['m5']))
+ def test_commission_with_cmd_invalid_arg_should_raise_an_error(self):
with mock.patch.object(
sys, 'argv', _FAKE_SYS_ARGV_FOR_COMMISSIONING_WITH_INVALID_LENGTH
):
@@ -561,6 +571,176 @@
cm.output[2], 'INFO:root:Successfully remove the device on GHA.'
)
+ @flagsaver.flagsaver((ui_automator._RUN_REGRESSION_TESTS, True))
+ @flagsaver.flagsaver((ui_automator._REPEAT, 5))
+ @flagsaver.flagsaver(
+ (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office'])
+ )
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'run_regression_tests', autospec=True
+ )
+ @mock.patch.object(sys, 'exit', autospec=True)
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'commission_device', autospec=True
+ )
+ def test_run_calls_run_regression_tests_with_valid_arguments(
+ self, mock_commission_device, mock_exit, mock_run_regression_tests
+ ):
+ with mock.patch.object(
+ sys, 'argv', _FAKE_VALID_SYS_ARGV_FOR_REGRESSION_TESTS
+ ):
+ ui_automator.run()
+
+ mock_run_regression_tests.assert_called_once_with(
+ mock.ANY, 5, *['m5stack', '34970112332', 'Office']
+ )
+ mock_commission_device.assert_not_called()
+ mock_exit.assert_called_once()
+
+ @flagsaver.flagsaver((ui_automator._RUN_REGRESSION_TESTS, True))
+ @flagsaver.flagsaver((ui_automator._REPEAT, 5))
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'run_regression_tests', autospec=True
+ )
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'commission_device', autospec=True
+ )
+ def test_run_regression_tests_with_invalid_args_should_raise_an_error(
+ self, mock_commission_device, mock_run_regression_tests
+ ):
+ with mock.patch.object(
+ sys, 'argv', _FAKE_INVALID_SYS_ARGV_FOR_REGRESSION_TESTS
+ ):
+ with self.assertRaisesRegex(
+ flags.IllegalFlagValueError,
+ (
+ 'Use --regtest to run regression tests infinitely. Add --repeat'
+ ' <repeat-times> to stop after repeat-times. `--regtest` must be'
+ ' used with `--commission`.'
+ ),
+ ):
+ ui_automator.run()
+
+ mock_commission_device.assert_not_called()
+ mock_run_regression_tests.assert_not_called()
+
+ @flagsaver.flagsaver(
+ (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office'])
+ )
+ @mock.patch.object(android_device, 'get_all_instances', autospec=True)
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'commission_device', autospec=True
+ )
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'decommission_device', autospec=True
+ )
+ def test_run_regression_tests_should_be_conducted_for_the_designated_number_of_cycles(
+ self,
+ mock_decommission_device,
+ mock_commission_device,
+ mock_get_all_instances,
+ ):
+ mock_get_all_instances.return_value = [self.mock_android_device]
+
+ with self.assertLogs() as cm:
+ self.ui_automator.run_regression_tests(
+ 5, *['m5stack', '34970112332', 'Office']
+ )
+
+ self.assertEqual(mock_commission_device.call_count, 5)
+ self.assertEqual(mock_decommission_device.call_count, 5)
+ self.assertEqual(
+ cm.output[0], 'INFO:root:Start running regression tests 5 times.'
+ )
+ self.assertEqual(
+ cm.output[1], 'INFO:root:Ran 5 times. Passed 5 times. Failed 0 times.'
+ )
+
+ @flagsaver.flagsaver(
+ (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office'])
+ )
+ @mock.patch.object(android_device, 'get_all_instances', autospec=True)
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'commission_device', autospec=True
+ )
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'decommission_device', autospec=True
+ )
+ def test_run_regression_tests_executes_for_given_number_of_times_with_failure(
+ self,
+ mock_decommission_device,
+ mock_commission_device,
+ mock_get_all_instances,
+ ):
+ mock_commission_device.side_effect = [
+ None,
+ None,
+ errors.MoblySnippetError('fake_error'),
+ None,
+ None,
+ ]
+ mock_get_all_instances.return_value = [self.mock_android_device]
+
+ with self.assertLogs() as cm:
+ self.ui_automator.run_regression_tests(
+ 5, *['m5stack', '34970112332', 'Office']
+ )
+
+ self.assertEqual(mock_commission_device.call_count, 5)
+ self.assertEqual(mock_decommission_device.call_count, 4)
+ self.assertEqual(
+ cm.output[0], 'INFO:root:Start running regression tests 5 times.'
+ )
+ self.assertEqual(
+ cm.output[1], 'INFO:root:Ran 5 times. Passed 4 times. Failed 1 times.'
+ )
+
+ def test_run_regression_tests_raises_an_error_with_invalid_input(self):
+ with self.assertRaisesRegex(
+ ValueError, 'Number placed after `--repeat` must be positive.'
+ ):
+ self.ui_automator.run_regression_tests(-5)
+
+ @flagsaver.flagsaver(
+ (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office'])
+ )
+ @mock.patch.object(android_device, 'get_all_instances', autospec=True)
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'commission_device', autospec=True
+ )
+ @mock.patch.object(
+ ui_automator.UIAutomator, 'decommission_device', autospec=True
+ )
+ def test_run_regression_tests_runs_continuously_until_keyboard_interrupts(
+ self,
+ mock_decommission_device,
+ mock_commission_device,
+ mock_get_all_instances,
+ ):
+ mock_commission_device.side_effect = [
+ None,
+ errors.MoblySnippetError('fake_error'),
+ None,
+ errors.MoblySnippetError('fake_error'),
+ None,
+ KeyboardInterrupt(),
+ ]
+ mock_get_all_instances.return_value = [self.mock_android_device]
+ with self.assertLogs() as cm:
+ self.ui_automator.run_regression_tests(
+ None, *['m5stack', '34970112332', 'Office']
+ )
+
+ self.assertEqual(mock_commission_device.call_count, 6)
+ self.assertEqual(mock_decommission_device.call_count, 3)
+ self.assertEqual(
+ cm.output[0], 'INFO:root:Start running regression tests continuously.'
+ )
+ self.assertEqual(cm.output[1], 'INFO:root:Tests interrupted by keyboard.')
+ self.assertEqual(
+ cm.output[2], 'INFO:root:Ran 5 times. Passed 3 times. Failed 2 times.'
+ )
+
if __name__ == '__main__':
unittest.main()
