Add regression flag PiperOrigin-RevId: 585548529
diff --git a/ui_automator/ui_automator.py b/ui_automator/ui_automator.py index 5a551a7..6670fb1 100644 --- a/ui_automator/ui_automator.py +++ b/ui_automator/ui_automator.py
@@ -48,17 +48,31 @@ 'Use --commission {DeviceName},{PairingCode},{GHARoom} to commission a' ' device to google fabric on GHA.' ) +_REGRESSION_TESTS_FLAG_USAGE_GUIDE = ( + 'Use --regtest to run regression tests infinitely. Add --repeat' + ' <repeat-times> to stop after repeat-times. `--regtest` must be' + ' used with `--commission`.' +) _COMMISSION = flags.DEFINE_list( name='commission', default=None, help=_COMMISSIONING_FLAG_USAGE_GUIDE, ) - _DECOMMISSION = flags.DEFINE_string( name='decommission', default='', help='Use --decommission {DeviceName} to remove the device on GHA.', ) +_RUN_REGRESSION_TESTS = flags.DEFINE_boolean( + name='regtest', + default=False, + help=_REGRESSION_TESTS_FLAG_USAGE_GUIDE, +) +_REPEAT = flags.DEFINE_integer( + name='repeat', + default=None, + help=_REGRESSION_TESTS_FLAG_USAGE_GUIDE, +) def _validate_commissioning_arg( @@ -294,6 +308,50 @@ f' on device({self._connected_device.device_info["serial"]}).' ) from e + def run_regression_tests( + self, repeat: int | None, *args: Any, + ) -> None: + """Executes automated regression tests. + + A single execution of both commissioning and decommissioning constitutes one + cycle. + + Args: + repeat: The value of flag `repeat`. If the value is None, regression + tests will be run repeatedly until keyboard interrupts. + *args: Any required value to run regression tests. + + Raises: + ValueError: When the value of flag `repeat` is not positive. + """ + if repeat and repeat <= 0: + raise ValueError('Number placed after `--repeat` must be positive.') + + failure_count = 0 + run_count = 0 + self._logger.info( + 'Start running regression tests' + f' {str(repeat) + " times" if repeat is not None else "continuously"}.' + ) + while repeat is None or run_count < repeat: + try: + device_name, _, _ = args + self.commission_device(*args) + self.decommission_device(device_name) + except errors.MoblySnippetError: + failure_count += 1 + except KeyboardInterrupt: + self._logger.info('Tests interrupted by keyboard.') + break + run_count += 1 + + self._logger.info( + 'Ran %d times. Passed %d times. Failed %d times.', + run_count, + run_count - failure_count, + failure_count, + ) + def _get_mbs_apk_path(self) -> str: return os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -308,11 +366,14 @@ if _COMMISSION.value: if len(_COMMISSION.value) != 3: raise flags.IllegalFlagValueError(_COMMISSIONING_FLAG_USAGE_GUIDE) - - ui_automator.commission_device(*_COMMISSION.value) - - if _DECOMMISSION.value: + if _RUN_REGRESSION_TESTS.value: + ui_automator.run_regression_tests(_REPEAT.value, *_COMMISSION.value) + else: + ui_automator.commission_device(*_COMMISSION.value) + elif _DECOMMISSION.value: ui_automator.decommission_device(_DECOMMISSION.value) + elif _RUN_REGRESSION_TESTS.value: + raise flags.IllegalFlagValueError(_REGRESSION_TESTS_FLAG_USAGE_GUIDE) # TODO(b/309745485): Type of argv should be Sequence[str].
diff --git a/ui_automator/ui_automator_test.py b/ui_automator/ui_automator_test.py index 671d41a..1e44ee9 100644 --- a/ui_automator/ui_automator_test.py +++ b/ui_automator/ui_automator_test.py
@@ -51,6 +51,20 @@ '--decommission', 'm5stack', ] +_FAKE_VALID_SYS_ARGV_FOR_REGRESSION_TESTS = [ + _PYTHON_BIN_PATH + 'ui-automator', + '--commission', + 'm5stack,34970112332,Office', + '--regtest', + '--repeat', + '5', +] +_FAKE_INVALID_SYS_ARGV_FOR_REGRESSION_TESTS = [ + _PYTHON_BIN_PATH + 'ui-automator', + '--regtest', + '--repeat', + '5', +] _FAKE_SYS_ARGV_FOR_COMMISSIONING_WITH_INVALID_LENGTH = [ _PYTHON_BIN_PATH + 'ui-automator', '--commission', @@ -457,12 +471,8 @@ ) mock_exit.assert_called_once() - @flagsaver.flagsaver( - (ui_automator._COMMISSION, ['m5']) - ) - def test_commission_with_cmd_invalid_arg_should_raise_an_error( - self - ): + @flagsaver.flagsaver((ui_automator._COMMISSION, ['m5'])) + def test_commission_with_cmd_invalid_arg_should_raise_an_error(self): with mock.patch.object( sys, 'argv', _FAKE_SYS_ARGV_FOR_COMMISSIONING_WITH_INVALID_LENGTH ): @@ -561,6 +571,176 @@ cm.output[2], 'INFO:root:Successfully remove the device on GHA.' ) + @flagsaver.flagsaver((ui_automator._RUN_REGRESSION_TESTS, True)) + @flagsaver.flagsaver((ui_automator._REPEAT, 5)) + @flagsaver.flagsaver( + (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office']) + ) + @mock.patch.object( + ui_automator.UIAutomator, 'run_regression_tests', autospec=True + ) + @mock.patch.object(sys, 'exit', autospec=True) + @mock.patch.object( + ui_automator.UIAutomator, 'commission_device', autospec=True + ) + def test_run_calls_run_regression_tests_with_valid_arguments( + self, mock_commission_device, mock_exit, mock_run_regression_tests + ): + with mock.patch.object( + sys, 'argv', _FAKE_VALID_SYS_ARGV_FOR_REGRESSION_TESTS + ): + ui_automator.run() + + mock_run_regression_tests.assert_called_once_with( + mock.ANY, 5, *['m5stack', '34970112332', 'Office'] + ) + mock_commission_device.assert_not_called() + mock_exit.assert_called_once() + + @flagsaver.flagsaver((ui_automator._RUN_REGRESSION_TESTS, True)) + @flagsaver.flagsaver((ui_automator._REPEAT, 5)) + @mock.patch.object( + ui_automator.UIAutomator, 'run_regression_tests', autospec=True + ) + @mock.patch.object( + ui_automator.UIAutomator, 'commission_device', autospec=True + ) + def test_run_regression_tests_with_invalid_args_should_raise_an_error( + self, mock_commission_device, mock_run_regression_tests + ): + with mock.patch.object( + sys, 'argv', _FAKE_INVALID_SYS_ARGV_FOR_REGRESSION_TESTS + ): + with self.assertRaisesRegex( + flags.IllegalFlagValueError, + ( + 'Use --regtest to run regression tests infinitely. Add --repeat' + ' <repeat-times> to stop after repeat-times. `--regtest` must be' + ' used with `--commission`.' + ), + ): + ui_automator.run() + + mock_commission_device.assert_not_called() + mock_run_regression_tests.assert_not_called() + + @flagsaver.flagsaver( + (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office']) + ) + @mock.patch.object(android_device, 'get_all_instances', autospec=True) + @mock.patch.object( + ui_automator.UIAutomator, 'commission_device', autospec=True + ) + @mock.patch.object( + ui_automator.UIAutomator, 'decommission_device', autospec=True + ) + def test_run_regression_tests_should_be_conducted_for_the_designated_number_of_cycles( + self, + mock_decommission_device, + mock_commission_device, + mock_get_all_instances, + ): + mock_get_all_instances.return_value = [self.mock_android_device] + + with self.assertLogs() as cm: + self.ui_automator.run_regression_tests( + 5, *['m5stack', '34970112332', 'Office'] + ) + + self.assertEqual(mock_commission_device.call_count, 5) + self.assertEqual(mock_decommission_device.call_count, 5) + self.assertEqual( + cm.output[0], 'INFO:root:Start running regression tests 5 times.' + ) + self.assertEqual( + cm.output[1], 'INFO:root:Ran 5 times. Passed 5 times. Failed 0 times.' + ) + + @flagsaver.flagsaver( + (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office']) + ) + @mock.patch.object(android_device, 'get_all_instances', autospec=True) + @mock.patch.object( + ui_automator.UIAutomator, 'commission_device', autospec=True + ) + @mock.patch.object( + ui_automator.UIAutomator, 'decommission_device', autospec=True + ) + def test_run_regression_tests_executes_for_given_number_of_times_with_failure( + self, + mock_decommission_device, + mock_commission_device, + mock_get_all_instances, + ): + mock_commission_device.side_effect = [ + None, + None, + errors.MoblySnippetError('fake_error'), + None, + None, + ] + mock_get_all_instances.return_value = [self.mock_android_device] + + with self.assertLogs() as cm: + self.ui_automator.run_regression_tests( + 5, *['m5stack', '34970112332', 'Office'] + ) + + self.assertEqual(mock_commission_device.call_count, 5) + self.assertEqual(mock_decommission_device.call_count, 4) + self.assertEqual( + cm.output[0], 'INFO:root:Start running regression tests 5 times.' + ) + self.assertEqual( + cm.output[1], 'INFO:root:Ran 5 times. Passed 4 times. Failed 1 times.' + ) + + def test_run_regression_tests_raises_an_error_with_invalid_input(self): + with self.assertRaisesRegex( + ValueError, 'Number placed after `--repeat` must be positive.' + ): + self.ui_automator.run_regression_tests(-5) + + @flagsaver.flagsaver( + (ui_automator._COMMISSION, ['m5stack', '34970112332', 'Office']) + ) + @mock.patch.object(android_device, 'get_all_instances', autospec=True) + @mock.patch.object( + ui_automator.UIAutomator, 'commission_device', autospec=True + ) + @mock.patch.object( + ui_automator.UIAutomator, 'decommission_device', autospec=True + ) + def test_run_regression_tests_runs_continuously_until_keyboard_interrupts( + self, + mock_decommission_device, + mock_commission_device, + mock_get_all_instances, + ): + mock_commission_device.side_effect = [ + None, + errors.MoblySnippetError('fake_error'), + None, + errors.MoblySnippetError('fake_error'), + None, + KeyboardInterrupt(), + ] + mock_get_all_instances.return_value = [self.mock_android_device] + with self.assertLogs() as cm: + self.ui_automator.run_regression_tests( + None, *['m5stack', '34970112332', 'Office'] + ) + + self.assertEqual(mock_commission_device.call_count, 6) + self.assertEqual(mock_decommission_device.call_count, 3) + self.assertEqual( + cm.output[0], 'INFO:root:Start running regression tests continuously.' + ) + self.assertEqual(cm.output[1], 'INFO:root:Tests interrupted by keyboard.') + self.assertEqual( + cm.output[2], 'INFO:root:Ran 5 times. Passed 3 times. Failed 2 times.' + ) + if __name__ == '__main__': unittest.main()