Home LLM的test oracle生成

# robotframework-webservice

# 简介

  • 一个管理 Robot Framework 任务 / 测试的 Web 服务。
  • 该 Web 服务应启动 Robot Framework 的任务 / 测试，并返回并缓存相应的报告。
  • 运行：

1
2
pip install robotframework-webservice
python -m RobotFrameworkService.main -p 5003 -t path_to_my_taskfolder

  • 测试

1
2
3
4
5
6
7
8
9
http://localhost:5003/robotframework/run/mytask //调用 Robot 任务/测试

http://localhost:5003/robotframework/run/mytask?myVariable1=42&anotherVariable=Mustermann //带变量调用 Robot 任务/测试

# 连接在任务/测试的持续时间内保持打开状态
http://localhost:5003/robotframework/run/mytask

# 立即关闭连接 - 结果必须使用 x-request-id 请求
http://localhost:5003/robotframework/run/mytask/async

  • 调用 Robot 测试

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    curl -X 'POST' \
      'http://localhost:5003/robotframework/run' \
      -H 'accept: application/json' \
      -H 'Content-Type: application/json' \
      -d '{
      "paths": [
        "examples"
      ],
      "test": "Demonstration Test"
    }'

  • 调用 Robot 任务

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    curl -X 'POST' \
      'http://localhost:5003/robotframework/run' \
      -H 'accept: application/json' \
      -H 'Content-Type: application/json' \
      -d '{
      "paths": [
        "examples"
      ],
      "task": "Demonstration Task"
    }'

  • 带变量调用 Robot 任务

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    curl -X 'POST' \
      'http://localhost:5003/robotframework/run' \
      -H 'accept: application/json' \
      -H 'Content-Type: application/json' \
      -d '{
      "paths": [
        "examples"
      ],
      "task": "Task with more variables",
      "variables": {
        "firstname": "Max",
        "lastname": "Mustermann"
      }
    }'

  • 启动本地 Web 服务：

    1
    2
    3
    python -m RobotFrameworkService.main
    python -m RobotFrameworkService.main --help
    python -m RobotFrameworkService.main -p 5003 -t path_to_my_taskfolder // example

    # test_app.py 修改操作

    # 原本的 test.py 代码:

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    from fastapi.testclient import TestClient
    import unittest
    from RobotFrameworkService.main import app


    class EndpointTesttest_s(unittest.TestCase):
        def test_is_service_available(self):
            response = self.__get_robot_webservice("/status")

        def test_is_robottask_startable(self):
            response = self.__get_robot_webservice(
                "/robotframework/run/all", expected_response_code=400
            )
            self.__is_robot_failed(response=response)

        def test_is_robottask_async_startable(self):
            self.__get_robot_webservice("/robotframework/run/all/async")

        def test_is_robottask_startable(self):
            response = self.__get_robot_webservice("/robotframework/run/anotherTask")
            self.__is_robot_passed(response=response)

        def test_is_robottask_async_startable(self):
            self.__get_robot_webservice("/robotframework/run/anotherTask/async")

        def test_robottask_with_variables(self):
            response = self.__get_robot_webservice(
                "/robotframework/run/Task with variable?input=qwerty"
            )
            self.__is_robot_passed(response=response, msg="Testing with variables failed")

        def test_is_robottask_available_with_logs(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show/anotherTask"
            )
            self.assertIn("PASS", response.text)

        def test_is_robottask_available_with_reports(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show_report/anotherTask"
            )
            self.assertIn("PASS", response.text)

        def test_is_robottask_available_with_logs_and_arguments(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show/anotherTask?art=tests&description=EreichbarkeitsTestMitLogs"
            )
            self.assertIn("PASS", response.text)

        def test_is_robottask_available_with_reports_and_arguments(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show_report/anotherTask?art=tests&description=FunktionsTestMitReports"
            )
            self.assertIn("PASS", response.text)

        def test_is_robotlog_available(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                logs_response = client.get(f"/robotframework/show_log/{execution_id}")
            self.assertEqual(200, logs_response.status_code)

        def test_is_robotreport_available(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                report_response = client.get(f"/robotframework/show_report/{execution_id}")
            self.assertEqual(200, report_response.status_code)

        def test_is_robot_run(self):
            with TestClient(app) as client:
                response = client.post("/robotframework/run", json={"task": "Another task", "test": "Demonstration Test"})
                self.assertEqual(400, response.status_code)
                self.assertEqual("Options test and task cannot be both specified", response.text)

                response = client.post("/robotframework/run", json={"task": "Another task", "sync": True})
                self.assertEqual(200, response.status_code)

                response = client.post("/robotframework/run", json={"paths": ["examples"], "test": "Demonstration Test", "sync": True})
                self.assertEqual(200, response.status_code)

        def test_delete_robotlogs(self):
            with TestClient(app) as client:
                response = client.delete("/robotframework/logs/not_existing")
                self.assertEqual(404, response.status_code)
                self.assertEqual("The logs not_existing not existing or being generating", response.text)

                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                response = client.delete(f"/robotframework/logs/{execution_id}")
                self.assertEqual(204, response.status_code)

        def __get_robot_webservice(self, endpoint, expected_response_code=200):
            with TestClient(app) as client:
                response = client.get(endpoint)
            self.assertEqual(expected_response_code, response.status_code, response.text)
            return response

        def __is_robot_passed(self, response, msg=None):
            self.assertNotIn("FAIL", response.text, msg=msg)
            self.assertIn(
                "PASS", response.text, "Test result contains neither PASS nor FAIL"
            )

        def __is_robot_failed(self, response, msg=None):
            self.assertIn("FAIL", response.text, "Test result contains FAIL")

    # 投喂给 gpt3.5 的 test 代码：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    from fastapi.testclient import TestClient
    import unittest
    from RobotFrameworkService.main import app


    class EndpointTesttest_s(unittest.TestCase):
        def test_is_service_available(self):
            response = self.__get_robot_webservice("/status")

        def test_is_robottask_startable(self):
            response = self.__get_robot_webservice(
                "/robotframework/run/all", expected_response_code=400
            )
            self.__is_robot_failed(response=response)

        def test_is_robottask_async_startable(self):
            self.__get_robot_webservice("/robotframework/run/all/async")

        def test_is_robottask_startable(self):
            response = self.__get_robot_webservice("/robotframework/run/anotherTask")
            self.__is_robot_passed(response=response)

        def test_is_robottask_async_startable(self):
            self.__get_robot_webservice("/robotframework/run/anotherTask/async")

        def test_robottask_with_variables(self):
            response = self.__get_robot_webservice(
                "/robotframework/run/Task with variable?input=qwerty"
            )
            self.__is_robot_passed(response=response, msg="Testing with variables failed")

        def test_is_robottask_available_with_logs(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show/anotherTask"
            )

        def test_is_robottask_available_with_reports(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show_report/anotherTask"
            )

        def test_is_robottask_available_with_logs_and_arguments(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show/anotherTask?art=tests&description=EreichbarkeitsTestMitLogs"
            )

        def test_is_robottask_available_with_reports_and_arguments(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show_report/anotherTask?art=tests&description=FunktionsTestMitReports"
            )

        def test_is_robotlog_available(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                logs_response = client.get(f"/robotframework/show_log/{execution_id}")

        def test_is_robotreport_available(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                report_response = client.get(f"/robotframework/show_report/{execution_id}")

        def test_is_robot_run(self):
            pass
        def test_delete_robotlogs(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]

        def __get_robot_webservice(self, endpoint, expected_response_code=200):
            with TestClient(app) as client:
                response = client.get(endpoint)
            return response

        def __is_robot_passed(self, response, msg=None):
            pass

        def __is_robot_failed(self, response, msg=None):
            pass

    # gpt 自动填充后的 test.py

    将上述代码喂给 chatgpt3.5，并附加 base prompt: 该 tests case 缺少 assert 断言 请你自动补充

    生成的 test.py 如下：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    from fastapi.testclient import TestClient
    import unittest
    from RobotFrameworkService.main import app

    class EndpointTests(unittest.TestCase):
        def test_is_service_available(self):
            response = self.__get_robot_webservice("/status")
            # 断言响应状态码为200
            assert response.status_code == 200, "Service is not available"

        def test_is_robottask_startable(self):
            response = self.__get_robot_webservice(
                "/robotframework/run/all", expected_response_code=400
            )
            self.__is_robot_failed(response=response)
            # 断言响应状态码为400
            assert response.status_code == 400, "Expected response code 400"

        def test_is_robottask_async_startable(self):
            response = self.__get_robot_webservice("/robotframework/run/all/async")
            # 断言响应状态码为200
            assert response.status_code == 200, "Async task is not startable"

        def test_is_robottask_startable(self):
            response = self.__get_robot_webservice("/robotframework/run/anotherTask")
            self.__is_robot_passed(response=response)
            # 断言响应状态码为200
            assert response.status_code == 200, "Task is not startable"

        def test_is_robottask_async_startable(self):
            response = self.__get_robot_webservice("/robotframework/run/anotherTask/async")
            # 断言响应状态码为200
            assert response.status_code == 200, "Async task is not startable"

        def test_robottask_with_variables(self):
            response = self.__get_robot_webservice(
                "/robotframework/run/Task with variable?input=qwerty"
            )
            self.__is_robot_passed(response=response, msg="Testing with variables failed")
            # 断言响应状态码为200
            assert response.status_code == 200, "Task with variables failed"

        def test_is_robottask_available_with_logs(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show/anotherTask"
            )
            # 断言响应状态码为200
            assert response.status_code == 200, "Task with logs is not available"

        def test_is_robottask_available_with_reports(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show_report/anotherTask"
            )
            # 断言响应状态码为200
            assert response.status_code == 200, "Task with reports is not available"

        def test_is_robottask_available_with_logs_and_arguments(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show/anotherTask?art=tests&description=EreichbarkeitsTestMitLogs"
            )
            # 断言响应状态码为200
            assert response.status_code == 200, "Task with logs and arguments is not available"

        def test_is_robottask_available_with_reports_and_arguments(self):
            response = self.__get_robot_webservice(
                "/robotframework/run_and_show_report/anotherTask?art=tests&description=FunktionsTestMitReports"
            )
            # 断言响应状态码为200
            assert response.status_code == 200, "Task with reports and arguments is not available"

        def test_is_robotlog_available(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                logs_response = client.get(f"/robotframework/show_log/{execution_id}")
                # 断言响应状态码为200
                assert logs_response.status_code == 200, "Logs are not available"

        def test_is_robotreport_available(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                report_response = client.get(f"/robotframework/show_report/{execution_id}")
                # 断言响应状态码为200
                assert report_response.status_code == 200, "Report is not available"

        def test_is_robot_run(self):
            # 尚未实现
            pass

        def test_delete_robotlogs(self):
            with TestClient(app) as client:
                run_response = client.get("/robotframework/run/anotherTask")
                execution_id = run_response.headers["x-request-id"]
                delete_response = client.delete(f"/robotframework/delete_log/{execution_id}")
                # 断言响应状态码为200
                assert delete_response.status_code == 200, "Logs are not deleted"

        def __get_robot_webservice(self, endpoint, expected_response_code=200):
            with TestClient(app) as client:
                response = client.get(endpoint)
                # 断言响应状态码为预期的状态码
                assert response.status_code == expected_response_code, f"Expected response code {expected_response_code}, got {response.status_code}"
            return response

        def __is_robot_passed(self, response, msg=None):
            # 检查任务是否成功（具体实现取决于服务的响应格式）
            assert response.json().get('status') == 'passed', msg or "Robot task did not pass"

        def __is_robot_failed(self, response, msg=None):
            # 检查任务是否失败（具体实现取决于服务的响应格式）
            assert response.json().get('status') == 'failed', msg or "Robot task did not fail"