Apm-6136 update python dependencies

.github/workflows/continuous-integration.yml

@@ -12,10 +12,10 @@ jobs:
               with:
                   fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function
 
-            - name: Install Python 3.8
+            - name: Install Python 3.10
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: "3.10"
 
             - name: Update apt repositories
               run: sudo apt update

.gitignore

@@ -17,6 +17,7 @@ __pycache__/
 *.orig
 *.squashfs
 *.retry
+.envrc
 
 # node
 node_modules/

.python-version

@@ -1 +1 @@
-3.8.2
+3.10

azure/azure-build-pipeline.yml

@@ -28,6 +28,7 @@ extends:
   parameters:
     service_name: ${{ variables.service_name }}
     short_service_name: ${{ variables.short_service_name }}
+    python_version: ${{ variables.python_version }}
     cache_steps:
       - task: s3-cache-action@1
         inputs:

azure/azure-pr-pipeline.yml

@@ -33,6 +33,7 @@ extends:
     service_name: ${{ variables.service_name }}
     service_base_path: ${{ variables.service_base_path }}
     short_service_name: ${{ variables.short_service_name }}
+    python_version: ${{ variables.python_version }}
     apigee_deployments:
       - environment: internal-dev
         post_deploy:

azure/azure-release-pipeline.yml

@@ -33,6 +33,7 @@ extends:
     service_base_path: ${{ variables.service_base_path }}
     enable_monitoring: true
     enable_status_monitoring: true
+    python_version: ${{ variables.python_version }}
     apigee_deployments:
       - environment: internal-dev
         post_deploy:

azure/project.yml

@@ -2,3 +2,4 @@ variables:
   service_name: sync-wrap
   service_base_path: sync-wrap
   short_service_name: sync-wrap
+  python_version: 3.10

azure/templates/e2e.yml

@@ -21,9 +21,10 @@ steps:
     - bash: |
         export RELEASE_RELEASEID=$(Build.BuildId)
         export SOURCE_COMMIT_ID=$(Build.SourceVersion)
+        export PROXY_NAME="$(FULLY_QUALIFIED_SERVICE_NAME)"
+        export APIGEE_ACCESS_TOKEN="$(secret.AccessToken)"
         export APIGEE_ENVIRONMENT="$(APIGEE_ENVIRONMENT)"
-        export SERVICE_BASE_PATH="$(SERVICE_BASE_PATH)"
-        export STATUS_ENDPOINT_API_KEY="$(status-endpoint-api-key)"
+        export APIGEE_ORGANIZATION="nhsd-$(APIGEE_ORGANIZATION)"
         make -C e2e ${{ parameters.test_type }}
 
       workingDirectory: $(SERVICE_DIR)

e2e/Makefile

@@ -17,16 +17,24 @@ guard-%:
 		exit 1; \
 	fi
 
+TEST_CMD := pytest \
+		--color=yes \
+		--api-name=sync-wrap \
+		--proxy-name=$(PROXY_NAME) \
+
+ifdef APIGEE_ORGANIZATION
+  TEST_CMD += --apigee-organization=$(APIGEE_ORGANIZATION)
+endif
 
 test:
-	$(activate) pytest
+	$(activate) $(TEST_CMD) -vs
 
 e2e:
 	rm -f ../reports/e2e.xml  > /dev/null || true
-	$(activate) coverage run --source ./ --module pytest -rxs -v --junit-xml=../reports/e2e.xml --ignore .venv || true
+	$(activate) coverage run --source ./ --module $(TEST_CMD) -rxs -v --junit-xml=../reports/e2e.xml --ignore .venv || true
 	@if [[ ! -f ../reports/e2e.xml ]]; then echo report not created; exit 1; fi
 
 smoketest:
 	rm -f ../reports/smoketest.xml  > /dev/null || true
-	$(activate) coverage run --source ./ --module pytest -m smoketest -rxs -v --junit-xml=../reports/smoketest.xml --ignore .venv || true
-	@if [[ ! -f ../reports/smoketest.xml ]]; then echo report not created; exit 1; fi
+	$(activate) coverage run --source ./ --module $(TEST_CMD) -m smoketest -rxs -v --junit-xml=../reports/smoketest.xml --ignore .venv || true
+	@if [[ ! -f ../reports/smoketest.xml ]]; then echo report not created; exit 1; fi

e2e/tests/api_tests.py

@@ -1,11 +1,9 @@
+from os import getenv
+from time import sleep
 from typing import List
 
 import pytest
-from aiohttp import ClientResponse
-from api_test_utils import poll_until
-from api_test_utils.api_session_client import APISessionClient
-from api_test_utils.api_test_session_config import APITestSessionConfig
-from api_test_utils import env
+import requests
 
 
 def dict_path(raw, path: List[str]):
@@ -24,114 +22,94 @@ def dict_path(raw, path: List[str]):
 
 @pytest.mark.e2e
 @pytest.mark.smoketest
-@pytest.mark.asyncio
-async def test_wait_for_ping(api_client: APISessionClient, api_test_config: APITestSessionConfig):
+def test_wait_for_ping(nhsd_apim_proxy_url):
+    retries = 0
+    resp = requests.get(f"{nhsd_apim_proxy_url}/_ping", timeout=30)
+    deployed_commit_id = resp.json().get("commitId")
 
-    async def _is_complete(resp: ClientResponse):
+    while deployed_commit_id != getenv("SOURCE_COMMIT_ID") and retries <= 30:
+        resp = requests.get(f"{nhsd_apim_proxy_url}/_ping", timeout=30)
 
-        if resp.status != 200:
-            return False
-        body = await resp.json()
-        return body.get("commitId") == api_test_config.commit_id
+        if resp.status_code != 200:
+            pytest.fail(f"Status code {resp.status_code}, expecting 200")
 
-    await poll_until(
-        make_request=lambda: api_client.get('_ping'),
-        until=_is_complete,
-        timeout=120
-    )
+        deployed_commit_id = resp.json().get("commitId")
+        retries += 1
+        sleep(1)
+
+    if retries >= 30:
+        pytest.fail("Timeout Error - max retries")
+
+    assert deployed_commit_id == getenv("SOURCE_COMMIT_ID")
 
 
 @pytest.mark.e2e
 @pytest.mark.smoketest
-@pytest.mark.asyncio
-async def test_check_status_is_secured(api_client: APISessionClient):
-
-    async with api_client.get("_status", allow_retries=True) as resp:
-        assert resp.status == 401
+def test_check_status_is_secured(nhsd_apim_proxy_url):
+    resp = requests.get(f"{nhsd_apim_proxy_url}/_status")
+    assert resp.status_code == 401
 
 
 @pytest.mark.e2e
 @pytest.mark.smoketest
-@pytest.mark.asyncio
-async def test_wait_for_status(
-        api_client: APISessionClient, api_test_config: APITestSessionConfig
-):
-    async def is_deployed(resp: ClientResponse):
-        if resp.status != 200:
-            return False
-        body = await resp.json()
-
-        if body.get("commitId") != api_test_config.commit_id:
-            return False
-
-        backend = dict_path(body, ["checks", "healthcheck", "outcome", "version"])
-
-        if type(backend) != dict:
-            return False
-
-        return backend.get("commitId") == api_test_config.commit_id
-
-    await poll_until(
-        make_request=lambda: api_client.get(
-            "_status", headers={"apikey": env.status_endpoint_api_key()}
-        ),
-        until=is_deployed,
-        timeout=120,
+@pytest.mark.parametrize("service_header", ["", "async-slowapp", "sync-wrap"])
+def test_wait_for_status(nhsd_apim_proxy_url, status_endpoint_auth_headers, service_header):
+    def _container_not_ready(resp: requests.Response):
+        """
+        Requests to ECS containers which are still starting up return with a
+        HTTP 503 (service unavailable).
+        """
+        return resp.json().get("checks", {}) \
+            .get("healthcheck", {}) \
+            .get("responseCode") == 503
+
+    retries = 0
+    headers = status_endpoint_auth_headers
+    if service_header:
+        headers["x-apim-service"] = service_header
+    resp = requests.get(
+        f"{nhsd_apim_proxy_url}/_status", headers=headers, timeout=30
     )
 
+    if resp.status_code != 200:
+        # Status should always be 200 we don't need to wait for it
+        pytest.fail(f"Status code {resp.status_code}, expecting 200")
+    if not resp.json().get("version"):
+        pytest.fail("version not found")
 
-@pytest.mark.e2e
-@pytest.mark.asyncio
-async def test_api_status_with_service_header_another_service(api_client: APISessionClient):
-
-    r = await api_client.get(
-        "_status", allow_retries=True, max_retries=5,
-        headers={
-            'x-apim-service': 'async-slowapp',
-            'apikey': env.status_endpoint_api_key()
-        }
-    )
-    assert r.status == 200, (r.status, r.reason, (await r.text())[:2000])
-    body = await r.json()
+    deployed_commit_id = resp.json().get("commitId")
 
-    service = dict_path(body, ["checks", "healthcheck", "outcome", "service"])
+    while deployed_commit_id != getenv("SOURCE_COMMIT_ID") or _container_not_ready(resp) and retries <= 45:
+        resp = requests.get(
+            f"{nhsd_apim_proxy_url}/_status", headers=status_endpoint_auth_headers, timeout=30
+        )
 
-    assert service == 'sync-wrap'
+        deployed_commit_id = resp.json().get("commitId")
+        retries += 1
+        sleep(1)
 
+    if retries >= 45:
+        pytest.fail("Timeout Error - max retries")
 
-@pytest.mark.e2e
-@pytest.mark.asyncio
-async def test_api_status_with_service_header(api_client: APISessionClient):
-
-    r = await api_client.get(
-        "_status", allow_retries=True, max_retries=5,
-        headers={
-            'x-apim-service': 'sync-wrap',
-            'apikey': env.status_endpoint_api_key()
-        }
-    )
-    assert r.status == 200, (r.status, r.reason, (await r.text())[:2000])
-    body = await r.json()
+    body = resp.json()
+    assert body.get("status") == "pass"
 
     service = dict_path(body, ["checks", "healthcheck", "outcome", "service"])
-
     assert service == 'sync-wrap'
 
 
 @pytest.mark.e2e
-@pytest.mark.asyncio
-async def test_api_slowapp_slower_than_sync_wait(api_client: APISessionClient):
-
-    r = await api_client.get(
-        "async-slowapp/slow?delay=5", allow_retries=True, max_retries=5, headers={'x-sync-wait': '0.25'}
+def test_api_slowapp_slower_than_sync_wait(nhsd_apim_proxy_url):
+    resp = requests.get(
+        f"{nhsd_apim_proxy_url}/async-slowapp/slow?delay=5", headers={'x-sync-wait': '0.25'}, timeout=45
     )
-    assert r.status == 504, (r.status, r.reason, (await r.text())[:2000])
+    assert resp.status_code == 504, (resp.status_code, resp.reason, resp.text[:2000])
 
 
 @pytest.mark.e2e
-@pytest.mark.asyncio
-async def test_api_slowapp_responds_test_final_status(api_client: APISessionClient):
-
-    r = await api_client.get("async-slowapp/slow?final_status=418&complete_in=0.5", allow_retries=True, max_retries=5)
-    assert r.status == 418, (r.status, r.reason, (await r.text())[:2000])
-    assert r.reason == "I'm a Teapot"
+def test_api_slowapp_responds_test_final_status(nhsd_apim_proxy_url):
+    resp = requests.get(
+        f"{nhsd_apim_proxy_url}/async-slowapp/slow?final_status=418&complete_in=0.5", timeout=30
+    )
+    assert resp.status_code == 418, (resp.status_code, resp.reason, resp.text[:2000])
+    assert resp.reason == "I'm a Teapot"