Azure: wait for active publishing before executing

THis commit creates a new method for `AzureService` named
`wait_active_publishing` which relies on `ensure_can_publish` to verify
whether changes are being made or not and wait until it's possible to
proceed or timeout if necessary.

The timeout and retry interval are now possible to be set during the
class construction as optional parameters.

With this the `publish` method will double check before doing any
changes:

1. Before loading the product information it will wait for active
   changes so it can retrieve the "latest" content

2. During the publishing phase, if it's no longer the latest change it
   will raise from `ensure_can_publish`

Refers to SPSTRAT-611 and SPSTRAT-549

Signed-off-by: Jonathan Gangi <[email protected]>

Author: JAVGan

cloudpub/ms_azure/service.py

@@ -6,13 +6,13 @@
 
 from deepdiff import DeepDiff
 from requests import HTTPError
-from tenacity import retry
-from tenacity.retry import retry_if_result
+from tenacity import RetryError, Retrying, retry
+from tenacity.retry import retry_if_exception_type, retry_if_result
 from tenacity.stop import stop_after_attempt, stop_after_delay
 from tenacity.wait import wait_chain, wait_fixed
 
 from cloudpub.common import BaseService
-from cloudpub.error import ConflictError, InvalidStateError, NotFoundError
+from cloudpub.error import ConflictError, InvalidStateError, NotFoundError, Timeout
 from cloudpub.models.ms_azure import (
     RESOURCE_MAPING,
     AzureResource,
@@ -82,18 +82,31 @@ class AzureService(BaseService[AzurePublishingMetadata]):
     CONFIGURE_SCHEMA = "https://schema.mp.microsoft.com/schema/configure/{AZURE_API_VERSION}"
     DIFF_EXCLUDES = [r"root\['resources'\]\[[0-9]+\]\['url'\]"]
 
-    def __init__(self, credentials: Dict[str, str]):
+    def __init__(
+        self,
+        credentials: Dict[str, str],
+        retry_interval: int = 300,
+        retry_timeout: int = 3600 * 24 * 7,
+    ):
         """
         Create a new AuzureService object.
 
         Args:
             credentials (dict)
                 Dictionary with Azure credentials to authenticate on Product Ingestion API.
+            retry_interval (int)
+                The wait time interval in seconds for retrying jobs.
+                Defaults to 300
+            retry_timeout (int)
+                The max time in seconds to attempt retries.
+                Defaults to 7 days.
         """
         self.session = PartnerPortalSession.make_graph_api_session(
             auth_keys=credentials, schema_version=self.AZURE_SCHEMA_VERSION
         )
         self._products: List[ProductSummary] = []
+        self.retry_interval = retry_interval
+        self.retry_timeout = retry_timeout
 
     def _configure(self, data: Dict[str, Any]) -> ConfigureStatus:
         """
@@ -490,6 +503,26 @@ def ensure_can_publish(self, product_id: str) -> None:
                 log.error(msg)
                 raise ConflictError(msg)
 
+    def wait_active_publishing(self, product_id: str) -> None:
+        """
+        Wait when there's an existing submission in progress.
+
+        Args:
+            product_id (str)
+                The product ID of to verify the submissions state.
+        """
+        r = Retrying(
+            retry=retry_if_exception_type(ConflictError),
+            wait=wait_fixed(self.retry_interval),
+            stop=stop_after_delay(max_delay=self.retry_timeout),
+        )
+        log.info("Checking for active changes on %s.", product_id)
+
+        try:
+            r(self.ensure_can_publish, product_id)
+        except RetryError:
+            self._raise_error(Timeout, f"Timed out waiting for {product_id} to be unlocked")
+
     def get_plan_tech_config(self, product: Product, plan: PlanSummary) -> VMIPlanTechConfig:
         """
         Return the VMIPlanTechConfig resource for the given product/plan.
@@ -815,6 +848,7 @@ def publish(self, metadata: AzurePublishingMetadata) -> None:
         plan_name = metadata.destination.split("/")[-1]
         product_id = self.get_productid(product_name)
         disk_version = None
+        self.wait_active_publishing(product_id=product_id)
         log.info(
             "Preparing to associate the image \"%s\" with the plan \"%s\" from product \"%s\"",
             metadata.image_path,

tests/ms_azure/conftest.py

@@ -56,7 +56,7 @@ def auth_dict() -> Dict[str, str]:
 @mock.patch("cloudpub.ms_azure.service.PartnerPortalSession")
 def azure_service(auth_dict: Dict[str, str]) -> AzureService:
     """Return an instance of AzureService with mocked PartnerPortalSession."""
-    return AzureService(auth_dict)
+    return AzureService(auth_dict, retry_interval=0, retry_timeout=10)
 
 
 def job_details(status: str, result: str, errors: List[Dict[str, Any]]) -> Dict[str, Any]:

tests/ms_azure/test_service.py

@@ -12,7 +12,7 @@
 from requests.exceptions import HTTPError
 
 from cloudpub.common import BaseService
-from cloudpub.error import ConflictError, InvalidStateError, NotFoundError
+from cloudpub.error import ConflictError, InvalidStateError, NotFoundError, Timeout
 from cloudpub.models.ms_azure import (
     ConfigureStatus,
     CustomerLeads,
@@ -816,6 +816,37 @@ def test_ensure_can_publish_raises(
         with pytest.raises(RuntimeError, match=err):
             azure_service.ensure_can_publish("ffffffff-ffff-ffff-ffff-ffffffffffff")
 
+    @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
+    def test_wait_active_publishing_success(
+        self, mock_ensure_publish: mock.MagicMock, azure_service: AzureService
+    ):
+        # The test will simlulate 3 submissoins in progress to wait for
+        mock_ensure_publish.side_effect = [
+            ConflictError("Submission in progress"),
+            ConflictError("Submission in progress"),
+            ConflictError("Submission in progress"),
+            None,
+        ]
+
+        # Test
+        azure_service.wait_active_publishing("fake-product")
+        mock_ensure_publish.assert_has_calls([mock.call("fake-product") for _ in range(4)])
+
+    @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
+    def test_wait_active_publishing_timeout(
+        self, mock_ensure_publish: mock.MagicMock, azure_service: AzureService
+    ) -> None:
+        mock_ensure_publish.side_effect = [
+            ConflictError("Submission in progress") for _ in range(15)
+        ]
+        err = "Timed out waiting for fake-product to be unlocked"
+        azure_service.retry_interval = 0.1
+        azure_service.retry_timeout = 0.5
+
+        # Test
+        with pytest.raises(Timeout, match=err):
+            azure_service.wait_active_publishing("fake-product")
+
     @mock.patch("cloudpub.ms_azure.AzureService.get_submission_state")
     @mock.patch("cloudpub.ms_azure.AzureService.submit_to_status")
     @mock.patch("cloudpub.ms_azure.AzureService._is_submission_in_preview")
@@ -947,6 +978,7 @@ def test_publish_live_fail_on_retry(
         with pytest.raises(RuntimeError, match=expected_err):
             azure_service._publish_live(product_obj, "test-product")
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
@@ -955,6 +987,7 @@ def test_publish_live_fail_conflict(
         mock_configure: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         token: Dict[str, Any],
         auth_dict: Dict[str, Any],
         configure_success_response: Dict[str, Any],
@@ -1014,7 +1047,9 @@ def test_publish_live_fail_conflict(
 
                 with pytest.raises(ConflictError, match=err):
                     azure_svc.publish(metadata=metadata_azure_obj)
+                mock_wait_publish.assert_called_once()
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
@@ -1037,6 +1072,7 @@ def test_publish_overwrite(
         mock_configure: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
         metadata_azure_obj: AzurePublishingMetadata,
@@ -1063,6 +1099,7 @@ def test_publish_overwrite(
 
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_called_once_with("example-product", "plan-1", 'draft')
         mock_filter.assert_called_once_with(
             product=product_obj, resource="virtual-machine-plan-technical-configuration"
@@ -1079,6 +1116,7 @@ def test_publish_overwrite(
         mock_configure.assert_called_once_with(resources=[technical_config_obj])
         mock_submit.assert_not_called()
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
@@ -1101,6 +1139,7 @@ def test_publish_nodiskversion(
         mock_configure: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
         metadata_azure_obj: AzurePublishingMetadata,
@@ -1135,6 +1174,7 @@ def test_publish_nodiskversion(
 
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_has_calls(
             [mock.call("example-product", "plan-1", tgt) for tgt in targets]
         )
@@ -1164,6 +1204,7 @@ def test_publish_nodiskversion(
         mock_submit.assert_not_called()
 
     @pytest.mark.parametrize("keepdraft", [True, False], ids=["nochannel", "push"])
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
@@ -1188,6 +1229,7 @@ def test_publish_saspresent(
         mock_configure: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         keepdraft: bool,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
@@ -1211,6 +1253,7 @@ def test_publish_saspresent(
 
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_called_once_with("example-product", "plan-1", "preview")
         mock_filter.assert_has_calls(
             [
@@ -1230,6 +1273,7 @@ def test_publish_saspresent(
         mock_configure.assert_not_called()
         mock_submit.assert_not_called()
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
@@ -1250,6 +1294,7 @@ def test_publish_novmimages(
         mock_configure: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
         metadata_azure_obj: AzurePublishingMetadata,
@@ -1291,6 +1336,7 @@ def test_publish_novmimages(
 
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_has_calls(
             [mock.call("example-product", "plan-1", tgt) for tgt in targets]
         )
@@ -1317,6 +1363,7 @@ def test_publish_novmimages(
         mock_configure.assert_called_once_with(resources=[expected_tech_config])
         mock_submit.assert_not_called()
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.configure")
@@ -1337,6 +1384,7 @@ def test_publish_disk_has_images(
         mock_configure: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
         metadata_azure_obj: AzurePublishingMetadata,
@@ -1377,6 +1425,7 @@ def test_publish_disk_has_images(
 
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_has_calls(
             [mock.call("example-product", "plan-1", tgt) for tgt in targets]
         )
@@ -1447,6 +1496,7 @@ def test_is_submission_in_preview(
             assert res is True
             mock_substt.assert_called_once_with(current.product_id, "live")
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
@@ -1473,6 +1523,7 @@ def test_publish_live_x64_only(
         mock_ensure_publish: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
         metadata_azure_obj: AzurePublishingMetadata,
@@ -1521,6 +1572,7 @@ def test_publish_live_x64_only(
         # Test
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_has_calls(
             [mock.call("example-product", "plan-1", tgt) for tgt in targets]
         )
@@ -1555,6 +1607,7 @@ def test_publish_live_x64_only(
         mock_submit.assert_has_calls(submit_calls)
         mock_ensure_publish.assert_called_once_with(product_obj.id)
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
     @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
@@ -1581,6 +1634,7 @@ def test_publish_live_arm64_only(
         mock_ensure_publish: mock.MagicMock,
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         product_obj: Product,
         plan_summary_obj: PlanSummary,
         metadata_azure_obj: AzurePublishingMetadata,
@@ -1630,6 +1684,7 @@ def test_publish_live_arm64_only(
         # Test
         azure_service.publish(metadata_azure_obj)
 
+        mock_wait_publish.assert_called_once()
         mock_getprpl_name.assert_has_calls(
             [mock.call("example-product", "plan-1", tgt) for tgt in targets]
         )
@@ -1664,6 +1719,7 @@ def test_publish_live_arm64_only(
         mock_submit.assert_has_calls(submit_calls)
         mock_ensure_publish.assert_called_once_with(product_obj.id)
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
@@ -1672,6 +1728,7 @@ def test_publish_live_when_state_is_preview(
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
         mock_ensure_publish: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         token: Dict[str, Any],
         auth_dict: Dict[str, Any],
         configure_running_response: Dict[str, Any],
@@ -1791,8 +1848,10 @@ def test_publish_live_when_state_is_preview(
             'Updating the technical configuration for "example-product/plan-1" on "preview".'
             not in caplog.text
         )
+        mock_wait_publish.assert_called_once()
         mock_ensure_publish.assert_called_once()
 
+    @mock.patch("cloudpub.ms_azure.AzureService.wait_active_publishing")
     @mock.patch("cloudpub.ms_azure.AzureService.ensure_can_publish")
     @mock.patch("cloudpub.ms_azure.AzureService.compute_targets")
     @mock.patch("cloudpub.ms_azure.AzureService.get_productid")
@@ -1803,6 +1862,7 @@ def test_publish_live_modular_push(
         mock_get_productid: mock.MagicMock,
         mock_compute_targets: mock.MagicMock,
         mock_ensure_publish: mock.MagicMock,
+        mock_wait_publish: mock.MagicMock,
         token: Dict[str, Any],
         auth_dict: Dict[str, Any],
         configure_success_response: Dict[str, Any],
@@ -1896,6 +1956,7 @@ def test_publish_live_modular_push(
             'Performing a modular push to "preview" for "ffffffff-ffff-ffff-ffff-ffffffffffff"'
             in caplog.text
         )
+        mock_wait_publish.assert_called_once()
         mock_ensure_publish.assert_called_once()
 
         # Configure request