fix flaky test test_update_status

Author: emilyalbini

nexus/tests/integration_tests/updates.rs

@@ -39,6 +39,8 @@ use tufaceous_lib::assemble::{ArtifactManifest, OmicronRepoAssembler};
 use tufaceous_lib::assemble::{DeserializedManifest, ManifestTweak};
 
 use crate::integration_tests::target_release::set_target_release_for_mupdate_recovery;
+use omicron_test_utils::dev::poll::CondCheckError;
+use omicron_test_utils::dev::poll::wait_for_condition;
 
 const TRUST_ROOTS_URL: &str = "/v1/system/update/trust-roots";
 
@@ -721,6 +723,32 @@ async fn test_update_status() -> Result<()> {
     let client = &cptestctx.external_client;
     let logctx = &cptestctx.logctx;
 
+    // During high contention the inventory might not be ready yet, which will
+    // cause the call to /v1/system/update/status to 500. We thus query the
+    // database to make sure we have an inventory before proceeding.
+    //
+    // Flaky test issue: https://github.com/oxidecomputer/omicron/issues/9316
+    {
+        let datastore = cptestctx.server.server_context().nexus.datastore();
+        let opctx = OpContext::for_tests(
+            cptestctx.logctx.log.new(o!()),
+            datastore.clone(),
+        );
+        wait_for_condition(
+            || async {
+                datastore
+                    .inventory_get_latest_collection(&opctx)
+                    .await
+                    .expect("failed to get inventory collection")
+                    .ok_or(CondCheckError::<()>::NotYet)
+            },
+            &Duration::from_millis(100),
+            &Duration::from_secs(30),
+        )
+        .await
+        .expect("no inventory collection available");
+    }
+
     // initial status
     let status: update::UpdateStatus =
         object_get(client, "/v1/system/update/status").await;