fix(gateway): keep ACME issuance off the startup path and bound ACME requests

dstack/certbot/src/http_client.rs

@@ -13,6 +13,11 @@ use reqwest::Client;
 use std::error::Error as StdError;
 use std::future::Future;
 use std::pin::Pin;
+use std::time::Duration;
+
+/// reqwest's async client has no default timeout; ACME account calls run under the
+/// cluster ACME lock with no outer bound.
+const ACME_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// A HTTP client that supports both HTTP and HTTPS connections.
 /// This is needed because the default instant_acme client only supports HTTPS.
@@ -26,6 +31,7 @@ impl ReqwestHttpClient {
     pub fn new() -> Result<Self> {
         let client = Client::builder()
             .user_agent("dstack-certbot/0.1")
+            .timeout(ACME_REQUEST_TIMEOUT)
             .build()
             .context("failed to build reqwest client")?;
         Ok(Self { client })

dstack/gateway/src/distributed_certbot.rs

@@ -459,40 +459,6 @@ impl DistributedCertBot {
         Ok(config.acme_url)
     }
 
-    /// Initialize all ZT-Domain certificates
-    pub async fn init_all(&self) -> Result<()> {
-        let configs = self.kv_store.list_zt_domain_configs();
-        for config in configs {
-            if let Err(err) = self.init_domain(&config.domain).await {
-                error!("cert[{}]: failed to initialize: {err:?}", config.domain);
-            }
-        }
-        Ok(())
-    }
-
-    /// Initialize certificate for a specific domain
-    pub async fn init_domain(&self, domain: &str) -> Result<()> {
-        // First, try to load from KvStore (synced from other nodes)
-        if let Some(cert_data) = self.kv_store.get_cert_data(domain) {
-            let now = now_secs();
-            if cert_data.not_after > now {
-                info!(
-                    domain,
-                    "loaded from KvStore (issued by node {}, expires in {} days)",
-                    cert_data.issued_by,
-                    (cert_data.not_after - now) / 86400
-                );
-                self.cert_resolver.update_cert(domain, &cert_data)?;
-                return Ok(());
-            }
-            info!(domain, "KvStore certificate expired, will request new one");
-        }
-
-        // No valid cert, need to request new one
-        info!(domain, "no valid certificate found, requesting from ACME");
-        self.request_new_cert(domain).await
-    }
-
     /// Set CAA records for every configured ZT domain.
     ///
     /// Runs under the shared ACME lock, so a rotation or another
@@ -686,35 +652,6 @@ impl DistributedCertBot {
         result
     }
 
-    /// Request new certificate for a domain
-    #[tracing::instrument(skip(self))]
-    async fn request_new_cert(&self, domain: &str) -> Result<()> {
-        let config = self
-            .kv_store
-            .get_zt_domain_config(domain)
-            .context("ZT-Domain config not found")?;
-
-        // Try to acquire lock first
-        let Some(lock) = self.try_acquire_cert_lock(domain) else {
-            // Another node is requesting, wait for it
-            info!("another node is requesting, waiting...");
-            tokio::time::sleep(Duration::from_secs(30)).await;
-            if let Some(cert_data) = self.kv_store.get_cert_data(domain) {
-                self.cert_resolver.update_cert(domain, &cert_data)?;
-                return Ok(());
-            }
-            bail!("failed to get certificate from KvStore after waiting");
-        };
-
-        let result = self.do_request_new(domain, &config).await;
-
-        if let Err(err) = self.release_cert_lock(domain, &lock) {
-            error!("failed to release lock: {err:?}");
-        }
-
-        result
-    }
-
     async fn do_request_new(&self, domain: &str, config: &ZtDomainConfig) -> Result<()> {
         let acme_client = self.get_or_create_acme_client(domain, config).await?;
 

dstack/gateway/src/main_service.rs

@@ -506,10 +506,7 @@ impl ProxyInner {
                 .clone()
                 .map(|service| service as Arc<dyn crate::kv::PersistentWriteNotifier>),
         ));
-        // Initialize any configured domains
-        if let Err(err) = certbot.init_all().await {
-            warn!("Failed to initialize multi-domain certbot: {err:?}");
-        }
+        // Issuance runs in `start_certbot_task`, not here, so it cannot delay startup.
 
         // Create TLS acceptors with CertResolver for SNI-based resolution
         // CertResolver allows atomic certificate updates without recreating acceptors