Report signals in job exit status

runner/internal/runner/executor/executor.go

@@ -315,9 +315,8 @@ func (ex *RunExecutor) Run(ctx context.Context) (err error) {
 
 		// todo fail reason?
 		log.Error(ctx, "Exec failed", "err", err)
-		var exitError *exec.ExitError
-		if errors.As(err, &exitError) {
-			ex.SetJobStateWithExitStatus(ctx, schemas.JobStateFailed, exitError.ExitCode())
+		if exitStatus, ok := jobExitStatus(err); ok {
+			ex.SetJobStateWithExitStatus(ctx, schemas.JobStateFailed, exitStatus)
 		} else {
 			ex.SetJobState(ctx, schemas.JobStateFailed)
 		}
@@ -717,6 +716,30 @@ func watchLogQuota(stopJob context.CancelFunc, quotaExceeded <-chan struct{}) (s
 	return func() { close(done) }
 }
 
+// jobExitStatus extracts the job's exit status from the error execJob returned: the status
+// itself if the job process exited, or the negated signal number if a signal killed it, as in
+// Python's subprocess. The stops the runner initiates never get here, Run handles them first.
+//
+// The job process is the shell running the job's commands, so the signal reaches the runner only
+// if the shell exec'd the command that got it. If the shell forked, it exits with 128+N instead.
+func jobExitStatus(err error) (int, bool) {
+	var exitErr *exec.ExitError
+	if !errors.As(err, &exitErr) {
+		return 0, false
+	}
+	ws, ok := exitErr.Sys().(syscall.WaitStatus)
+	if !ok {
+		return 0, false
+	}
+	switch {
+	case ws.Exited():
+		return ws.ExitStatus(), true
+	case ws.Signaled():
+		return -int(ws.Signal()), true
+	}
+	return 0, false
+}
+
 // setupGitCredentials must be called from Run after setJobUser
 func (ex *RunExecutor) setupGitCredentials(ctx context.Context) (func(), error) {
 	if ex.repoCredentials == nil {

runner/internal/runner/executor/executor_test.go

@@ -71,18 +71,33 @@ func TestExecutor_HomeDir(t *testing.T) {
 	assert.Equal(t, ex.currentUser.HomeDir+"\n", strings.ReplaceAll(b.String(), "\r\n", "\n"))
 }
 
-func TestExecutor_NonZeroExit(t *testing.T) {
-	ex := makeTestExecutor(t)
-	ex.jobSpec.Commands = append(ex.jobSpec.Commands, "exit 100")
-	makeCodeTar(t, ex)
-
-	setUpTestExecutor(t, ex)
-	err := ex.Run(t.Context())
-	assert.Error(t, err)
-	assert.NotEmpty(t, ex.jobStateHistory)
-	exitStatus := ex.jobStateHistory[len(ex.jobStateHistory)-1].ExitStatus
-	assert.NotNil(t, exitStatus)
-	assert.Equal(t, 100, *exitStatus)
+func TestExecutor_ExitStatus(t *testing.T) {
+	testCases := []struct {
+		name     string
+		command  string
+		expected int
+	}{
+		{"exited", "exit 100", 100},
+		// $$ is the shell itself, the runner's direct child, so no shell is left to turn the
+		// signal into 128+N
+		{"killed_by_signal", "kill -KILL $$", -9},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ex := makeTestExecutor(t)
+			ex.jobSpec.Commands = append(ex.jobSpec.Commands, tc.command)
+			makeCodeTar(t, ex)
+
+			setUpTestExecutor(t, ex)
+			err := ex.Run(t.Context())
+			assert.Error(t, err)
+			require.NotEmpty(t, ex.jobStateHistory)
+			lastState := ex.jobStateHistory[len(ex.jobStateHistory)-1]
+			assert.Equal(t, schemas.JobStateFailed, lastState.State)
+			require.NotNil(t, lastState.ExitStatus)
+			assert.Equal(t, tc.expected, *lastState.ExitStatus)
+		})
+	}
 }
 
 func TestExecutor_SSHCredentials(t *testing.T) {

src/dstack/_internal/core/models/runs.py

@@ -448,6 +448,11 @@ class JobSubmission(CoreModel):
     """
     termination_reason_message: Optional[str] = None
     exit_status: Optional[int] = None
+    """`exit_status` is `-N` if the job's process was killed by signal `N`, e.g. `-9` for
+    `SIGKILL`, as in Python's `subprocess`. The process is the shell running the job's commands,
+    so this happens only if the shell has replaced itself with the command that got the signal.
+    Otherwise, the shell exits with `128 + N`. Older runners report `-1` for any signal.
+    """
     job_provisioning_data: Optional[JobProvisioningData] = None
     job_runtime_data: Optional[JobRuntimeData] = None
     error: Optional[str] = None

src/dstack/_internal/server/schemas/runner.py

@@ -25,6 +25,9 @@ class JobStateEvent(CoreModel):
     termination_reason: Optional[str] = None
     termination_message: Optional[str] = None
     exit_status: Optional[int] = None
+    """`exit_status` is `-N` if the job's process was killed by signal `N`.
+    See `JobSubmission.exit_status`.
+    """
 
 
 class LogEvent(CoreModel):

src/dstack/_internal/server/services/jobs/__init__.py

@@ -618,8 +618,12 @@ def _get_job_status_message(job_model: JobModel) -> str:
     if job_model.status == JobStatus.DONE:
         return "exited (0)"
     elif job_model.status == JobStatus.FAILED:
-        if job_model.termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR:
-            return f"exited ({job_model.exit_status})"
+        if (
+            job_model.termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR
+            # Not set if the job failed outside of its process, e.g. the repo setup failed
+            and job_model.exit_status is not None
+        ):
+            return f"exited ({_get_shell_exit_status(job_model.exit_status)})"
         elif (
             job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
         ):
@@ -641,6 +645,17 @@ def _get_job_status_message(job_model: JobModel) -> str:
     return job_model.status.value
 
 
+def _get_shell_exit_status(exit_status: int) -> int:
+    """
+    Converts a signal `N`, stored as `-N`, to `128 + N`, the way shells and Docker report it.
+    A forking shell already reports the signal as `128 + N`, so this makes a kill look the same
+    whichever shell the job runs in. `-1` is kept as is: older runners report it for any signal.
+    """
+    if exit_status >= 0 or exit_status == -1:
+        return exit_status
+    return 128 - exit_status
+
+
 def _get_job_error(job_model: JobModel) -> Optional[str]:
     if job_model.termination_reason is None:
         return None

src/tests/_internal/cli/utils/test_run.py

@@ -250,6 +250,30 @@ async def test_simple_run(self, session: AsyncSession):
                 "exited (42)",
                 "indian_red1",
             ),
+            (
+                JobStatus.FAILED,
+                JobTerminationReason.CONTAINER_EXITED_WITH_ERROR,
+                -9,
+                None,
+                "exited (137)",
+                "indian_red1",
+            ),
+            (
+                JobStatus.FAILED,
+                JobTerminationReason.CONTAINER_EXITED_WITH_ERROR,
+                -1,
+                None,
+                "exited (-1)",
+                "indian_red1",
+            ),
+            (
+                JobStatus.FAILED,
+                JobTerminationReason.CONTAINER_EXITED_WITH_ERROR,
+                None,
+                None,
+                "error",
+                "indian_red1",
+            ),
             (
                 JobStatus.FAILED,
                 JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,