@@ -908,10 +908,34 @@
   (let ((dparts  (string-split dir "/"))
 	(count   (if (null? params) 1 (car params))))
     (conc "/" (string-intersperse 
 	       (take dparts (- (length dparts) count))
 	       "/"))))
+
+(define (runs:recursive-delete-with-error-msg real-dir)
+  (if (> (system (conc "rm -rf " real-dir)) 0)
+      (debug:print 0 "ERROR: There was a problem removing " real-dir " with rm -f")))
+
+(define (runs:safe-delete-test-dir real-dir)
+  ;; first delete all sub-directories
+  (directory-fold 
+   (lambda (f x)
+     (let ((fullname (conc real-dir "/" f)))
+       (if (directory? fullname)(runs:recursive-delete-with-error-msg fullname)))
+     (+ 1 x))
+   0 real-dir)
+  ;; then files other than *testdat.db*
+  (directory-fold 
+   (lambda (f x)
+     (let ((fullname (conc real-dir "/" f)))
+       (if (not (string-search (regexp "testdat.db") f))
+	   (runs:recursive-delete-with-error-msg fullname)))
+     (+ 1 x))
+   0 real-dir)
+  ;; then the entire directory
+  (runs:recursive-delete-with-error-msg real-dir))
+
 ;; Remove runs
 ;; fields are passing in through 
 ;; action:
 ;;    'remove-runs
 ;;    'set-state-status
@@ -989,11 +1013,13 @@
 			     ((remove-runs)
 			      (debug:print-info 0 "test: " test-name " itest-state: " test-state)
 			      (if (member test-state (list "RUNNING" "LAUNCHED" "REMOTEHOSTSTART" "KILLREQ"))
 				  (begin
 				    (if (not (hash-table-ref/default test-retry-time test-fulln #f))
-					(hash-table-set! test-retry-time test-fulln (current-seconds)))
+					(begin
+					  ;; want to set to REMOVING BUT CANNOT do it here?
+					  (hash-table-set! test-retry-time test-fulln (current-seconds))))
 				    (if (> (- (current-seconds)(hash-table-ref test-retry-time test-fulln)) allow-run-time)
 				      ;; This test is not in a correct state for cleaning up. Let's try some graceful shutdown steps first
 				      ;; Set the test to "KILLREQ" and wait five seconds then try again. Repeat up to five times then give
 				      ;; up and blow it away.
 				      (begin
@@ -1006,20 +1032,19 @@
 				    ;; NOTE: This is suboptimal as the testdata will be used later and the state/status may have changed ...
 				    (if (null? tal)
 					(loop new-test-dat tal)
 					(loop (car tal)(append tal (list new-test-dat)))))
 				  (begin
-				    (cdb:remote-run db:delete-test-records db #f (db:test-get-id test))
+				    (cdb:remote-run db:test-set-state-status-by-id db (db:test-get-id test) "REMOVING" "LOCKED" #f)
 				    (debug:print-info 1 "Attempting to remove " (if real-dir (conc " dir " real-dir " and ") "") " link " run-dir)
 				    (if (and real-dir 
 					     (> (string-length real-dir) 5)
 					     (file-exists? real-dir)) ;; bad heuristic but should prevent /tmp /home etc.
 					(begin ;; let* ((realpath (resolve-pathname run-dir)))
 					  (debug:print-info 1 "Recursively removing " real-dir)
 					  (if (file-exists? real-dir)
-					      (if (> (system (conc "rm -rf " real-dir)) 0)
-						  (debug:print 0 "ERROR: There was a problem removing " real-dir " with rm -f"))
+					      (runs:safe-delete-test-dir real-dir)
 					      (debug:print 0 "WARNING: test dir " real-dir " appears to not exist or is not readable")))
 					(if real-dir 
 					    (debug:print 0 "WARNING: directory " real-dir " does not exist")
 					    (debug:print 0 "WARNING: no real directory corrosponding to link " run-dir ", nothing done")))
 				    (if (symbolic-link? run-dir)
@@ -1038,10 +1063,12 @@
 						 (delete-directory run-dir)))
 					    (if run-dir
 						(debug:print 0 "WARNING: not removing " run-dir " as it either doesn't exist or is not a symlink")
 						(debug:print 0 "NOTE: the run dir for this test is undefined. Test may have already been deleted."))
 					    ))
+				    ;; Only delete the records *after* removing the directory. If things fail we have a record 
+				    (cdb:remote-run db:delete-test-records db #f (db:test-get-id test))
 				    (if (not (null? tal))
 					(loop (car tal)(cdr tal))))))
 			     ((set-state-status)
 			      (debug:print-info 2 "new state " (car state-status) ", new status " (cadr state-status))
 			      (cdb:remote-run db:test-set-state-status-by-id db (db:test-get-id test) (car state-status)(cadr state-status) #f)