Fix unstable integration tests. (apple#1060)

- TestCLIRunCommand now run so many tests concurrently that the API
server gets swamped and tests randomly time out.
- The parallelism options on `swift test` only work for XCTest, not
swift-testing.
- Work around this while retaining some parallelism (good for stress
testing) by breaking the tests into two suites.

Author: jglogan

Makefile

@@ -184,7 +184,8 @@ integration: init-block
  $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunLifecycle || exit_code=1 ; \
  $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIExecCommand || exit_code=1 ; \
  $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLICreateCommand || exit_code=1 ; \
- $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand || exit_code=1 ; \
+ $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand1 || exit_code=1 ; \
+ $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand2 || exit_code=1 ; \
  $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatsCommand || exit_code=1 ; \
  $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIImagesCommand || exit_code=1 ; \
  $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunBase || exit_code=1 ; \

Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift

@@ -21,12 +21,19 @@ import ContainerizationOS
 import Foundation
 import Testing
 
-class TestCLIRunCommand: CLITest {
- private func getTestName() -> String {
+// FIXME: We've split the tests into two suites to prevent swamping
+// the API server with so many run commands that all wind up pulling
+// images.
+//
+// When https://github.com/swiftlang/swift-testing/pull/1390 lands
+// and is available on the CI runners, we can try setting the
+// environment variable to limit concurrency and rejoin these suites.
+class TestCLIRunCommand1: CLITest {
+ func getTestName() -> String {
  Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
  }
 
- private func getLowercasedTestName() -> String {
+ func getLowercasedTestName() -> String {
  getTestName().lowercased()
  }
 
@@ -194,6 +201,16 @@ class TestCLIRunCommand: CLITest {
  return
  }
  }
+}
+
+class TestCLIRunCommand2: CLITest {
+ func getTestName() -> String {
+ Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
+ }
+
+ func getLowercasedTestName() -> String {
+ getTestName().lowercased()
+ }
 
  @Test func testRunCommandMount() throws {
  do {

Tests/CLITests/TestCLINoParallelCases.swift

@@ -22,13 +22,18 @@ import Testing
 /// Tests that need total control over environment to avoid conflicts.
 @Suite(.serialized)
 class TestCLINoParallelCases: CLITest {
- private func getTestName() -> String {
+ func getTestName() -> String {
  Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
  }
 
+ func getLowercasedTestName() -> String {
+ getTestName().lowercased()
+ }
+
  @Test func testImageSingleConcurrentDownload() throws {
         // removing this image during parallel tests breaks stuff!
         _ = try? run(arguments: ["image", "rm", alpine])
+ defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
  do {
  try doPull(imageName: alpine, args: ["--max-concurrent-downloads", "1"])
  let imagePresent = try isImagePresent(targetImage: alpine)
@@ -42,6 +47,7 @@ class TestCLINoParallelCases: CLITest {
  @Test func testImageManyConcurrentDownloads() throws {
         // removing this image during parallel tests breaks stuff!
         _ = try? run(arguments: ["image", "rm", alpine])
+ defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
  do {
  try doPull(imageName: alpine, args: ["--max-concurrent-downloads", "64"])
  let imagePresent = try isImagePresent(targetImage: alpine)
@@ -54,6 +60,7 @@ class TestCLINoParallelCases: CLITest {
 
  @Test func testImagePruneNoImages() throws {
         // Prune with no images should succeed
+        _ = try? run(arguments: ["image", "rm", "--all"])
  let (_, output, error, status) = try run(arguments: ["image", "prune"])
  if status != 0 {
  throw CLIError.executionFailed("image prune failed: \(error)")
@@ -64,6 +71,8 @@ class TestCLINoParallelCases: CLITest {
 
  @Test func testImagePruneUnusedImages() throws {
         // 1. Pull the images
+        _ = try? run(arguments: ["image", "rm", "--all"])
+ defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
  try doPull(imageName: alpine)
  try doPull(imageName: busybox)
 
@@ -93,6 +102,10 @@ class TestCLINoParallelCases: CLITest {
  let containerName = "\(name)_container"
 
         // 1. Pull the images
+        _ = try? run(arguments: ["image", "rm", "--all"])
+ defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
+        _ = try? run(arguments: ["rm", "--all", "--force"])
+ defer { _ = try? run(arguments: ["rm", "--all", "--force"]) }
  try doPull(imageName: alpine)
  try doPull(imageName: busybox)