SDL_test_harness.c (24430B)
1/* 2 Simple DirectMedia Layer 3 Copyright (C) 1997-2014 Sam Lantinga <slouken@libsdl.org> 4 5 This software is provided 'as-is', without any express or implied 6 warranty. In no event will the authors be held liable for any damages 7 arising from the use of this software. 8 9 Permission is granted to anyone to use this software for any purpose, 10 including commercial applications, and to alter it and redistribute it 11 freely, subject to the following restrictions: 12 13 1. The origin of this software must not be misrepresented; you must not 14 claim that you wrote the original software. If you use this software 15 in a product, an acknowledgment in the product documentation would be 16 appreciated but is not required. 17 2. Altered source versions must be plainly marked as such, and must not be 18 misrepresented as being the original software. 19 3. This notice may not be removed or altered from any source distribution. 20*/ 21 22#include "SDL_config.h" 23 24#include "SDL_test.h" 25 26#include <stdio.h> 27#include <stdlib.h> 28#include <string.h> 29#include <time.h> 30 31/* Invalid test name/description message format */ 32const char *SDLTest_InvalidNameFormat = "(Invalid)"; 33 34/* Log summary message format */ 35const char *SDLTest_LogSummaryFormat = "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"; 36 37/* Final result message format */ 38const char *SDLTest_FinalResultFormat = ">>> %s '%s': %s\n"; 39 40/* ! \brief Timeout for single test case execution */ 41static Uint32 SDLTest_TestCaseTimeout = 3600; 42 43/** 44* Generates a random run seed string for the harness. The generated seed 45* will contain alphanumeric characters (0-9A-Z). 46* 47* Note: The returned string needs to be deallocated by the caller. 48* 49* \param length The length of the seed string to generate 50* 51* \returns The generated seed string 52*/ 53char * 54SDLTest_GenerateRunSeed(const int length) 55{ 56 char *seed = NULL; 57 SDLTest_RandomContext randomContext; 58 int counter; 59 60 /* Sanity check input */ 61 if (length <= 0) { 62 SDLTest_LogError("The length of the harness seed must be >0."); 63 return NULL; 64 } 65 66 /* Allocate output buffer */ 67 seed = (char *)SDL_malloc((length + 1) * sizeof(char)); 68 if (seed == NULL) { 69 SDLTest_LogError("SDL_malloc for run seed output buffer failed."); 70 SDL_Error(SDL_ENOMEM); 71 return NULL; 72 } 73 74 /* Generate a random string of alphanumeric characters */ 75 SDLTest_RandomInitTime(&randomContext); 76 for (counter = 0; counter < length; counter++) { 77 unsigned int number = SDLTest_Random(&randomContext); 78 char ch = (char) (number % (91 - 48)) + 48; 79 if (ch >= 58 && ch <= 64) { 80 ch = 65; 81 } 82 seed[counter] = ch; 83 } 84 seed[length] = '\0'; 85 86 return seed; 87} 88 89/** 90* Generates an execution key for the fuzzer. 91* 92* \param runSeed The run seed to use 93* \param suiteName The name of the test suite 94* \param testName The name of the test 95* \param iteration The iteration count 96* 97* \returns The generated execution key to initialize the fuzzer with. 98* 99*/ 100Uint64 101SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration) 102{ 103 SDLTest_Md5Context md5Context; 104 Uint64 *keys; 105 char iterationString[16]; 106 Uint32 runSeedLength; 107 Uint32 suiteNameLength; 108 Uint32 testNameLength; 109 Uint32 iterationStringLength; 110 Uint32 entireStringLength; 111 char *buffer; 112 113 if (runSeed == NULL || runSeed[0] == '\0') { 114 SDLTest_LogError("Invalid runSeed string."); 115 return -1; 116 } 117 118 if (suiteName == NULL || suiteName[0] == '\0') { 119 SDLTest_LogError("Invalid suiteName string."); 120 return -1; 121 } 122 123 if (testName == NULL || testName[0] == '\0') { 124 SDLTest_LogError("Invalid testName string."); 125 return -1; 126 } 127 128 if (iteration <= 0) { 129 SDLTest_LogError("Invalid iteration count."); 130 return -1; 131 } 132 133 /* Convert iteration number into a string */ 134 SDL_memset(iterationString, 0, sizeof(iterationString)); 135 SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration); 136 137 /* Combine the parameters into single string */ 138 runSeedLength = SDL_strlen(runSeed); 139 suiteNameLength = SDL_strlen(suiteName); 140 testNameLength = SDL_strlen(testName); 141 iterationStringLength = SDL_strlen(iterationString); 142 entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1; 143 buffer = (char *)SDL_malloc(entireStringLength); 144 if (buffer == NULL) { 145 SDLTest_LogError("Failed to allocate buffer for execKey generation."); 146 SDL_Error(SDL_ENOMEM); 147 return 0; 148 } 149 SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration); 150 151 /* Hash string and use half of the digest as 64bit exec key */ 152 SDLTest_Md5Init(&md5Context); 153 SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength); 154 SDLTest_Md5Final(&md5Context); 155 SDL_free(buffer); 156 keys = (Uint64 *)md5Context.digest; 157 158 return keys[0]; 159} 160 161/** 162* \brief Set timeout handler for test. 163* 164* Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before. 165* 166* \param timeout Timeout interval in seconds. 167* \param callback Function that will be called after timeout has elapsed. 168* 169* \return Timer id or -1 on failure. 170*/ 171SDL_TimerID 172SDLTest_SetTestTimeout(int timeout, void (*callback)()) 173{ 174 Uint32 timeoutInMilliseconds; 175 SDL_TimerID timerID; 176 177 if (callback == NULL) { 178 SDLTest_LogError("Timeout callback can't be NULL"); 179 return -1; 180 } 181 182 if (timeout < 0) { 183 SDLTest_LogError("Timeout value must be bigger than zero."); 184 return -1; 185 } 186 187 /* Init SDL timer if not initialized before */ 188 if (SDL_WasInit(SDL_INIT_TIMER) == 0) { 189 if (SDL_InitSubSystem(SDL_INIT_TIMER)) { 190 SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError()); 191 return -1; 192 } 193 } 194 195 /* Set timer */ 196 timeoutInMilliseconds = timeout * 1000; 197 timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0); 198 if (timerID == 0) { 199 SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError()); 200 return -1; 201 } 202 203 return timerID; 204} 205 206/** 207* \brief Timeout handler. Aborts test run and exits harness process. 208*/ 209void 210 SDLTest_BailOut() 211{ 212 SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run."); 213 exit(TEST_ABORTED); /* bail out from the test */ 214} 215 216/** 217* \brief Execute a test using the given execution key. 218* 219* \param testSuite Suite containing the test case. 220* \param testCase Case to execute. 221* \param execKey Execution key for the fuzzer. 222* \param forceTestRun Force test to run even if test was disabled in suite. 223* 224* \returns Test case result. 225*/ 226int 227SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun) 228{ 229 SDL_TimerID timer = 0; 230 int testCaseResult = 0; 231 int testResult = 0; 232 int fuzzerCount; 233 234 if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL) 235 { 236 SDLTest_LogError("Setup failure: testSuite or testCase references NULL"); 237 return TEST_RESULT_SETUP_FAILURE; 238 } 239 240 if (!testCase->enabled && forceTestRun == SDL_FALSE) 241 { 242 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Disabled)"); 243 return TEST_RESULT_SKIPPED; 244 } 245 246 /* Initialize fuzzer */ 247 SDLTest_FuzzerInit(execKey); 248 249 /* Reset assert tracker */ 250 SDLTest_ResetAssertSummary(); 251 252 /* Set timeout timer */ 253 timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut); 254 255 /* Maybe run suite initalizer function */ 256 if (testSuite->testSetUp) { 257 testSuite->testSetUp(0x0); 258 if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) { 259 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite Setup", testSuite->name, "Failed"); 260 return TEST_RESULT_SETUP_FAILURE; 261 } 262 } 263 264 /* Run test case function */ 265 testCaseResult = testCase->testCase(0x0); 266 267 /* Convert test execution result into harness result */ 268 if (testCaseResult == TEST_SKIPPED) { 269 /* Test was programatically skipped */ 270 testResult = TEST_RESULT_SKIPPED; 271 } else if (testCaseResult == TEST_STARTED) { 272 /* Test did not return a TEST_COMPLETED value; assume it failed */ 273 testResult = TEST_RESULT_FAILED; 274 } else if (testCaseResult == TEST_ABORTED) { 275 /* Test was aborted early; assume it failed */ 276 testResult = TEST_RESULT_FAILED; 277 } else { 278 /* Perform failure analysis based on asserts */ 279 testResult = SDLTest_AssertSummaryToTestResult(); 280 } 281 282 /* Maybe run suite cleanup function (ignore failed asserts) */ 283 if (testSuite->testTearDown) { 284 testSuite->testTearDown(0x0); 285 } 286 287 /* Cancel timeout timer */ 288 if (timer) { 289 SDL_RemoveTimer(timer); 290 } 291 292 /* Report on asserts and fuzzer usage */ 293 fuzzerCount = SDLTest_GetFuzzerInvocationCount(); 294 if (fuzzerCount > 0) { 295 SDLTest_Log("Fuzzer invocations: %d", fuzzerCount); 296 } 297 298 /* Final log based on test execution result */ 299 if (testCaseResult == TEST_SKIPPED) { 300 /* Test was programatically skipped */ 301 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)"); 302 } else if (testCaseResult == TEST_STARTED) { 303 /* Test did not return a TEST_COMPLETED value; assume it failed */ 304 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)"); 305 } else if (testCaseResult == TEST_ABORTED) { 306 /* Test was aborted early; assume it failed */ 307 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)"); 308 } else { 309 SDLTest_LogAssertSummary(); 310 } 311 312 return testResult; 313} 314 315/* Prints summary of all suites/tests contained in the given reference */ 316void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites) 317{ 318 int suiteCounter; 319 int testCounter; 320 SDLTest_TestSuiteReference *testSuite; 321 SDLTest_TestCaseReference *testCase; 322 323 /* Loop over all suites */ 324 suiteCounter = 0; 325 while(&testSuites[suiteCounter]) { 326 testSuite=&testSuites[suiteCounter]; 327 suiteCounter++; 328 SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 329 (testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat); 330 331 /* Loop over all test cases */ 332 testCounter = 0; 333 while(testSuite->testCases[testCounter]) 334 { 335 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter]; 336 testCounter++; 337 SDLTest_Log(" Test Case %i - %s: %s", testCounter, 338 (testCase->name) ? testCase->name : SDLTest_InvalidNameFormat, 339 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat); 340 } 341 } 342} 343 344/* Gets a timer value in seconds */ 345float GetClock() 346{ 347 float currentClock = (float)clock(); 348 return currentClock / (float)CLOCKS_PER_SEC; 349} 350 351/** 352* \brief Execute a test suite using the given run seed and execution key. 353* 354* The filter string is matched to the suite name (full comparison) to select a single suite, 355* or if no suite matches, it is matched to the test names (full comparison) to select a single test. 356* 357* \param testSuites Suites containing the test case. 358* \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one. 359* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. 360* \param filter Filter specification. NULL disables. Case sensitive. 361* \param testIterations Number of iterations to run each test case. 362* 363* \returns Test run result; 0 when all tests passed, 1 if any tests failed. 364*/ 365int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations) 366{ 367 int totalNumberOfTests = 0; 368 int failedNumberOfTests = 0; 369 int suiteCounter; 370 int testCounter; 371 int iterationCounter; 372 SDLTest_TestSuiteReference *testSuite; 373 SDLTest_TestCaseReference *testCase; 374 const char *runSeed = NULL; 375 char *currentSuiteName; 376 char *currentTestName; 377 Uint64 execKey; 378 float runStartSeconds; 379 float suiteStartSeconds; 380 float testStartSeconds; 381 float runEndSeconds; 382 float suiteEndSeconds; 383 float testEndSeconds; 384 float runtime; 385 int suiteFilter = 0; 386 char *suiteFilterName = NULL; 387 int testFilter = 0; 388 char *testFilterName = NULL; 389 SDL_bool forceTestRun = SDL_FALSE; 390 int testResult = 0; 391 int runResult = 0; 392 Uint32 totalTestFailedCount = 0; 393 Uint32 totalTestPassedCount = 0; 394 Uint32 totalTestSkippedCount = 0; 395 Uint32 testFailedCount = 0; 396 Uint32 testPassedCount = 0; 397 Uint32 testSkippedCount = 0; 398 Uint32 countSum = 0; 399 char *logFormat = (char *)SDLTest_LogSummaryFormat; 400 SDLTest_TestCaseReference **failedTests; 401 402 /* Sanitize test iterations */ 403 if (testIterations < 1) { 404 testIterations = 1; 405 } 406 407 /* Generate run see if we don't have one already */ 408 if (userRunSeed == NULL || userRunSeed[0] == '\0') { 409 runSeed = SDLTest_GenerateRunSeed(16); 410 if (runSeed == NULL) { 411 SDLTest_LogError("Generating a random seed failed"); 412 return 2; 413 } 414 } else { 415 runSeed = userRunSeed; 416 } 417 418 419 /* Reset per-run counters */ 420 totalTestFailedCount = 0; 421 totalTestPassedCount = 0; 422 totalTestSkippedCount = 0; 423 424 /* Take time - run start */ 425 runStartSeconds = GetClock(); 426 427 /* Log run with fuzzer parameters */ 428 SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed); 429 430 /* Count the total number of tests */ 431 suiteCounter = 0; 432 while (testSuites[suiteCounter]) { 433 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter]; 434 suiteCounter++; 435 testCounter = 0; 436 while (testSuite->testCases[testCounter]) 437 { 438 testCounter++; 439 totalNumberOfTests++; 440 } 441 } 442 443 /* Pre-allocate an array for tracking failed tests (potentially all test cases) */ 444 failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *)); 445 if (failedTests == NULL) { 446 SDLTest_LogError("Unable to allocate cache for failed tests"); 447 SDL_Error(SDL_ENOMEM); 448 return -1; 449 } 450 451 /* Initialize filtering */ 452 if (filter != NULL && filter[0] != '\0') { 453 /* Loop over all suites to check if we have a filter match */ 454 suiteCounter = 0; 455 while (testSuites[suiteCounter] && suiteFilter == 0) { 456 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter]; 457 suiteCounter++; 458 if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) { 459 /* Matched a suite name */ 460 suiteFilter = 1; 461 suiteFilterName = testSuite->name; 462 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName); 463 break; 464 } 465 466 /* Within each suite, loop over all test cases to check if we have a filter match */ 467 testCounter = 0; 468 while (testSuite->testCases[testCounter] && testFilter == 0) 469 { 470 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter]; 471 testCounter++; 472 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) { 473 /* Matched a test name */ 474 suiteFilter = 1; 475 suiteFilterName = testSuite->name; 476 testFilter = 1; 477 testFilterName = testCase->name; 478 SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName); 479 break; 480 } 481 } 482 } 483 484 if (suiteFilter == 0 && testFilter == 0) { 485 SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter); 486 SDLTest_Log("Exit code: 2"); 487 return 2; 488 } 489 } 490 491 /* Loop over all suites */ 492 suiteCounter = 0; 493 while(testSuites[suiteCounter]) { 494 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter]; 495 currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat); 496 suiteCounter++; 497 498 /* Filter suite if flag set and we have a name */ 499 if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL && 500 SDL_strcmp(suiteFilterName, testSuite->name) != 0) { 501 /* Skip suite */ 502 SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 503 suiteCounter, 504 currentSuiteName); 505 } else { 506 507 /* Reset per-suite counters */ 508 testFailedCount = 0; 509 testPassedCount = 0; 510 testSkippedCount = 0; 511 512 /* Take time - suite start */ 513 suiteStartSeconds = GetClock(); 514 515 /* Log suite started */ 516 SDLTest_Log("===== Test Suite %i: '%s' started\n", 517 suiteCounter, 518 currentSuiteName); 519 520 /* Loop over all test cases */ 521 testCounter = 0; 522 while(testSuite->testCases[testCounter]) 523 { 524 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter]; 525 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat); 526 testCounter++; 527 528 /* Filter tests if flag set and we have a name */ 529 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL && 530 SDL_strcmp(testFilterName, testCase->name) != 0) { 531 /* Skip test */ 532 SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 533 suiteCounter, 534 testCounter, 535 currentTestName); 536 } else { 537 /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */ 538 if (testFilter == 1 && !testCase->enabled) { 539 SDLTest_Log("Force run of disabled test since test filter was set"); 540 forceTestRun = SDL_TRUE; 541 } 542 543 /* Take time - test start */ 544 testStartSeconds = GetClock(); 545 546 /* Log test started */ 547 SDLTest_Log("----- Test Case %i.%i: '%s' started", 548 suiteCounter, 549 testCounter, 550 currentTestName); 551 if (testCase->description != NULL && testCase->description[0] != '\0') { 552 SDLTest_Log("Test Description: '%s'", 553 (testCase->description) ? testCase->description : SDLTest_InvalidNameFormat); 554 } 555 556 /* Loop over all iterations */ 557 iterationCounter = 0; 558 while(iterationCounter < testIterations) 559 { 560 iterationCounter++; 561 562 if (userExecKey != 0) { 563 execKey = userExecKey; 564 } else { 565 execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter); 566 } 567 568 SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey); 569 testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun); 570 571 if (testResult == TEST_RESULT_PASSED) { 572 testPassedCount++; 573 totalTestPassedCount++; 574 } else if (testResult == TEST_RESULT_SKIPPED) { 575 testSkippedCount++; 576 totalTestSkippedCount++; 577 } else { 578 testFailedCount++; 579 totalTestFailedCount++; 580 } 581 } 582 583 /* Take time - test end */ 584 testEndSeconds = GetClock(); 585 runtime = testEndSeconds - testStartSeconds; 586 if (runtime < 0.0f) runtime = 0.0f; 587 588 if (testIterations > 1) { 589 /* Log test runtime */ 590 SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime); 591 SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations); 592 } else { 593 /* Log test runtime */ 594 SDLTest_Log("Total Test runtime: %.1f sec", runtime); 595 } 596 597 /* Log final test result */ 598 switch (testResult) { 599 case TEST_RESULT_PASSED: 600 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed"); 601 break; 602 case TEST_RESULT_FAILED: 603 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Failed"); 604 break; 605 case TEST_RESULT_NO_ASSERT: 606 SDLTest_LogError((char *)SDLTest_FinalResultFormat,"Test", currentTestName, "No Asserts"); 607 break; 608 } 609 610 /* Collect failed test case references for repro-step display */ 611 if (testResult == TEST_RESULT_FAILED) { 612 failedTests[failedNumberOfTests] = testCase; 613 failedNumberOfTests++; 614 } 615 } 616 } 617 618 /* Take time - suite end */ 619 suiteEndSeconds = GetClock(); 620 runtime = suiteEndSeconds - suiteStartSeconds; 621 if (runtime < 0.0f) runtime = 0.0f; 622 623 /* Log suite runtime */ 624 SDLTest_Log("Total Suite runtime: %.1f sec", runtime); 625 626 /* Log summary and final Suite result */ 627 countSum = testPassedCount + testFailedCount + testSkippedCount; 628 if (testFailedCount == 0) 629 { 630 SDLTest_Log(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount); 631 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Passed"); 632 } 633 else 634 { 635 SDLTest_LogError(logFormat, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount); 636 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Suite", currentSuiteName, "Failed"); 637 } 638 639 } 640 } 641 642 /* Take time - run end */ 643 runEndSeconds = GetClock(); 644 runtime = runEndSeconds - runStartSeconds; 645 if (runtime < 0.0f) runtime = 0.0f; 646 647 /* Log total runtime */ 648 SDLTest_Log("Total Run runtime: %.1f sec", runtime); 649 650 /* Log summary and final run result */ 651 countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount; 652 if (totalTestFailedCount == 0) 653 { 654 runResult = 0; 655 SDLTest_Log(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount); 656 SDLTest_Log((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Passed"); 657 } 658 else 659 { 660 runResult = 1; 661 SDLTest_LogError(logFormat, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount); 662 SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Run /w seed", runSeed, "Failed"); 663 } 664 665 /* Print repro steps for failed tests */ 666 if (failedNumberOfTests > 0) { 667 SDLTest_Log("Harness input to repro failures:"); 668 for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) { 669 SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name); 670 } 671 } 672 SDL_free(failedTests); 673 674 SDLTest_Log("Exit code: %d", runResult); 675 return runResult; 676}