cscg22-gearboy

CSCG 2022 Challenge 'Gearboy'
git clone https://git.sinitax.com/sinitax/cscg22-gearboy
Log | Files | Refs | sfeed.txt

testharness.c (15599B)

      1/* See COPYING.txt for the full license governing this code. */
      2/**
      3 *  \file testharness.c 
      4 *
      5 *  Source file for the test harness.
      6 */
      7
      8#include <stdlib.h>
      9#include <SDL_test.h>
     10#include <SDL.h>
     11#include <SDL_assert.h>
     12#include "SDL_visualtest_harness_argparser.h"
     13#include "SDL_visualtest_process.h"
     14#include "SDL_visualtest_variators.h"
     15#include "SDL_visualtest_screenshot.h"
     16#include "SDL_visualtest_mischelper.h"
     17
     18#if defined(__WIN32__) && !defined(__CYGWIN__)
     19#include <direct.h>
     20#elif defined(__WIN32__) && defined(__CYGWIN__)
     21#include <signal.h>
     22#elif defined(__LINUX__)
     23#include <sys/stat.h>
     24#include <sys/types.h>
     25#include <signal.h>
     26#else
     27#error "Unsupported platform"
     28#endif
     29
     30/** Code for the user event triggered when a new action is to be executed */
     31#define ACTION_TIMER_EVENT 0
     32/** Code for the user event triggered when the maximum timeout is reached */
     33#define KILL_TIMER_EVENT 1
     34/** FPS value used for delays in the action loop */
     35#define ACTION_LOOP_FPS 10
     36
     37/** Value returned by RunSUTAndTest() when the test has passed */
     38#define TEST_PASSED 1
     39/** Value returned by RunSUTAndTest() when the test has failed */
     40#define TEST_FAILED 0
     41/** Value returned by RunSUTAndTest() on a fatal error */
     42#define TEST_ERROR -1
     43
     44static SDL_ProcessInfo pinfo;
     45static SDL_ProcessExitStatus sut_exitstatus;
     46static SDLVisualTest_HarnessState state;
     47static SDLVisualTest_Variator variator;
     48static SDLVisualTest_ActionNode* current; /* the current action being performed */
     49static SDL_TimerID action_timer, kill_timer;
     50
     51/* returns a char* to be passed as the format argument of a printf-style function. */
     52static char*
     53usage()
     54{
     55    return "Usage: \n%s --sutapp xyz"
     56           " [--sutargs abc | --parameter-config xyz.parameters"
     57           " [--variator exhaustive|random]" 
     58           " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
     59           " [--action-config xyz.actions]"
     60           " [--output-dir /path/to/output]"
     61           " [--verify-dir /path/to/verify]"
     62           " or --config app.config";
     63}
     64
     65/* register Ctrl+C handlers */
     66#if defined(__LINUX__) || defined(__CYGWIN__)
     67static void
     68CtrlCHandlerCallback(int signum)
     69{
     70    SDL_Event event;
     71    SDLTest_Log("Ctrl+C received");
     72    event.type = SDL_QUIT;
     73    SDL_PushEvent(&event);
     74}
     75#endif
     76
     77static Uint32
     78ActionTimerCallback(Uint32 interval, void* param)
     79{
     80    SDL_Event event;
     81    SDL_UserEvent userevent;
     82    Uint32 next_action_time;
     83
     84    /* push an event to handle the action */
     85    userevent.type = SDL_USEREVENT;
     86    userevent.code = ACTION_TIMER_EVENT;
     87    userevent.data1 = &current->action;
     88    userevent.data2 = NULL;
     89
     90    event.type = SDL_USEREVENT;
     91    event.user = userevent;
     92    SDL_PushEvent(&event);
     93
     94    /* calculate the new interval and return it */
     95    if(current->next)
     96        next_action_time = current->next->action.time - current->action.time;
     97    else
     98    {
     99        next_action_time = 0;
    100        action_timer = 0;
    101    }
    102
    103    current = current->next;
    104    return next_action_time;
    105}
    106
    107static Uint32
    108KillTimerCallback(Uint32 interval, void* param)
    109{
    110    SDL_Event event;
    111    SDL_UserEvent userevent;
    112
    113    userevent.type = SDL_USEREVENT;
    114    userevent.code = KILL_TIMER_EVENT;
    115    userevent.data1 = NULL;
    116    userevent.data2 = NULL;
    117
    118    event.type = SDL_USEREVENT;
    119    event.user = userevent;
    120    SDL_PushEvent(&event);
    121
    122    kill_timer = 0;
    123    return 0;
    124}
    125
    126static int
    127ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
    128{
    129    if(!action || !sut_running)
    130        return TEST_ERROR;
    131
    132    switch(action->type)
    133    {
    134        case SDL_ACTION_KILL:
    135            SDLTest_Log("Action: Kill SUT");
    136            if(SDL_IsProcessRunning(&pinfo) == 1 &&
    137               !SDL_KillProcess(&pinfo, &sut_exitstatus))
    138            {
    139                SDLTest_LogError("SDL_KillProcess() failed");
    140                return TEST_ERROR;
    141            }
    142            *sut_running = 0;
    143        break;
    144
    145        case SDL_ACTION_QUIT:
    146            SDLTest_Log("Action: Quit SUT");
    147            if(SDL_IsProcessRunning(&pinfo) == 1 &&
    148               !SDL_QuitProcess(&pinfo, &sut_exitstatus))
    149            {
    150                SDLTest_LogError("SDL_QuitProcess() failed");
    151                return TEST_FAILED;
    152            }
    153            *sut_running = 0;
    154        break;
    155
    156        case SDL_ACTION_LAUNCH:
    157        {
    158            char* path;
    159            char* args;
    160            SDL_ProcessInfo action_process;
    161            SDL_ProcessExitStatus ps;
    162
    163            path = action->extra.process.path;
    164            args = action->extra.process.args;
    165            if(args)
    166            {
    167                SDLTest_Log("Action: Launch process: %s with arguments: %s",
    168                            path, args);
    169            }
    170            else
    171                SDLTest_Log("Action: Launch process: %s", path);
    172            if(!SDL_LaunchProcess(path, args, &action_process))
    173            {
    174                SDLTest_LogError("SDL_LaunchProcess() failed");
    175                return TEST_ERROR;
    176            }
    177
    178            /* small delay so that the process can do its job */
    179            SDL_Delay(1000);
    180
    181            if(SDL_IsProcessRunning(&action_process) > 0)
    182            {
    183                SDLTest_LogError("Process %s took too long too complete."
    184                                    " Force killing...", action->extra);
    185                if(!SDL_KillProcess(&action_process, &ps))
    186                {
    187                    SDLTest_LogError("SDL_KillProcess() failed");
    188                    return TEST_ERROR;
    189                }
    190            }
    191        }
    192        break;
    193
    194        case SDL_ACTION_SCREENSHOT:
    195        {
    196            char path[MAX_PATH_LEN], hash[33];
    197
    198            SDLTest_Log("Action: Take screenshot");
    199            /* can't take a screenshot if the SUT isn't running */
    200            if(SDL_IsProcessRunning(&pinfo) != 1)
    201            {
    202                SDLTest_LogError("SUT has quit.");
    203                *sut_running = 0;
    204                return TEST_FAILED;
    205            }
    206
    207            /* file name for the screenshot image */
    208            SDLVisualTest_HashString(args, hash);
    209            SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
    210            if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
    211            {
    212                SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
    213                return TEST_ERROR;
    214            }
    215        }
    216        break;
    217
    218        case SDL_ACTION_VERIFY:
    219        {
    220            int ret;
    221
    222            SDLTest_Log("Action: Verify screenshot");
    223            ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
    224                                                  state.verify_dir);
    225
    226            if(ret == -1)
    227            {
    228                SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
    229                return TEST_ERROR;
    230            }
    231            else if(ret == 0)
    232            {
    233                SDLTest_Log("Verification failed: Images were not equal.");
    234                return TEST_FAILED;
    235            }
    236            else if(ret == 1)
    237                SDLTest_Log("Verification successful.");
    238            else
    239            {
    240                SDLTest_Log("Verfication skipped.");
    241                return TEST_FAILED;
    242            }
    243        }
    244        break;
    245
    246        default:
    247            SDLTest_LogError("Invalid action type");
    248            return TEST_ERROR;
    249        break;
    250    }
    251
    252    return TEST_PASSED;
    253}
    254
    255static int
    256RunSUTAndTest(char* sutargs, int variation_num)
    257{
    258    int success, sut_running, return_code;
    259    char hash[33];
    260    SDL_Event event;
    261
    262    return_code = TEST_PASSED;
    263
    264    if(!sutargs)
    265    {
    266        SDLTest_LogError("sutargs argument cannot be NULL");
    267        return_code = TEST_ERROR;
    268        goto runsutandtest_cleanup_generic;
    269    }
    270
    271    SDLVisualTest_HashString(sutargs, hash);
    272    SDLTest_Log("Hash: %s", hash);
    273
    274    success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
    275    if(!success)
    276    {
    277        SDLTest_Log("Could not launch SUT.");
    278        return_code = TEST_ERROR;
    279        goto runsutandtest_cleanup_generic;
    280    }
    281    SDLTest_Log("SUT launch successful.");
    282    SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
    283    sut_running = 1;
    284
    285    /* launch the timers */
    286    SDLTest_Log("Performing actions..");
    287    current = state.action_queue.front;
    288    action_timer = 0;
    289    kill_timer = 0;
    290    if(current)
    291    {
    292        action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
    293        if(!action_timer)
    294        {
    295            SDLTest_LogError("SDL_AddTimer() failed");
    296            return_code = TEST_ERROR;
    297            goto runsutandtest_cleanup_timer;
    298        }
    299    }
    300    kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
    301    if(!kill_timer)
    302    {
    303        SDLTest_LogError("SDL_AddTimer() failed");
    304        return_code = TEST_ERROR;
    305        goto runsutandtest_cleanup_timer;
    306    }
    307
    308    /* the timer stops running if the actions queue is empty, and the
    309       SUT stops running if it crashes or if we encounter a KILL/QUIT action */
    310    while(sut_running)
    311    {
    312        /* process the actions by using an event queue */
    313        while(SDL_PollEvent(&event))
    314        {
    315            if(event.type == SDL_USEREVENT)
    316            {
    317                if(event.user.code == ACTION_TIMER_EVENT)
    318                {
    319                    SDLVisualTest_Action* action;
    320
    321                    action = (SDLVisualTest_Action*)event.user.data1;
    322
    323                    switch(ProcessAction(action, &sut_running, sutargs))
    324                    {
    325                        case TEST_PASSED:
    326                        break;
    327
    328                        case TEST_FAILED:
    329                            return_code = TEST_FAILED;
    330                            goto runsutandtest_cleanup_timer;
    331                        break;
    332
    333                        default:
    334                            SDLTest_LogError("ProcessAction() failed");
    335                            return_code = TEST_ERROR;
    336                            goto runsutandtest_cleanup_timer;
    337                    }
    338                }
    339                else if(event.user.code == KILL_TIMER_EVENT)
    340                {
    341                    SDLTest_LogError("Maximum timeout reached. Force killing..");
    342                    return_code = TEST_FAILED;
    343                    goto runsutandtest_cleanup_timer;
    344                }
    345            }
    346            else if(event.type == SDL_QUIT)
    347            {
    348                SDLTest_LogError("Received QUIT event. Testharness is quitting..");
    349                return_code = TEST_ERROR;
    350                goto runsutandtest_cleanup_timer;
    351            }
    352        }
    353        SDL_Delay(1000/ACTION_LOOP_FPS);
    354    }
    355
    356    SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
    357    if(sut_exitstatus.exit_status == 0)
    358    {
    359        return_code = TEST_PASSED;
    360        goto runsutandtest_cleanup_timer;
    361    }
    362    else
    363    {
    364        return_code = TEST_FAILED;
    365        goto runsutandtest_cleanup_timer;
    366    }
    367
    368    return_code = TEST_ERROR;
    369    goto runsutandtest_cleanup_generic;
    370
    371runsutandtest_cleanup_timer:
    372    if(action_timer && !SDL_RemoveTimer(action_timer))
    373    {
    374        SDLTest_Log("SDL_RemoveTimer() failed");
    375        return_code = TEST_ERROR;
    376    }
    377
    378    if(kill_timer && !SDL_RemoveTimer(kill_timer))
    379    {
    380        SDLTest_Log("SDL_RemoveTimer() failed");
    381        return_code = TEST_ERROR;
    382    }
    383/* runsutandtest_cleanup_process: */
    384    if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
    385    {
    386        SDLTest_Log("SDL_KillProcess() failed");
    387        return_code = TEST_ERROR;
    388    }
    389runsutandtest_cleanup_generic:
    390    return return_code;
    391}
    392
    393/** Entry point for testharness */
    394int
    395main(int argc, char* argv[])
    396{
    397    int i, passed, return_code, failed;
    398
    399    /* freeing resources, linux style! */
    400    return_code = 0;
    401
    402    if(argc < 2)
    403    {
    404        SDLTest_Log(usage(), argv[0]);
    405        goto cleanup_generic;
    406    }
    407
    408#if defined(__LINUX__) || defined(__CYGWIN__)
    409    signal(SIGINT, CtrlCHandlerCallback);
    410#endif
    411
    412    /* parse arguments */
    413    if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
    414    {
    415        SDLTest_Log(usage(), argv[0]);
    416        return_code = 1;
    417        goto cleanup_generic;
    418    }
    419    SDLTest_Log("Parsed harness arguments successfully.");
    420
    421    /* initialize SDL */
    422    if(SDL_Init(SDL_INIT_TIMER) == -1)
    423    {
    424        SDLTest_LogError("SDL_Init() failed.");
    425        SDLVisualTest_FreeHarnessState(&state);
    426        return_code = 1;
    427        goto cleanup_harness_state;
    428    }
    429
    430    /* create an output directory if none exists */
    431#if defined(__LINUX__) || defined(__CYGWIN__)
    432    mkdir(state.output_dir, 0777);
    433#elif defined(__WIN32__)
    434    _mkdir(state.output_dir);
    435#else
    436#error "Unsupported platform"
    437#endif
    438
    439    /* test with sutargs */
    440    if(SDL_strlen(state.sutargs))
    441    {
    442        SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
    443        if(!state.no_launch)
    444        {
    445            switch(RunSUTAndTest(state.sutargs, 0))
    446            {
    447                case TEST_PASSED:
    448                    SDLTest_Log("Status: PASSED");
    449                break;
    450
    451                case TEST_FAILED:
    452                    SDLTest_Log("Status: FAILED");
    453                break;
    454
    455                case TEST_ERROR:
    456                    SDLTest_LogError("Some error occurred while testing.");
    457                    return_code = 1;
    458                    goto cleanup_sdl;
    459                break;
    460            }
    461        }
    462    }
    463
    464    if(state.sut_config.num_options > 0)
    465    {
    466        char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
    467                              "RANDOM" : "EXHAUSTIVE";
    468        if(state.num_variations > 0)
    469            SDLTest_Log("Testing SUT with variator: %s for %d variations",
    470                        variator_name, state.num_variations);
    471        else
    472            SDLTest_Log("Testing SUT with variator: %s and ALL variations",
    473                        variator_name);
    474        /* initialize the variator */
    475        if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
    476                                       state.variator_type, 0))
    477        {
    478            SDLTest_LogError("Could not initialize variator");
    479            return_code = 1;
    480            goto cleanup_sdl;
    481        }
    482
    483        /* iterate through all the variations */
    484        passed = 0;
    485        failed = 0;
    486        for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
    487        {
    488            char* args = SDLVisualTest_GetNextVariation(&variator);
    489            if(!args)
    490                break;
    491            SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
    492
    493            if(!state.no_launch)
    494            {
    495                switch(RunSUTAndTest(args, i + 1))
    496                {
    497                    case TEST_PASSED:
    498                        SDLTest_Log("Status: PASSED");
    499                        passed++;
    500                    break;
    501
    502                    case TEST_FAILED:
    503                        SDLTest_Log("Status: FAILED");
    504                        failed++;
    505                    break;
    506
    507                    case TEST_ERROR:
    508                        SDLTest_LogError("Some error occurred while testing.");
    509                        goto cleanup_variator;
    510                    break;
    511                }
    512            }
    513        }
    514        if(!state.no_launch)
    515        {
    516            /* report stats */
    517            SDLTest_Log("Testing complete.");
    518            SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
    519        }
    520        goto cleanup_variator;
    521    }
    522 
    523    goto cleanup_sdl;
    524
    525cleanup_variator:
    526    SDLVisualTest_FreeVariator(&variator);
    527cleanup_sdl:
    528    SDL_Quit();
    529cleanup_harness_state:
    530    SDLVisualTest_FreeHarnessState(&state);
    531cleanup_generic:
    532    return return_code;
    533}