Commit 8ca7bc42 authored by SimonB's avatar SimonB

Adds verbose mode to the test suites

Added a verbose option to the generated test suites which can list the
dependencies not met for skipped test cases.
Also clarifies internal interfaces between the main_test.function and test code,
and fixed a bug on calculating available tests in run-test-suites.pl.
parent 098a3b50
......@@ -2,6 +2,8 @@
# generate_code.pl
#
# This file is part of mbed TLS (https://tls.mbed.org)
#
# Copyright (c) 2009-2016, ARM Limited, All Rights Reserved
#
# Purpose
......@@ -202,7 +204,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\//
if( substr($def, 0, 4) eq "int " )
{
$param_defs .= " int param$i;\n";
$param_checks .= " if( verify_int( params[$i], &param$i ) != 0 ) return( 2 );\n";
$param_checks .= " if( verify_int( params[$i], &param$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
push @dispatch_params, "param$i";
$mapping_regex .= ":([\\d\\w |\\+\\-\\(\\)]+)";
......@@ -211,7 +213,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\//
elsif( substr($def, 0, 6) eq "char *" )
{
$param_defs .= " char *param$i = params[$i];\n";
$param_checks .= " if( verify_string( &param$i ) != 0 ) return( 2 );\n";
$param_checks .= " if( verify_string( &param$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
push @dispatch_params, "param$i";
$mapping_regex .= ":[^:\n]+";
}
......@@ -248,14 +250,14 @@ $param_defs
if( cnt != $param_count )
{
mbedtls_fprintf( stderr, "\\nIncorrect argument count (%d != %d)\\n", cnt, $param_count );
return( 2 );
return( DISPATCH_INVALID_TEST_DATA );
}
$param_checks
test_suite_$function_name( $call_params );
return ( 0 );
return ( DISPATCH_TEST_SUCCESS );
$function_post_code
return ( 3 );
return ( DISPATCH_UNSUPPORTED_SUITE );
}
else
END
......@@ -283,9 +285,9 @@ while( my ($key, $value) = each(%case_deps) )
if( strcmp( str, "$key" ) == 0 )
{
#if defined($key)
return( 0 );
return( DEPENDENCY_SUPPORTED );
#else
return( 1 );
return( DEPENDENCY_NOT_SUPPORTED );
#endif
}
END
......@@ -298,7 +300,7 @@ while( my ($key, $value) = each(%mapping_values) )
if( strcmp( str, "$key" ) == 0 )
{
*value = ( $key );
return( 0 );
return( KEY_VALUE_MAPPING_FOUND );
}
END
......@@ -315,7 +317,7 @@ END
$dispatch_code =~ s/^(.+)/ $1/mg;
$test_main =~ s/TEST_FILENAME/$test_case_data/;
$test_main =~ s/TEST_FILENAME/$test_case_data/g;
$test_main =~ s/FUNCTION_CODE//;
$test_main =~ s/DEP_CHECK_CODE/$dep_check_code/;
$test_main =~ s/DISPATCH_FUNCTION/$dispatch_code/;
......
......@@ -2,6 +2,8 @@
# run-test-suites.pl
#
# This file is part of mbed TLS (https://tls.mbed.org)
#
# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved
#
# Purpose
......@@ -66,7 +68,8 @@ for my $suite (@suites)
print "(test cases passed:", $suite_cases_passed,
" failed:", $suite_cases_failed,
" skipped:", $suite_cases_skipped,
" of total:", ( $suite_cases_passed + $suite_cases_failed ),
" of total:", ($suite_cases_passed + $suite_cases_failed +
$suite_cases_skipped),
")\n"
}
......
......@@ -32,9 +32,18 @@ typedef UINT32 uint32_t;
/*----------------------------------------------------------------------------*/
/* Global variables */
/* Constants */
static int test_errors = 0;
#define DEPENDENCY_SUPPORTED 0
#define DEPENDENCY_NOT_SUPPORTED 1
#define KEY_VALUE_MAPPING_FOUND 0
#define KEY_VALUE_MAPPING_NOT_FOUND -1
#define DISPATCH_TEST_SUCCESS 0
#define DISPATCH_TEST_FN_NOT_FOUND 1
#define DISPATCH_INVALID_TEST_DATA 2
#define DISPATCH_UNSUPPORTED_SUITE 3
/*----------------------------------------------------------------------------*/
......@@ -80,6 +89,12 @@ static int test_errors = 0;
#endif
/*----------------------------------------------------------------------------*/
/* Global variables */
static int test_errors = 0;
/*----------------------------------------------------------------------------*/
/* Helper Functions */
......
......@@ -60,7 +60,7 @@ int verify_int( char *str, int *value )
MAPPING_CODE
mbedtls_printf( "Expected integer for parameter and got: %s\n", str );
return( -1 );
return( KEY_VALUE_MAPPING_NOT_FOUND );
}
......@@ -81,7 +81,7 @@ int dep_check( char *str )
DEP_CHECK_CODE
return( 1 );
return( DEPENDENCY_NOT_SUPPORTED );
}
int dispatch_test(int cnt, char *params[50])
......@@ -91,14 +91,18 @@ int dispatch_test(int cnt, char *params[50])
((void) params);
#if defined(TEST_SUITE_ACTIVE)
ret = DISPATCH_TEST_SUCCESS;
DISPATCH_FUNCTION
{
mbedtls_fprintf( stdout, "FAILED\nSkipping unknown test function '%s'\n", params[0] );
mbedtls_fprintf( stdout,
"FAILED\nSkipping unknown test function '%s'\n",
params[0] );
fflush( stdout );
return( 1 );
ret = DISPATCH_TEST_FN_NOT_FOUND;
}
#else
return( 3 );
ret = DISPATCH_UNSUPPORTED_SUITE;
#endif
return( ret );
}
......@@ -107,6 +111,19 @@ DISPATCH_FUNCTION
/*----------------------------------------------------------------------------*/
/* Main Test code */
#define USAGE \
"Usage: %s [OPTIONS] files...\n\n" \
" Command line arguments:\n" \
" files... One or more test data file. If no file is specified\n" \
" the followimg default test case is used:\n" \
" %s\n\n" \
" Options:\n" \
" -v | --verbose Display full information about each test\n" \
" -h | --help Display this information\n\n", \
argv[0], \
"TEST_FILENAME"
int get_line( FILE *f, char *buf, size_t len )
{
char *ret;
......@@ -216,11 +233,18 @@ static int run_test_snprintf( void )
int main(int argc, const char *argv[])
{
int testfile_index, testfile_count, ret, i, cnt;
int total_errors = 0, total_tests = 0, total_skipped = 0;
/* Local Configurations and options */
const char *default_filename = "TEST_FILENAME";
const char *test_filename = NULL;
const char **test_files = NULL;
int testfile_count = 0;
int option_verbose = 0;
/* Other Local variables */
int arg_index = 1;
const char *next_arg;
int testfile_index, ret, i, cnt;
int total_errors = 0, total_tests = 0, total_skipped = 0;
FILE *file;
char buf[5000];
char *params[50];
......@@ -253,17 +277,41 @@ int main(int argc, const char *argv[])
return( 0 );
}
if ( argc <= 1 )
while( arg_index < argc)
{
test_files = &default_filename;
testfile_count = 1;
next_arg = argv[ arg_index ];
if( strcmp(next_arg, "--verbose" ) == 0 ||
strcmp(next_arg, "-v" ) == 0 )
{
option_verbose = 1;
}
else if( strcmp(next_arg, "--help" ) == 0 ||
strcmp(next_arg, "-h" ) == 0 )
{
mbedtls_fprintf( stdout, USAGE );
mbedtls_exit( EXIT_SUCCESS );
}
else
{
/* Not an option, therefore treat all further arguments as the file
* list.
*/
test_files = &argv[ arg_index ];
testfile_count = argc - arg_index;
}
arg_index++;
}
else
/* If no files were specified, assume a default */
if ( test_files == NULL || testfile_count == 0 )
{
test_files = &argv[1];
testfile_count = argc - 1;
test_files = &default_filename;
testfile_count = 1;
}
/* Now begin to execute the tests in the testfiles */
for ( testfile_index = 0;
testfile_index < testfile_count;
testfile_index++ )
......@@ -280,7 +328,8 @@ int main(int argc, const char *argv[])
while( !feof( file ) )
{
int skip = 0;
int unmet_dep_count = 0;
char *unmet_dependencies[20];
if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
break;
......@@ -300,32 +349,61 @@ int main(int argc, const char *argv[])
if( strcmp( params[0], "depends_on" ) == 0 )
{
for( i = 1; i < cnt; i++ )
if( dep_check( params[i] ) != 0 )
skip = 1;
{
if( dep_check( params[i] ) != DEPENDENCY_SUPPORTED )
{
unmet_dependencies[ i-1 ] = strdup(params[i]);
if( unmet_dependencies[ i-1 ] == NULL )
{
mbedtls_printf("FATAL: Out of memory\n");
mbedtls_exit( MBEDTLS_PLATFORM_STD_EXIT_FAILURE );
}
unmet_dep_count++;
}
}
if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
break;
cnt = parse_arguments( buf, strlen(buf), params );
}
if( skip == 0 )
// If there are no unmet dependencies execute the test
if( unmet_dep_count == 0 )
{
test_errors = 0;
ret = dispatch_test( cnt, params );
}
if( skip == 1 || ret == 3 )
if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE )
{
total_skipped++;
mbedtls_fprintf( stdout, "----\n" );
if( 1 == option_verbose && ret == DISPATCH_UNSUPPORTED_SUITE )
{
mbedtls_fprintf( stdout, " Test Suite not enabled" );
}
if( 1 == option_verbose && unmet_dep_count > 0 )
{
mbedtls_fprintf( stdout, " Unmet dependencies: " );
while( unmet_dep_count > 0)
{
mbedtls_fprintf(stdout, "%s ",
unmet_dependencies[unmet_dep_count - 1]);
free(unmet_dependencies[unmet_dep_count - 1]);
unmet_dep_count--;
}
mbedtls_fprintf( stdout, "\n" );
}
fflush( stdout );
}
else if( ret == 0 && test_errors == 0 )
else if( ret == DISPATCH_TEST_SUCCESS && test_errors == 0 )
{
mbedtls_fprintf( stdout, "PASS\n" );
fflush( stdout );
}
else if( ret == 2 )
else if( ret == DISPATCH_INVALID_TEST_DATA )
{
mbedtls_fprintf( stderr, "FAILED: FATAL PARSE ERROR\n" );
fclose(file);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment