Latest update.

test/asynciotest.c

@@ -393,6 +393,8 @@ static int test_asyncio(int test)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("certname privkey\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert = test_get_argument(0))

test/bftest.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -434,28 +434,53 @@ static int test_bf_ofb64(void)
 }
 #endif
 
+typedef enum OPTION_choice {
+    OPT_ERR = -1,
+    OPT_EOF = 0,
+    OPT_PRINT,
+    OPT_TEST_ENUM
+} OPTION_CHOICE;
+
+const OPTIONS *test_get_options(void)
+{
+    static const OPTIONS test_options[] = {
+        OPT_TEST_OPTIONS_DEFAULT_USAGE,
+        { "print", OPT_PRINT, '-', "Output test tables instead of running tests"},
+        { NULL }
+    };
+    return test_options;
+}
+
 int setup_tests(void)
 {
 #ifndef OPENSSL_NO_BF
+    OPTION_CHOICE o;
 # ifdef CHARSET_EBCDIC
     int n;
-
     ebcdic2ascii(cbc_data, cbc_data, strlen(cbc_data));
     for (n = 0; n < 2; n++) {
         ebcdic2ascii(bf_key[n], bf_key[n], strlen(bf_key[n]));
     }
 # endif
 
-    if (test_get_argument(0) != NULL) {
-        print_test_data();
-    } else {
-        ADD_ALL_TESTS(test_bf_ecb_raw, 2);
-        ADD_ALL_TESTS(test_bf_ecb, NUM_TESTS);
-        ADD_ALL_TESTS(test_bf_set_key, KEY_TEST_NUM-1);
-        ADD_TEST(test_bf_cbc);
-        ADD_TEST(test_bf_cfb64);
-        ADD_TEST(test_bf_ofb64);
+    while ((o = opt_next()) != OPT_EOF) {
+        switch(o) {
+        case OPT_PRINT:
+            print_test_data();
+            return 1;
+        case OPT_TEST_CASES:
+            break;
+        default:
+           return 0;
+        }
     }
+
+    ADD_ALL_TESTS(test_bf_ecb_raw, 2);
+    ADD_ALL_TESTS(test_bf_ecb, NUM_TESTS);
+    ADD_ALL_TESTS(test_bf_set_key, KEY_TEST_NUM-1);
+    ADD_TEST(test_bf_cbc);
+    ADD_TEST(test_bf_cfb64);
+    ADD_TEST(test_bf_ofb64);
 #endif
     return 1;
 }

test/bioprinttest.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -252,10 +252,38 @@ static int test_big(void)
     return 1;
 }
 
+typedef enum OPTION_choice {
+    OPT_ERR = -1,
+    OPT_EOF = 0,
+    OPT_PRINT,
+    OPT_TEST_ENUM
+} OPTION_CHOICE;
+
+const OPTIONS *test_get_options(void)
+{
+    static const OPTIONS options[] = {
+        OPT_TEST_OPTIONS_DEFAULT_USAGE,
+        { "expected", OPT_PRINT, '-', "Output values" },
+        { NULL }
+    };
+    return options;
+}
 
 int setup_tests(void)
 {
-    justprint = test_has_option("-expected");
+    OPTION_CHOICE o;
+
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_PRINT:
+            justprint = 1;
+            break;
+        case OPT_TEST_CASES:
+            break;
+        default:
+            return 0;
+        }
+    }
 
     ADD_TEST(test_big);
     ADD_ALL_TESTS(test_fp, nelem(pw_params));
@@ -300,3 +328,4 @@ int test_flush_stderr(void)
 {
     return fflush(stderr);
 }
+

test/bntest.c

@@ -2261,6 +2261,17 @@ static int run_file_tests(int i)
     return c == 0;
 }
 
+const OPTIONS *test_get_options(void)
+{
+    enum { OPT_TEST_ENUM };
+    static const OPTIONS test_options[] = {
+        OPT_TEST_OPTIONS_WITH_EXTRA_USAGE("[file...]\n"),
+        { OPT_HELP_STR, 1, '-',
+          "file\tFile to run tests on. Normal tests are not run\n" },
+        { NULL }
+    };
+    return test_options;
+}
 
 int setup_tests(void)
 {

test/build.info

@@ -13,8 +13,9 @@ IF[{- !$disabled{tests} -}]
   SOURCE[libtestutil.a]=testutil/basic_output.c testutil/output_helpers.c \
           testutil/driver.c testutil/tests.c testutil/cb.c testutil/stanza.c \
           testutil/format_output.c testutil/tap_bio.c \
-          testutil/test_cleanup.c testutil/main.c testutil/init.c
-  INCLUDE[libtestutil.a]=../include
+          testutil/test_cleanup.c testutil/main.c testutil/init.c \
+          testutil/options.c testutil/test_options.c ../apps/opt.c
+  INCLUDE[libtestutil.a]=../include ..
   DEPEND[libtestutil.a]=../libcrypto
 
   PROGRAMS{noinst}=\
@@ -398,11 +399,9 @@ IF[{- !$disabled{tests} -}]
     DEPEND[cipher_overhead_test]=../libcrypto ../libssl libtestutil.a
   ENDIF
 
-  SOURCE[uitest]=uitest.c \
-                 {- rebase_files("../apps",
-                                 split(/\s+/, $target{apps_init_src})) -}
+  SOURCE[uitest]=uitest.c ../apps/apps_ui.c
   INCLUDE[uitest]=.. ../include ../apps
-  DEPEND[uitest]=../apps/libapps.a ../libcrypto ../libssl libtestutil.a
+  DEPEND[uitest]=../libcrypto ../libssl libtestutil.a
 
   SOURCE[cipherbytes_test]=cipherbytes_test.c
   INCLUDE[cipherbytes_test]=../include

test/clienthellotest.c

@@ -240,6 +240,8 @@ end:
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("sessionfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(sessionfile = test_get_argument(0)))

test/cmsapitest.c

@@ -1,3 +1,12 @@
+/*
+ * Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
 #include <string.h>
 
 #include <openssl/cms.h>
@@ -49,6 +58,8 @@ static int test_encrypt_decrypt(void)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     char *certin = NULL, *privkeyin = NULL;

test/conf_include_test.c

@@ -178,26 +178,50 @@ static int test_check_overflow(void)
     return 1;
 }
 
+typedef enum OPTION_choice {
+    OPT_ERR = -1,
+    OPT_EOF = 0,
+    OPT_FAIL,
+    OPT_TEST_ENUM
+} OPTION_CHOICE;
+
+const OPTIONS *test_get_options(void)
+{
+    static const OPTIONS test_options[] = {
+        OPT_TEST_OPTIONS_WITH_EXTRA_USAGE("conf_file\n"),
+        { "f", OPT_FAIL, '-', "A failure is expected" },
+        { NULL }
+    };
+    return test_options;
+}
+
 int setup_tests(void)
 {
     const char *conf_file;
-    const char *arg2;
+    OPTION_CHOICE o;
 
     if (!TEST_ptr(conf = NCONF_new(NULL)))
         return 0;
 
-    conf_file = test_get_argument(0);
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_FAIL:
+            expect_failure = 1;
+            break;
+        case OPT_TEST_CASES:
+            break;
+        default:
+            return 0;
+        }
+    }
 
+    conf_file = test_get_argument(0);
     if (!TEST_ptr(conf_file)
         || !TEST_ptr(in = BIO_new_file(conf_file, "r"))) {
         TEST_note("Unable to open the file argument");
         return 0;
     }
 
-    if ((arg2 = test_get_argument(1)) != NULL && *arg2 == 'f') {
-       expect_failure = 1;
-    }
-
     /*
      * For this test we need to chdir as we use relative
      * path names in the config files.

test/curve448_internal_test.c

@@ -682,19 +682,49 @@ static int test_x448(void)
     return 1;
 }
 
+typedef enum OPTION_choice {
+    OPT_ERR = -1,
+    OPT_EOF = 0,
+    OPT_PROGRESS,
+    OPT_SLOW,
+    OPT_TEST_ENUM
+} OPTION_CHOICE;
+
+const OPTIONS *test_get_options(void)
+{
+    static const OPTIONS test_options[] = {
+        OPT_TEST_OPTIONS_WITH_EXTRA_USAGE("conf_file\n"),
+        { "f", OPT_SLOW, '-', "Enables a slow test" },
+        { "v", OPT_PROGRESS, '-',
+              "Enables verbose mode (prints progress dots)" },
+        { NULL }
+    };
+    return test_options;
+}
+
 int setup_tests(void)
 {
-    /*
-     * The test vectors contain one test which takes a very long time to run,
-     * so we don't do that be default. Using the -f option will cause it to be
-     * run.
-     */
-    if (test_has_option("-f"))
-        max = 1000000;
+    OPTION_CHOICE o;
 
-    /* Print progress dots */
-    if (test_has_option("-v"))
-        verbose = 1;
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_TEST_CASES:
+            break;
+        default:
+            return 0;
+        /*
+         * The test vectors contain one test which takes a very long time to run
+         * so we don't do that be default. Using the -f option will cause it to
+         * be run.
+         */
+        case OPT_SLOW:
+            max = 1000000;
+            break;
+        case OPT_PROGRESS:
+            verbose = 1; /* Print progress dots */
+            break;
+        }
+    }
 
     ADD_TEST(test_x448);
     ADD_TEST(test_ed448);

test/d2i_test.c

@@ -106,6 +106,8 @@ static int test_bad_asn1(void)
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("item_name expected_error test_file.der\n")
+
 /*
  * Usage: d2i_test <name> <type> <file>, e.g.
  * d2i_test generalname bad_generalname.der
@@ -127,10 +129,8 @@ int setup_tests(void)
 
     if (!TEST_ptr(test_type_name = test_get_argument(0))
             || !TEST_ptr(expected_error_string = test_get_argument(1))
-            || !TEST_ptr(test_file = test_get_argument(2))) {
-        TEST_note("Usage: d2i_test item_name expected_error file.der");
+            || !TEST_ptr(test_file = test_get_argument(2)))
         return 0;
-    }
 
     item_type = ASN1_ITEM_lookup(test_type_name);
 

test/danetest.c

@@ -409,14 +409,14 @@ end:
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("basedomain CAfile tlsafile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(basedomain = test_get_argument(0))
             || !TEST_ptr(CAfile = test_get_argument(1))
-            || !TEST_ptr(tlsafile = test_get_argument(2))) {
-        TEST_error("Usage error: danetest basedomain CAfile tlsafile");
+            || !TEST_ptr(tlsafile = test_get_argument(2)))
         return 0;
-    }
 
     ADD_TEST(run_tlsatest);
     return 1;

test/dtlstest.c

@@ -328,6 +328,8 @@ static int test_dtls_duplicate_records(void)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert = test_get_argument(0))

test/ecstresstest.c

@@ -18,7 +18,7 @@
 
 #define NUM_REPEATS "1000000"
 
-static int64_t num_repeats;
+static intmax_t num_repeats;
 static int print_mode = 0;
 
 #ifndef OPENSSL_NO_EC
@@ -39,10 +39,10 @@ static const char *kP256DefaultResult =
  * point multiplication.
  * Returns the X-coordinate of the end result or NULL on error.
  */
-static BIGNUM *walk_curve(const EC_GROUP *group, EC_POINT *point, int64_t num)
+static BIGNUM *walk_curve(const EC_GROUP *group, EC_POINT *point, intmax_t num)
 {
     BIGNUM *scalar = NULL;
-    int64_t i;
+    intmax_t i;
 
     if (!TEST_ptr(scalar = BN_new())
             || !TEST_true(EC_POINT_get_affine_coordinates(group, point, scalar,
@@ -101,20 +101,21 @@ err:
 }
 #endif
 
-static int atoi64(const char *in, int64_t *result)
+typedef enum OPTION_choice {
+    OPT_ERR = -1,
+    OPT_EOF = 0,
+    OPT_NUM_REPEATS,
+    OPT_TEST_ENUM
+} OPTION_CHOICE;
+
+const OPTIONS *test_get_options(void)
 {
-    int64_t ret = 0;
-
-    for ( ; *in != '\0'; in++) {
-        char c = *in;
-
-        if (!isdigit((unsigned char)c))
-            return 0;
-        ret *= 10;
-        ret += (c - '0');
-    }
-    *result = ret;
-    return 1;
+    static const OPTIONS test_options[] = {
+        OPT_TEST_OPTIONS_DEFAULT_USAGE,
+        { "num", OPT_NUM_REPEATS, 'M', "Number of repeats" },
+        { NULL }
+    };
+    return test_options;
 }
 
 /*
@@ -124,22 +125,27 @@ static int atoi64(const char *in, int64_t *result)
  */
 int setup_tests(void)
 {
-    const char *p;
+    OPTION_CHOICE o;
 
-    if (!atoi64(NUM_REPEATS, &num_repeats)) {
+    if (!opt_imax(NUM_REPEATS, &num_repeats)) {
         TEST_error("Cannot parse " NUM_REPEATS);
         return 0;
     }
-    /*
-     * TODO(openssl-team): code under test/ should be able to reuse the option
-     * parsing framework currently in apps/.
-     */
-    p = test_get_option_argument("-num");
-    if (p != NULL) {
-        if (!atoi64(p, &num_repeats)
-                || num_repeats < 0)
-            return 0;
-        print_mode = 1;
+
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_NUM_REPEATS:
+            if (!opt_imax(opt_arg(), &num_repeats)
+                    || num_repeats < 0)
+                return 0;
+            print_mode = 1;
+            break;
+        case OPT_TEST_CASES:
+           break;
+        default:
+        case OPT_ERR:
+	    return 0;
+        }
     }
 
 #ifndef OPENSSL_NO_EC

test/evp_test.c

@@ -2901,14 +2901,14 @@ static int run_file_tests(int i)
     return c == 0;
 }
 
+OPT_TEST_DECLARE_USAGE("file...\n")
+
 int setup_tests(void)
 {
     size_t n = test_get_argument_count();
 
-    if (n == 0) {
-        TEST_error("Usage: %s file...", test_get_program_name());
+    if (n == 0)
         return 0;
-    }
 
     ADD_ALL_TESTS(run_file_tests, n);
     return 1;

test/fatalerrtest.c

@@ -82,6 +82,8 @@ static int test_fatalerr(void)
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert = test_get_argument(0))

test/gosttest.c

@@ -78,6 +78,8 @@ static int test_tls13(int idx)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile1 privkeyfile1 certfile2 privkeyfile2\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert1 = test_get_argument(0))

test/ocspapitest.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -133,6 +133,8 @@ static int test_resp_signer(void)
 }
 #endif
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(certstr = test_get_argument(0))

test/recipes/02-test_errstr.t

@@ -38,6 +38,43 @@ plan skip_all => 'OpenSSL is configured "no-autoerrinit" or "no-err"'
 # (this is documented)
 my @posix_errors = @{$Errno::EXPORT_TAGS{POSIX}};
 
+if ($^O eq 'MSWin32') {
+    # On Windows, these errors have been observed to not always be loaded by
+    # apps/openssl, while they are in perl, which causes a difference that we
+    # consider a false alarm.  So we skip checking these errors.
+    # Because we can't know exactly what symbols exist in a perticular perl
+    # version, we resort to discovering them directly in the Errno package
+    # symbol table.
+    my @error_skiplist = qw(
+        ENETDOWN
+        ENETUNREACH
+        ENETRESET
+        ECONNABORTED
+        EISCONN
+        ENOTCONN
+        ESHUTDOWN
+        ETOOMANYREFS
+        ETIMEDOUT
+        EHOSTDOWN
+        EHOSTUNREACH
+        EALREADY
+        EINPROGRESS
+        ESTALE
+        EUCLEAN
+        ENOTNAM
+        ENAVAIL
+        ENOMEDIUM
+        ENOKEY
+    );
+    @posix_errors =
+        grep {
+            my $x = $_;
+            ! grep {
+                exists $Errno::{$_} && $x == $Errno::{$_}
+            } @error_skiplist
+        } @posix_errors;
+}
+
 plan tests => scalar @posix_errors
     +1                          # Checking that error 128 gives 'reason(128)'
     +1                          # Checking that error 0 gives the library name

test/recipes/90-test_includes.t

@@ -11,15 +11,17 @@ plan skip_all => "test_includes doesn't work without posix-io"
     if disabled("posix-io");
 
 plan tests =>                   # The number of tests being performed
-    3
+    5
     + ($^O eq "VMS" ? 2 : 0);
 
 ok(run(test(["conf_include_test", data_file("includes.cnf")])), "test directory includes");
 ok(run(test(["conf_include_test", data_file("includes-file.cnf")])), "test file includes");
+ok(run(test(["conf_include_test", data_file("includes-eq.cnf")])), "test includes with equal character");
+ok(run(test(["conf_include_test", data_file("includes-eq-ws.cnf")])), "test includes with equal and whitespaces");
 if ($^O eq "VMS") {
     ok(run(test(["conf_include_test", data_file("vms-includes.cnf")])),
        "test directory includes, VMS syntax");
     ok(run(test(["conf_include_test", data_file("vms-includes-file.cnf")])),
        "test file includes, VMS syntax");
 }
-ok(run(test(["conf_include_test", data_file("includes-broken.cnf"), "f"])), "test broken includes");
+ok(run(test(["conf_include_test", "-f", data_file("includes-broken.cnf")])), "test broken includes");

test/recipes/90-test_includes_data/includes-eq-ws.cnf

@@ -0,0 +1,5 @@
+#
+# Example configuration file using includes.
+#
+
+.include = conf-includes

test/recipes/90-test_includes_data/includes-eq.cnf

@@ -0,0 +1,5 @@
+#
+# Example configuration file using includes.
+#
+
+.include=conf-includes

test/recordlentest.c

@@ -181,6 +181,8 @@ static int test_record_overflow(int idx)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert = test_get_argument(0))

test/ssl_test.c

@@ -500,6 +500,8 @@ err:
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("conf_file\n")
+
 int setup_tests(void)
 {
     long num_tests;

test/ssl_test_ctx_test.c

@@ -240,15 +240,15 @@ static int test_bad_configuration(int idx)
     return 1;
 }
 
+OPT_TEST_DECLARE_USAGE("conf_file\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(conf = NCONF_new(NULL)))
         return 0;
     /* argument should point to test/ssl_test_ctx_test.conf */
-    if (!TEST_int_gt(NCONF_load(conf, test_get_argument(0), NULL), 0)) {
-        TEST_note("Missing file argument");
+    if (!TEST_int_gt(NCONF_load(conf, test_get_argument(0), NULL), 0))
         return 0;
-    }
 
     ADD_TEST(test_empty_configuration);
     ADD_TEST(test_good_configuration);

test/sslapitest.c

@@ -6030,6 +6030,9 @@ static int test_ca_names(int tst)
     return testresult;
 }
 
+
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile srpvfile tmpfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert = test_get_argument(0))

test/sslbuffertest.c

@@ -157,6 +157,8 @@ int global_init(void)
     return 1;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     char *cert, *pkey;

test/sslcorrupttest.c

@@ -244,15 +244,15 @@ static int test_ssl_corrupt(int testidx)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     int n;
 
     if (!TEST_ptr(cert = test_get_argument(0))
-            || !TEST_ptr(privkey = test_get_argument(1))) {
-        TEST_note("Usage error: require cert and private key files");
+            || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;
-    }
 
     n = setup_cipher_list();
     if (n > 0)

test/testutil.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2014-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -15,6 +15,7 @@
 #include <openssl/err.h>
 #include <openssl/e_os2.h>
 #include <openssl/bn.h>
+#include "../apps/opt.h"
 
 /*-
  * Simple unit tests should implement setup_tests().
@@ -117,22 +118,80 @@
 #  define TEST_CASE_NAME __func__
 # endif                         /* __STDC_VERSION__ */
 
+
+/* The default test enum which should be common to all tests */
+#define OPT_TEST_ENUM \
+    OPT_TEST_HELP = 500, \
+    OPT_TEST_LIST, \
+    OPT_TEST_SINGLE, \
+    OPT_TEST_ITERATION, \
+    OPT_TEST_INDENT, \
+    OPT_TEST_SEED
+
+/* The Default test OPTIONS common to all tests (without a usage string) */
+#define OPT_TEST_OPTIONS \
+    { OPT_HELP_STR, 1,  '-', "Valid options are:\n" }, \
+    { "help", OPT_TEST_HELP, '-', "Display this summary" }, \
+    { "list", OPT_TEST_LIST, '-', "Display the list of tests available" }, \
+    { "test", OPT_TEST_SINGLE, 's', "Run a single test by id or name" }, \
+    { "iter", OPT_TEST_ITERATION, 'n', "Run a single iteration of a test" }, \
+    { "indent", OPT_TEST_INDENT,'p', "Number of tabs added to output" }, \
+    { "seed", OPT_TEST_SEED, 'n', "Seed value to randomize tests with" }
+
+/* The Default test OPTIONS common to all tests starting with an additional usage string */
+#define OPT_TEST_OPTIONS_WITH_EXTRA_USAGE(usage) \
+    { OPT_HELP_STR, 1, '-', "Usage: %s [options] " usage }, \
+    OPT_TEST_OPTIONS
+
+/* The Default test OPTIONS common to all tests with an default usage string */
+#define OPT_TEST_OPTIONS_DEFAULT_USAGE \
+    { OPT_HELP_STR, 1, '-', "Usage: %s [options]\n" }, \
+    OPT_TEST_OPTIONS
+
 /*
- * Tests that need access to command line arguments should use the functions:
- * test_get_argument(int n) to get the nth argument, the first argument is
- *      argument 0.  This function returns NULL on error.
- * test_get_argument_count() to get the count of the arguments.
- * test_has_option(const char *) to check if the specified option was passed.
- * test_get_option_argument(const char *) to get an option which includes an
- *      argument.  NULL is returns if the option is not found.
- * const char *test_get_program_name(void) returns the name of the test program
- *      being executed.
+ * Optional Cases that need to be ignored by the test app when using opt_next(),
+ * (that are handled internally).
+ */
+#define OPT_TEST_CASES \
+         OPT_TEST_HELP: \
+    case OPT_TEST_LIST: \
+    case OPT_TEST_SINGLE: \
+    case OPT_TEST_ITERATION: \
+    case OPT_TEST_INDENT: \
+    case OPT_TEST_SEED
+
+/*
+ * Tests that use test_get_argument() that dont have any additional options
+ * (i.e- dont use opt_next()) can use this to set the usage string.
+ * It embeds test_get_options() which gives default command line options for
+ * the test system.
+ *
+ * Tests that need to use opt_next() need to specify
+ *  (1) test_get_options() containing an options[] (Which should include either
+ *    OPT_TEST_OPTIONS_DEFAULT_USAGE OR
+ *    OPT_TEST_OPTIONS_WITH_EXTRA_USAGE).
+ *  (2) An enum outside the test_get_options() which contains OPT_TEST_ENUM, as
+ *      well as the additional options that need to be handled.
+ *  (3) case OPT_TEST_CASES: break; inside the opt_next() handling code.
+ */
+#define OPT_TEST_DECLARE_USAGE(usage_str) \
+const OPTIONS *test_get_options(void) \
+{ \
+    enum { OPT_TEST_ENUM }; \
+    static const OPTIONS options[] = { \
+        OPT_TEST_OPTIONS_WITH_EXTRA_USAGE(usage_str), \
+        { NULL } \
+    }; \
+    return options; \
+}
+
+/*
+ * Used to read non optional command line values that follow after the options.
+ * Returns NULL if there is no argument.
  */
-const char *test_get_program_name(void);
 char *test_get_argument(size_t n);
+/* Return the number of additional non optional command line arguments */
 size_t test_get_argument_count(void);
-int test_has_option(const char *option);
-const char *test_get_option_argument(const char *option);
 
 /*
  * Internal helpers. Test programs shouldn't use these directly, but should
@@ -150,6 +209,16 @@ void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num,
 int global_init(void);
 int setup_tests(void);
 void cleanup_tests(void);
+/*
+ * Used to supply test specific command line options,
+ * If non optional parameters are used, then the first entry in the OPTIONS[]
+ * should contain:
+ * { OPT_HELP_STR, 1, '-', "list of non optional commandline params\n"},
+ * The last entry should always be { NULL }.
+ *
+ * Run the test locally using './test/test_name -help' to check the usage.
+ */
+const OPTIONS *test_get_options(void);
 
 /*
  *  Test assumption verification helpers.

test/testutil/driver.c

@@ -21,6 +21,7 @@
 # define strdup _strdup
 #endif
 
+
 /*
  * Declares the structures needed to register each test case function.
  */
@@ -36,14 +37,21 @@ typedef struct test_info {
 
 static TEST_INFO all_tests[1024];
 static int num_tests = 0;
+static int show_list = 0;
+static int single_test = -1;
+static int single_iter = -1;
+static int level = 0;
 static int seed = 0;
 /*
- * A parameterised tests runs a loop of test cases.
+ * A parameterised test runs a loop of test cases.
  * |num_test_cases| counts the total number of test cases
  * across all tests.
  */
 static int num_test_cases = 0;
 
+static int process_shared_options(void);
+
+
 void add_test(const char *test_case_name, int (*test_fn) (void))
 {
     assert(num_tests != OSSL_NELEM(all_tests));
@@ -66,8 +74,6 @@ void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
     num_test_cases += num;
 }
 
-static int level = 0;
-
 int subtest_level(void)
 {
     return level;
@@ -99,21 +105,26 @@ static int gcd(int a, int b)
     return a;
 }
 
-void setup_test_framework(void)
+static void set_seed(int s)
+{
+    seed = s;
+    if (seed <= 0)
+        seed = (int)time(NULL);
+    test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
+    test_flush_stdout();
+    srand(seed);
+}
+
+
+int setup_test_framework(int argc, char *argv[])
 {
-    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
     char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
+    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
 
-    level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;
-
-    if (test_seed != NULL) {
-        seed = atoi(test_seed);
-        if (seed <= 0)
-            seed = (int)time(NULL);
-        test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
-        test_flush_stdout();
-        srand(seed);
-    }
+    if (TAP_levels != NULL)
+        level = 4 * atoi(TAP_levels);
+    if (test_seed != NULL)
+        set_seed(atoi(test_seed));
 
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
     if (should_report_leaks()) {
@@ -121,8 +132,115 @@ void setup_test_framework(void)
         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
     }
 #endif
+    if (!opt_init(argc, argv, test_get_options()))
+        return 0;
+    return 1;
 }
 
+
+/*
+ * This can only be called after setup() has run, since num_tests and
+ * all_tests[] are setup at this point
+ */
+static int check_single_test_params(char *name, char *testname, char *itname)
+{
+    if (name != NULL) {
+        int i;
+        for (i = 0; i < num_tests; ++i) {
+            if (strcmp(name, all_tests[i].test_case_name) == 0) {
+                single_test = 1 + i;
+                break;
+	    } 
+        }
+        if (i >= num_tests) 
+	    single_test = atoi(name);
+    }
+
+
+    /* if only iteration is specified, assume we want the first test */
+    if (single_test == -1 && single_iter != -1)
+        single_test = 1;
+
+    if (single_test != -1) {
+        if (single_test < 1 || single_test > num_tests) {
+            test_printf_stderr("Invalid -%s value "
+                               "(Value must be a valid test name OR a value between %d..%d)\n",
+                               testname, 1, num_tests);
+            return 0;
+        }
+    }
+    if (single_iter != -1) {
+        if (all_tests[single_test - 1].num == -1) {
+            test_printf_stderr("-%s option is not valid for test %d:%s\n",
+                               itname,
+                               single_test,
+                               all_tests[single_test - 1].test_case_name);
+            return 0;
+        } else if (single_iter < 1
+                   || single_iter > all_tests[single_test - 1].num) {
+            test_printf_stderr("Invalid -%s value for test %d:%s\t"
+                               "(Value must be in the range %d..%d)\n",
+                               itname, single_test,
+                               all_tests[single_test - 1].test_case_name,
+                               1, all_tests[single_test - 1].num);
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static int process_shared_options(void)
+{
+    OPTION_CHOICE_DEFAULT o;
+    int value;
+    int ret = -1;
+    char *flag_test = "";
+    char *flag_iter = "";
+    char *testname = NULL;
+
+    opt_begin();
+    while ((o = opt_next()) != OPT_EOF) {
+        switch (o) {
+        /* Ignore any test options at this level */
+        default:
+            break;
+        case OPT_ERR:
+            return ret;
+        case OPT_TEST_HELP:
+            opt_help(test_get_options());
+            return 0;
+        case OPT_TEST_LIST:
+            show_list = 1;
+            break;
+        case OPT_TEST_SINGLE:
+            flag_test = opt_flag();
+            testname = opt_arg();
+            break;
+        case OPT_TEST_ITERATION:
+            flag_iter = opt_flag();
+            if (!opt_int(opt_arg(), &single_iter))
+                goto end;
+            break;
+        case OPT_TEST_INDENT:
+            if (!opt_int(opt_arg(), &value))
+                goto end;
+            level = 4 * value;
+            break;
+        case OPT_TEST_SEED:
+            if (!opt_int(opt_arg(), &value))
+                goto end;
+            set_seed(value);
+            break;
+        }
+    }
+    if (!check_single_test_params(testname, flag_test, flag_iter))
+        goto end;
+    ret = 1;
+end:
+    return ret;
+}
+
+
 int pulldown_test_framework(int ret)
 {
     set_test_title(NULL);
@@ -131,7 +249,6 @@ int pulldown_test_framework(int ret)
         && CRYPTO_mem_leaks_cb(openssl_error_cb, NULL) <= 0)
         return EXIT_FAILURE;
 #endif
-
     return ret;
 }
 
@@ -176,14 +293,21 @@ int run_tests(const char *test_prog_name)
     int ii, i, jj, j, jstep;
     int permute[OSSL_NELEM(all_tests)];
 
+    i = process_shared_options();
+    if (i == 0)
+        return EXIT_SUCCESS;
+    if (i == -1)
+        return EXIT_FAILURE;
+
     if (num_tests < 1) {
         test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
                            test_prog_name);
-    } else {
+    } else if (show_list == 0 && single_test == -1) {
         if (level > 0)
             test_printf_stdout("%*s# Subtest: %s\n", level, "", test_prog_name);
         test_printf_stdout("%*s1..%d\n", level, "", num_tests);
     }
+
     test_flush_stdout();
 
     for (i = 0; i < num_tests; i++)
@@ -198,12 +322,25 @@ int run_tests(const char *test_prog_name)
 
     for (ii = 0; ii != num_tests; ++ii) {
         i = permute[ii];
-        if (all_tests[i].num == -1) {
+
+        if (single_test != -1 && ((i+1) != single_test)) {
+            continue;
+        }
+        else if (show_list) {
+            if (all_tests[i].num != -1) {
+                test_printf_stdout("%d - %s (%d..%d)\n", ii + 1,
+                                   all_tests[i].test_case_name, 1,
+                                   all_tests[i].num);
+            } else {
+                test_printf_stdout("%d - %s\n", ii + 1,
+                                   all_tests[i].test_case_name);
+            }
+            test_flush_stdout();
+        } else if (all_tests[i].num == -1) {
             int ret = 0;
 
             set_test_title(all_tests[i].test_case_name);
             ret = all_tests[i].test_fn();
-
             verdict = 1;
             if (!ret) {
                 verdict = 0;
@@ -215,7 +352,7 @@ int run_tests(const char *test_prog_name)
             int num_failed_inner = 0;
 
             level += 4;
-            if (all_tests[i].subtest) {
+            if (all_tests[i].subtest && single_iter == -1) {
                 test_printf_stdout("%*s# Subtest: %s\n", level, "",
                                    all_tests[i].test_case_name);
                 test_printf_stdout("%*s%d..%d\n", level, "", 1,
@@ -235,6 +372,8 @@ int run_tests(const char *test_prog_name)
                 int ret;
 
                 j = (j + jstep) % all_tests[i].num;
+                if (single_iter != -1 && ((jj + 1) != single_iter))
+                    continue;
                 set_test_title(NULL);
                 ret = all_tests[i].param_test_fn(j);
 

test/testutil/main.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -8,28 +8,9 @@
  */
 
 #include "../testutil.h"
-#include "internal/nelem.h"
 #include "output.h"
 #include "tu_local.h"
 
-#include <string.h>
-
-static size_t arg_count;
-static char **args;
-static unsigned char arg_used[1000];
-
-static void check_arg_usage(void)
-{
-    size_t i, n = arg_count < OSSL_NELEM(arg_used) ? arg_count
-                                                   : OSSL_NELEM(arg_used);
-
-    for (i = 0; i < n; i++)
-        if (!arg_used[i+1])
-            test_printf_stderr("Warning ignored command-line argument %zu: %s\n",
-                               i, args[i+1]);
-    if (i < arg_count)
-        test_printf_stderr("Warning arguments %zu and later unchecked\n", i);
-}
 
 int main(int argc, char *argv[])
 {
@@ -42,65 +23,18 @@ int main(int argc, char *argv[])
         return ret;
     }
 
-    arg_count = argc - 1;
-    args = argv;
+    if (!setup_test_framework(argc, argv))
+        goto end;
 
-    setup_test_framework();
-
-    if (setup_tests())
+    if (setup_tests()) {
         ret = run_tests(argv[0]);
-    cleanup_tests();
-    check_arg_usage();
-
+        cleanup_tests();
+        opt_check_usage();
+    } else {
+        opt_help(test_get_options());
+    }
+end:
     ret = pulldown_test_framework(ret);
     test_close_streams();
     return ret;
 }
-
-const char *test_get_program_name(void)
-{
-    return args[0];
-}
-
-char *test_get_argument(size_t n)
-{
-    if (n > arg_count)
-        return NULL;
-    if (n + 1 < OSSL_NELEM(arg_used))
-        arg_used[n + 1] = 1;
-    return args[n + 1];
-}
-
-size_t test_get_argument_count(void)
-{
-    return arg_count;
-}
-
-int test_has_option(const char *option)
-{
-    size_t i;
-
-    for (i = 1; i <= arg_count; i++)
-        if (strcmp(args[i], option) == 0) {
-            arg_used[i] = 1;
-            return 1;
-        }
-    return 0;
-}
-
-const char *test_get_option_argument(const char *option)
-{
-    size_t i, n = strlen(option);
-
-    for (i = 1; i <= arg_count; i++)
-        if (strncmp(args[i], option, n) == 0) {
-            arg_used[i] = 1;
-            if (args[i][n] == '\0' && i + 1 < arg_count) {
-                arg_used[++i] = 1;
-                return args[i];
-            }
-            return args[i] + n;
-        }
-    return NULL;
-}
-

test/testutil/options.c

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+#include "internal/nelem.h"
+#include "tu_local.h"
+#include "output.h"
+
+
+static int used[100] = { 0 };
+
+
+size_t test_get_argument_count(void)
+{
+    return opt_num_rest();
+}
+
+char *test_get_argument(size_t n)
+{
+    char **argv = opt_rest();
+
+    OPENSSL_assert(n < sizeof(used));
+    if ((int)n >= opt_num_rest() || argv == NULL)
+        return NULL;
+    used[n] = 1;
+    return argv[n];
+}
+
+void opt_check_usage(void)
+{
+    int i;
+    char **argv = opt_rest();
+    int n, arg_count = opt_num_rest();
+
+    if (arg_count > (int)OSSL_NELEM(used))
+        n = (int)OSSL_NELEM(used);
+    else
+        n = arg_count;
+    for (i = 0; i < n; i++) {
+        if (used[i] == 0)
+            test_printf_stderr("Warning ignored command-line argument %d: %s\n",
+                               i, argv[i]);
+    }
+    if (i < arg_count)
+        test_printf_stderr("Warning arguments %d and later unchecked\n", i);
+}
+
+int opt_printf_stderr(const char *fmt, ...)
+{
+    va_list ap;
+    int ret;
+
+    va_start(ap, fmt);
+    ret = test_vprintf_stderr(fmt, ap);
+    va_end(ap);
+    return ret;
+}
+

test/testutil/test_options.c

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+#include "tu_local.h"
+
+/* An overridable list of command line options */
+const OPTIONS *test_get_options(void)
+{
+    static const OPTIONS default_options[] = {
+        OPT_TEST_OPTIONS_DEFAULT_USAGE,
+        { NULL }
+    };
+    return default_options;
+}

test/testutil/tu_local.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -44,8 +44,16 @@ void test_fail_memory_message(const char *prefix, const char *file,
                               const unsigned char *m1, size_t l1,
                               const unsigned char *m2, size_t l2);
 
-void setup_test_framework(void);
+__owur int setup_test_framework(int argc, char *argv[]);
 __owur int pulldown_test_framework(int ret);
 
 __owur int run_tests(const char *test_prog_name);
 void set_test_title(const char *title);
+
+typedef enum OPTION_choice_default {
+    OPT_ERR = -1,
+    OPT_EOF = 0,
+    OPT_TEST_ENUM
+} OPTION_CHOICE_DEFAULT;
+void opt_check_usage(void);
+

test/tls13ccstest.c

@@ -481,6 +481,8 @@ static int test_tls13ccs(int tst)
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(cert = test_get_argument(0))

test/uitest.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2002-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -11,11 +11,9 @@
 #include <string.h>
 #include <openssl/opensslconf.h>
 #include <openssl/err.h>
-#include "apps.h"
+#include "apps_ui.h"
 #include "testutil.h"
 
-/* apps/apps.c depend on these */
-char *default_config_file = NULL;
 
 #include <openssl/ui.h>
 

test/v3ext.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -37,6 +37,8 @@ end:
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("cert.pem\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(infile = test_get_argument(0)))

test/verify_extra_test.c

@@ -175,14 +175,14 @@ static int test_store_ctx(void)
     return testresult;
 }
 
+OPT_TEST_DECLARE_USAGE("roots.pem untrusted.pem bad.pem\n")
+
 int setup_tests(void)
 {
     if (!TEST_ptr(roots_f = test_get_argument(0))
             || !TEST_ptr(untrusted_f = test_get_argument(1))
-            || !TEST_ptr(bad_f = test_get_argument(2))) {
-        TEST_error("usage: verify_extra_test roots.pem untrusted.pem bad.pem\n");
+            || !TEST_ptr(bad_f = test_get_argument(2)))
         return 0;
-    }
 
     ADD_TEST(test_alt_chains_cert_forgery);
     ADD_TEST(test_store_ctx);

test/x509_check_cert_pkey_test.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -106,14 +106,25 @@ failed:
     return ret;
 }
 
+const OPTIONS *test_get_options(void)
+{
+    enum { OPT_TEST_ENUM };
+    static const OPTIONS test_options[] = {
+        OPT_TEST_OPTIONS_WITH_EXTRA_USAGE("certname key.pem type expected\n"),
+        { OPT_HELP_STR, 1, '-', "certname\tCertificate filename .pem/.req\n" },
+        { OPT_HELP_STR, 1, '-', "type\t\tvalue must be 'pem' or 'req'\n" },
+        { OPT_HELP_STR, 1, '-', "expected\tthe expected return value\n" },
+        { NULL }
+    };
+    return test_options;
+}
+
 int setup_tests(void)
 {
     if (!TEST_ptr(c = test_get_argument(0))
             || !TEST_ptr(k = test_get_argument(1))
             || !TEST_ptr(t = test_get_argument(2))
             || !TEST_ptr(e = test_get_argument(3))) {
-        TEST_note("usage: x509_check_cert_pkey cert.pem|cert.req"
-                  " key.pem cert|req <expected>");
         return 0;
     }
 

test/x509_dup_cert_test.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved.
  * Copyright (c) 2017, Oracle and/or its affiliates.  All rights reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
@@ -33,14 +33,14 @@ static int test_509_dup_cert(int n)
     return ret;
 }
 
+OPT_TEST_DECLARE_USAGE("cert.pem...\n")
+
 int setup_tests(void)
 {
     size_t n = test_get_argument_count();
 
-    if (!TEST_int_gt(n, 0)) {
-        TEST_note("usage: x509_dup_cert_test cert.pem...");
+    if (!TEST_int_gt(n, 0))
         return 0;
-    }
 
     ADD_ALL_TESTS(test_509_dup_cert, n);
     return 1;

test/x509aux.c

@@ -161,14 +161,13 @@ static int test_certs(int num)
     return 0;
 }
 
+OPT_TEST_DECLARE_USAGE("certfile...\n")
+
 int setup_tests(void)
 {
     size_t n = test_get_argument_count();
-
-    if (n == 0) {
-        TEST_error("usage: %s certfile...", test_get_program_name());
+    if (n == 0)
         return 0;
-    }
 
     ADD_ALL_TESTS(test_certs, (int)n);
     return 1;