Add benchmark sync
authorNeal H. Walfield <neal@pep.foundation>
Sat, 01 Jun 2019 21:24:45 +0200
branchsync
changeset 37995f0990b24c15
parent 3797 657dc02b1cc3
child 3800 fb9769f33508
child 3801 8854f7fd79a8
Add benchmark

- By default, this is just a normal test.

- Defining LOG=1 in the source changes it to a benchmark. See the
file for documentation.
test/include/LotsOfKeysTests.h
test/src/engine_tests/LotsOfKeysTests.cc
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/test/include/LotsOfKeysTests.h	Sat Jun 01 21:24:45 2019 +0200
     1.3 @@ -0,0 +1,19 @@
     1.4 +// This file is under GNU General Public License 3.0
     1.5 +// see LICENSE.txt
     1.6 +
     1.7 +#ifndef LOTS_OF_KEYS_H
     1.8 +#define LOTS_OF_KEYS_H
     1.9 +
    1.10 +#include <string>
    1.11 +#include "EngineTestIndividualSuite.h"
    1.12 +
    1.13 +using namespace std;
    1.14 +
    1.15 +class LotsOfKeysTests : public EngineTestIndividualSuite {
    1.16 +    public:
    1.17 +        LotsOfKeysTests(string test_suite, string test_home_dir);
    1.18 +    private:
    1.19 +        void check();
    1.20 +};
    1.21 +
    1.22 +#endif
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/test/src/engine_tests/LotsOfKeysTests.cc	Sat Jun 01 21:24:45 2019 +0200
     2.3 @@ -0,0 +1,275 @@
     2.4 +// This file is under GNU General Public License 3.0
     2.5 +// see LICENSE.txt
     2.6 +
     2.7 +#include <stdlib.h>
     2.8 +#include <string>
     2.9 +#include <cstring>
    2.10 +#include <cpptest.h>
    2.11 +#include <fstream>
    2.12 +#include <sys/time.h>
    2.13 +#include <assert.h>
    2.14 +#include <sys/types.h>
    2.15 +#include <unistd.h>
    2.16 +#include <time.h>
    2.17 +
    2.18 +#include "pEpEngine.h"
    2.19 +
    2.20 +#include "test_util.h"
    2.21 +#include "EngineTestIndividualSuite.h"
    2.22 +#include "LotsOfKeysTests.h"
    2.23 +
    2.24 +// Whether to log to the file system (in /tmp).
    2.25 +#define LOG 0
    2.26 +// How to plot (in R).  You need to enable LOG above.
    2.27 +#if 0
    2.28 +
    2.29 +library(ggplot2);
    2.30 +
    2.31 +t = "2019XXXX-XXXXXX";
    2.32 +find_keys = read.csv(sprintf("/tmp/pep-benchmark-find-keys-%s.csv", t),
    2.33 +                     header=T, sep=",")
    2.34 +get_ids = read.csv(sprintf("/tmp/pep-benchmark-get-identity-%s.csv", t),
    2.35 +                   header=T, sep=",")
    2.36 +ggplot(find_keys, aes(x=Keys, y=run0_us_per_op), tag="FOO") +
    2.37 +    geom_point() +
    2.38 +    geom_point(data = get_ids, aes(Keys, run0_us_per_op), colour = 'red') +
    2.39 +    scale_x_log10() + guides(fill = guide_legend()) +
    2.40 +    labs(title="Microseconds/Op for a Key/Management DB with x keys",
    2.41 +         x="Keys in DB", y="usec")
    2.42 +
    2.43 +ggsave("/tmp/pep-benchmark.pdf", width = 4, height = 4)
    2.44 +#endif
    2.45 +
    2.46 +
    2.47 +using namespace std;
    2.48 +
    2.49 +LotsOfKeysTests::LotsOfKeysTests(string suitename, string test_home_dir) :
    2.50 +    EngineTestIndividualSuite::EngineTestIndividualSuite(suitename, test_home_dir) {
    2.51 +    add_test_to_suite(std::pair<std::string, void (Test::Suite::*)()>(string("LotsOfKeysTests::check"),
    2.52 +                                                                      static_cast<Func>(&LotsOfKeysTests::check)));
    2.53 +}
    2.54 +
    2.55 +// Only really run the bench mark if logging is enabled.
    2.56 +#if LOG > 0
    2.57 +// Maximum number of keys.
    2.58 +#  define N 31622
    2.59 +// Amount of time to run each benchmark, in usecs.
    2.60 +#  define TIME 2 * 1000000
    2.61 +#else
    2.62 +// Don't actually run the benchmark.
    2.63 +#  define N 3
    2.64 +#  define TIME 0
    2.65 +#endif
    2.66 +
    2.67 +// Number of times to run each benchmark.
    2.68 +#define REPITIONS 3
    2.69 +
    2.70 +// 10^x, x=0.5.. step 0.5.
    2.71 +const int exp[] = { 3, 10, 31, 100, 316, 1000, 3162, 10000, 31622,
    2.72 +                    100000, 316227, 1000000, 3162277, 10000000,
    2.73 +                    31622776, 100000000, 316227766, 1000000000 };
    2.74 +#define BENCHMARKS (sizeof(exp) / sizeof(exp[0]))
    2.75 +
    2.76 +struct stats {
    2.77 +    FILE *fp;
    2.78 +    const char *name;
    2.79 +    struct {
    2.80 +        // Number of keys in the db.
    2.81 +        int keys;
    2.82 +        unsigned long long time[REPITIONS];
    2.83 +        unsigned long long ops[REPITIONS];
    2.84 +    } benchmarks[BENCHMARKS];
    2.85 +};
    2.86 +
    2.87 +struct stats *stats_new(const char *name)
    2.88 +{
    2.89 +    struct stats *stats = (struct stats *) calloc(1, sizeof(*stats));
    2.90 +    stats->name = name;
    2.91 +    return stats;
    2.92 +}
    2.93 +
    2.94 +unsigned long long time() {
    2.95 +    struct timeval tv;
    2.96 +    int err = gettimeofday(&tv, NULL);
    2.97 +    assert(err == 0);
    2.98 +
    2.99 +    return (unsigned long long) (tv.tv_sec * 1000000 + tv.tv_usec);
   2.100 +}
   2.101 +
   2.102 +static char start_time[100];
   2.103 +
   2.104 +static void dump_stats(struct stats *stats, int benchmark) {
   2.105 +    if (LOG && ! stats->fp) {
   2.106 +        char fn[1024];
   2.107 +        sprintf(fn, "/tmp/pep-benchmark-%s-%s.csv", stats->name, start_time);
   2.108 +        stats->fp = fopen(fn , "w");
   2.109 +        if (! stats->fp) {
   2.110 +            printf("Opening %s failed.\n", fn);
   2.111 +            abort();
   2.112 +        }
   2.113 +
   2.114 +        fprintf(stats->fp, "Keys");
   2.115 +        for (int iter = 0; iter < REPITIONS; iter ++) {
   2.116 +            fprintf(stats->fp, ", run%d_us_per_op, run%d_ops_per_sec", iter, iter);
   2.117 +        }
   2.118 +        fprintf(stats->fp, "\n");
   2.119 +    }
   2.120 +
   2.121 +    printf("%-8d keys", stats->benchmarks[benchmark].keys);
   2.122 +    if (stats->fp)
   2.123 +        fprintf(stats->fp, "%d", stats->benchmarks[benchmark].keys);
   2.124 +
   2.125 +    for (int iter = 0; iter < REPITIONS; iter ++) {
   2.126 +        double us_per_lookup = (double) stats->benchmarks[benchmark].time[iter]
   2.127 +            / (double) stats->benchmarks[benchmark].ops[iter];
   2.128 +        double ops_per_second = (double) stats->benchmarks[benchmark].ops[iter]
   2.129 +            / ((double) stats->benchmarks[benchmark].time[iter] / 1000000);
   2.130 +
   2.131 +        printf("\t%.02f us/op (%.1f ops/s)", us_per_lookup, ops_per_second);
   2.132 +        if (stats->fp)
   2.133 +            fprintf(stats->fp, ", %f, %f", us_per_lookup, ops_per_second);
   2.134 +    }
   2.135 +
   2.136 +    printf("\n");
   2.137 +    if (stats->fp) {
   2.138 +        fprintf(stats->fp, "\n");
   2.139 +        fflush(stats->fp);
   2.140 +    }
   2.141 +}
   2.142 +
   2.143 +void LotsOfKeysTests::check() {
   2.144 +    struct tm tm;
   2.145 +    time_t t = time((time_t) NULL);
   2.146 +    localtime_r(&t, &tm);
   2.147 +    strftime(start_time, sizeof(start_time), "%Y%m%d-%H%M%S", &tm);
   2.148 +
   2.149 +    struct stats *find_keys_stats = stats_new("find-keys");
   2.150 +    struct stats *get_identity_stats = stats_new("get-identity");
   2.151 +
   2.152 +    int benchmark = 0;
   2.153 +    PEP_STATUS status;
   2.154 +    pEp_identity **ids = (pEp_identity **) calloc(N, sizeof(*ids));
   2.155 +    assert(ids);
   2.156 +
   2.157 +    status = config_cipher_suite(session, PEP_CIPHER_SUITE_CV25519);
   2.158 +    TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
   2.159 +
   2.160 +    for (int key = 0; key < N; key ++) {
   2.161 +        // Create key
   2.162 +        char email[1024];
   2.163 +        sprintf(email, "%09d@example.org", key);
   2.164 +
   2.165 +        ids[key] = new_identity(strdup(email), NULL, NULL, "Test User");
   2.166 +        status = update_identity(session, ids[key]);
   2.167 +        TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
   2.168 +
   2.169 +        if (key == 0) {
   2.170 +            printf("\nRaw identity:\n");
   2.171 +            printf("   address = %s\n", ids[0]->address);
   2.172 +            printf("       fpr = %s\n", ids[0]->fpr);
   2.173 +            printf("   user_id = %s\n", ids[0]->user_id);
   2.174 +            printf("  username = %s\n", ids[0]->username);
   2.175 +        }
   2.176 +
   2.177 +        status = generate_keypair(session, ids[key]);
   2.178 +        TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
   2.179 +
   2.180 +        if (key == 0) {
   2.181 +            printf("\nAfter generating a key:\n");
   2.182 +            printf("   address = %s\n", ids[0]->address);
   2.183 +            printf("       fpr = %s\n", ids[0]->fpr);
   2.184 +            printf("   user_id = %s\n", ids[0]->user_id);
   2.185 +            printf("  username = %s\n", ids[0]->username);
   2.186 +        }
   2.187 +
   2.188 +        status = set_identity(session, ids[key]);
   2.189 +        TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
   2.190 +
   2.191 +        if (key == 0) {
   2.192 +            printf("\nSetting identity:\n");
   2.193 +            printf("   address = %s\n", ids[0]->address);
   2.194 +            printf("       fpr = %s\n", ids[0]->fpr);
   2.195 +            printf("   user_id = %s\n", ids[0]->user_id);
   2.196 +            printf("  username = %s\n", ids[0]->username);
   2.197 +        }
   2.198 +
   2.199 +        bool do_benchmark = false;
   2.200 +        for (int i = 0; i < sizeof(exp) / sizeof(exp[0]); i ++) {
   2.201 +            if (key + 1 == exp[i]) {
   2.202 +                do_benchmark = true;
   2.203 +            } else if (key + 1 < exp[i]) {
   2.204 +                break;
   2.205 +            }
   2.206 +        }
   2.207 +
   2.208 +        if (! do_benchmark)
   2.209 +            continue;
   2.210 +
   2.211 +
   2.212 +        // Look up a random key by its email address.
   2.213 +        //
   2.214 +        // This doesn't use the engine, only the pgp
   2.215 +        // implementation.  For Sequoia, this should run in O(log
   2.216 +        // N).
   2.217 +        find_keys_stats->benchmarks[benchmark].keys = key + 1;
   2.218 +        for (int iter = 0; iter < REPITIONS; iter ++) {
   2.219 +            unsigned long long start = time();
   2.220 +            unsigned long long elapsed;
   2.221 +            int ops = 0;
   2.222 +
   2.223 +            do {
   2.224 +                int i = random() % (key + 1);
   2.225 +                assert(i <= key);
   2.226 +
   2.227 +                stringlist_t* keylist = NULL;
   2.228 +                status = find_keys(session, ids[i]->address, &keylist);
   2.229 +                free_stringlist(keylist);
   2.230 +                TEST_ASSERT_MSG(status == PEP_STATUS_OK,
   2.231 +                                tl_status_string(status));
   2.232 +                ops ++;
   2.233 +                elapsed = time() - start;
   2.234 +            } while (elapsed < TIME);
   2.235 +
   2.236 +            find_keys_stats->benchmarks[benchmark].ops[iter] = ops;
   2.237 +            find_keys_stats->benchmarks[benchmark].time[iter] = elapsed;
   2.238 +        }
   2.239 +
   2.240 +        dump_stats(find_keys_stats, benchmark);
   2.241 +
   2.242 +
   2.243 +        // Look up a random key by its pep user id.
   2.244 +        //
   2.245 +        // This uses the engine's management databank and doesn't
   2.246 +        // touch the pgp engine's DB.
   2.247 +        get_identity_stats->benchmarks[benchmark].keys = key + 1;
   2.248 +        for (int iter = 0; iter < REPITIONS; iter ++) {
   2.249 +            unsigned long long start = time();
   2.250 +            unsigned long long elapsed;
   2.251 +            int ops = 0;
   2.252 +
   2.253 +            do {
   2.254 +                int i = random() % (key + 1);
   2.255 +                pEp_identity *id = NULL;
   2.256 +                status = get_identity(session, ids[i]->address,
   2.257 +                                      ids[i]->user_id, &id);
   2.258 +                TEST_ASSERT_MSG(status == PEP_STATUS_OK,
   2.259 +                                tl_status_string(status));
   2.260 +                TEST_ASSERT_MSG(id->fpr, "no fingerprint");
   2.261 +                TEST_ASSERT_MSG(strcmp(ids[i]->fpr, id->fpr) == 0,
   2.262 +                                "fingerprint mismatch");
   2.263 +                free_identity(id);
   2.264 +
   2.265 +                ops ++;
   2.266 +                elapsed = time() - start;
   2.267 +            } while (elapsed < TIME);
   2.268 +
   2.269 +            get_identity_stats->benchmarks[benchmark].ops[iter] = ops;
   2.270 +            get_identity_stats->benchmarks[benchmark].time[iter] = elapsed;
   2.271 +        }
   2.272 +
   2.273 +        dump_stats(get_identity_stats, benchmark);
   2.274 +
   2.275 +
   2.276 +        benchmark++;
   2.277 +    }
   2.278 +}