ENGINE-524: merge in sync ENGINE-524
authorClaudio Luck <claudio.luck@pep.foundation>
Thu, 13 Jun 2019 17:25:55 +0200
branchENGINE-524
changeset 3841f9d1e79ae49b
parent 3840 bfef390b48e2
parent 3831 44e274f79ec1
ENGINE-524: merge in sync
src/pEpEngine.c
src/pgp_sequoia.c
     1.1 --- a/src/baseprotocol.c	Sat Jun 01 13:18:11 2019 +0200
     1.2 +++ b/src/baseprotocol.c	Thu Jun 13 17:25:55 2019 +0200
     1.3 @@ -164,7 +164,7 @@
     1.4      char *_fpr = NULL;
     1.5      if (_sign) {
     1.6          status = verify_text(session, _payload, _payload_size, _sign, _sign_size, &keylist);
     1.7 -        if (status != PEP_VERIFIED || !keylist || !keylist->value) {
     1.8 +        if (!(status == PEP_VERIFIED || status == PEP_VERIFIED_AND_TRUSTED) || !keylist || !keylist->value) {
     1.9              // signature invalid or does not match; ignore sync message
    1.10              status = PEP_STATUS_OK;
    1.11              goto the_end;
    1.12 @@ -187,4 +187,3 @@
    1.13      free_stringlist(keylist);
    1.14      return status;
    1.15  }
    1.16 -
     2.1 --- a/src/pEpEngine.c	Sat Jun 01 13:18:11 2019 +0200
     2.2 +++ b/src/pEpEngine.c	Thu Jun 13 17:25:55 2019 +0200
     2.3 @@ -981,6 +981,7 @@
     2.4                  "   timestamp integer default (datetime('now')),\n"
     2.5                  "   primary key (address, user_id)\n"
     2.6                  ");\n"
     2.7 +                "create index if not exists identity_userid_addr on identity(address, user_id);\n"
     2.8                  "create table if not exists trust (\n"
     2.9                  "   user_id text not null\n"
    2.10                  "       references person (id)\n"
    2.11 @@ -1436,6 +1437,17 @@
    2.12                  if (status != PEP_STATUS_OK)
    2.13                      return status;
    2.14              }
    2.15 +            if (version < 12) {
    2.16 +                int_result = sqlite3_exec(
    2.17 +                    _session->db,
    2.18 +                    "create index if not exists identity_userid_addr on identity(address, user_id);\n"
    2.19 +                    ,
    2.20 +                    NULL,
    2.21 +                    NULL,
    2.22 +                    NULL
    2.23 +                );
    2.24 +                assert(int_result == SQLITE_OK);                
    2.25 +            }
    2.26          }        
    2.27          else { 
    2.28              // Version from DB was 0, it means this is initial setup.
     3.1 --- a/src/pgp_sequoia.c	Sat Jun 01 13:18:11 2019 +0200
     3.2 +++ b/src/pgp_sequoia.c	Thu Jun 13 17:25:55 2019 +0200
     3.3 @@ -844,15 +844,12 @@
     3.4  
     3.5      free(email);
     3.6      free(name);
     3.7 -    if (user_id_iter)
     3.8 -        pgp_user_id_binding_iter_free(user_id_iter);
     3.9 -    if (key_iter)
    3.10 -        pgp_tpk_key_iter_free(key_iter);
    3.11 +    pgp_user_id_binding_iter_free(user_id_iter);
    3.12 +    pgp_tpk_key_iter_free(key_iter);
    3.13      if (stmt)
    3.14        sqlite3_reset(stmt);
    3.15      free(tsk_buffer);
    3.16 -    if (tpk)
    3.17 -        pgp_tpk_free(tpk);
    3.18 +    pgp_tpk_free(tpk);
    3.19      free(fpr);
    3.20      pgp_fingerprint_free(pgp_fpr);
    3.21  
    3.22 @@ -1004,13 +1001,10 @@
    3.23          cookie->decrypted = 1;
    3.24  
    3.25      eol:
    3.26 -        if (sk)
    3.27 -            pgp_session_key_free (sk);
    3.28 +        pgp_session_key_free (sk);
    3.29          free(keyid_str);
    3.30 -        if (key_iter)
    3.31 -            pgp_tpk_key_iter_free(key_iter);
    3.32 -        if (tpk)
    3.33 -            pgp_tpk_free(tpk);
    3.34 +        pgp_tpk_key_iter_free(key_iter);
    3.35 +        pgp_tpk_free(tpk);
    3.36      }
    3.37  
    3.38      // Consider wildcard recipients.
    3.39 @@ -1083,11 +1077,9 @@
    3.40              key_iter = NULL;
    3.41          }
    3.42      eol2:
    3.43 -        if (sk)
    3.44 -            pgp_session_key_free (sk);
    3.45 +        pgp_session_key_free (sk);
    3.46          free(keyid_str);
    3.47 -        if (key_iter)
    3.48 -            pgp_tpk_key_iter_free(key_iter);
    3.49 +        pgp_tpk_key_iter_free(key_iter);
    3.50      }
    3.51  
    3.52      if (tsks) {
    3.53 @@ -1421,12 +1413,9 @@
    3.54          free(*ptext);
    3.55      }
    3.56  
    3.57 -    if (reader)
    3.58 -        pgp_reader_free(reader);
    3.59 -    if (decryptor)
    3.60 -        pgp_reader_free(decryptor);
    3.61 -    if (writer)
    3.62 -        pgp_writer_free(writer);
    3.63 +    pgp_reader_free(reader);
    3.64 +    pgp_reader_free(decryptor);
    3.65 +    pgp_writer_free(writer);
    3.66  
    3.67      T("-> %s", pEp_status_to_string(status));
    3.68      return status;
    3.69 @@ -1520,12 +1509,9 @@
    3.70          free_stringlist(cookie.signer_keylist);
    3.71      }
    3.72  
    3.73 -    if (verifier)
    3.74 -        pgp_reader_free(verifier);
    3.75 -    if (reader)
    3.76 -        pgp_reader_free(reader);
    3.77 -    if (dsig_reader)
    3.78 -        pgp_reader_free(dsig_reader);
    3.79 +    pgp_reader_free(verifier);
    3.80 +    pgp_reader_free(reader);
    3.81 +    pgp_reader_free(dsig_reader);
    3.82  
    3.83      T("-> %s", pEp_status_to_string(status));
    3.84      return status;
    3.85 @@ -1607,14 +1593,10 @@
    3.86      (*stext)[*ssize] = 0;
    3.87  
    3.88   out:
    3.89 -    if (signer)
    3.90 -        pgp_signer_free (signer);
    3.91 -    if (signing_keypair)
    3.92 -        pgp_key_pair_free (signing_keypair);
    3.93 -    if (iter)
    3.94 -        pgp_tpk_key_iter_free (iter);
    3.95 -    if (signer_tpk)
    3.96 -        pgp_tpk_free(signer_tpk);
    3.97 +    pgp_signer_free (signer);
    3.98 +    pgp_key_pair_free (signing_keypair);
    3.99 +    pgp_tpk_key_iter_free (iter);
   3.100 +    pgp_tpk_free(signer_tpk);
   3.101  
   3.102      T("(%s)-> %s", fpr, pEp_status_to_string(status));
   3.103      return status;
   3.104 @@ -1728,14 +1710,10 @@
   3.105      (*ctext)[*csize] = 0;
   3.106  
   3.107   out:
   3.108 -    if (signer)
   3.109 -        pgp_signer_free (signer);
   3.110 -    if (signing_keypair)
   3.111 -        pgp_key_pair_free (signing_keypair);
   3.112 -    if (iter)
   3.113 -        pgp_tpk_key_iter_free (iter);
   3.114 -    if (signer_tpk)
   3.115 -        pgp_tpk_free(signer_tpk);
   3.116 +    pgp_signer_free (signer);
   3.117 +    pgp_key_pair_free (signing_keypair);
   3.118 +    pgp_tpk_key_iter_free (iter);
   3.119 +    pgp_tpk_free(signer_tpk);
   3.120  
   3.121      for (int i = 0; i < keys_count; i ++)
   3.122          pgp_tpk_free(keys[i]);
   3.123 @@ -1787,7 +1765,7 @@
   3.124      size_t userid_len = 0;
   3.125      const uint8_t *raw = pgp_user_id_value(userid_packet, &userid_len);
   3.126  
   3.127 -    // Null terminate it.
   3.128 +    // NUL terminate it.
   3.129      userid = malloc(userid_len + 1);
   3.130      if (!userid)
   3.131          ERROR_OUT(NULL, PEP_OUT_OF_MEMORY, "out of memory");
   3.132 @@ -1821,14 +1799,11 @@
   3.133      fpr = NULL;
   3.134  
   3.135   out:
   3.136 -    if (pgp_fpr)
   3.137 -        pgp_fingerprint_free(pgp_fpr);
   3.138 +    pgp_fingerprint_free(pgp_fpr);
   3.139      free(fpr);
   3.140 -    if (tpk)
   3.141 -        pgp_tpk_free(tpk);
   3.142 +    pgp_tpk_free(tpk);
   3.143      free(userid);
   3.144 -    if (userid_packet)
   3.145 -        pgp_packet_free(userid_packet);
   3.146 +    pgp_packet_free(userid_packet);
   3.147  
   3.148      T("-> %s", pEp_status_to_string(status));
   3.149      return status;
   3.150 @@ -1982,8 +1957,7 @@
   3.151      }
   3.152  
   3.153   out:
   3.154 -    if (parser)
   3.155 -        pgp_tpk_parser_free(parser);
   3.156 +    pgp_tpk_parser_free(parser);
   3.157  
   3.158      T("-> %s", pEp_status_to_string(status));
   3.159      return status;
   3.160 @@ -2055,12 +2029,12 @@
   3.161      return status;
   3.162  }
   3.163  
   3.164 -char* _undot_address(const char* address) {
   3.165 +static char *_undot_address(const char* address) {
   3.166      if (!address)
   3.167          return NULL;
   3.168  
   3.169      int addr_len = strlen(address);
   3.170 -    const char* at = strstr(address, "@");
   3.171 +    const char* at = memchr(address, '@', addr_len);
   3.172  
   3.173      if (!at)
   3.174          at = address + addr_len;
   3.175 @@ -2209,10 +2183,8 @@
   3.176      }
   3.177  
   3.178   out:
   3.179 -    if (tpk)
   3.180 -        pgp_tpk_free(tpk);
   3.181 -    if (fpr)
   3.182 -        pgp_fingerprint_free(fpr);
   3.183 +    pgp_tpk_free(tpk);
   3.184 +    pgp_fingerprint_free(fpr);
   3.185  
   3.186      if (status == PEP_KEY_NOT_FOUND)
   3.187          status = PEP_STATUS_OK;
   3.188 @@ -2357,8 +2329,7 @@
   3.189      }
   3.190  
   3.191   out:
   3.192 -    if (tpk)
   3.193 -        pgp_tpk_free(tpk);
   3.194 +    pgp_tpk_free(tpk);
   3.195  
   3.196      T("(%s) -> %s", fpr, pEp_comm_type_to_string(*comm_type));
   3.197      return status;
   3.198 @@ -2418,14 +2389,10 @@
   3.199      ERROR_OUT(NULL, status, "Saving %s", fpr);
   3.200  
   3.201   out:
   3.202 -    if (signer)
   3.203 -        pgp_signer_free (signer);
   3.204 -    if (keypair)
   3.205 -        pgp_key_pair_free (keypair);
   3.206 -    if (iter)
   3.207 -        pgp_tpk_key_iter_free (iter);
   3.208 -    if (tpk)
   3.209 -        pgp_tpk_free(tpk);
   3.210 +    pgp_signer_free (signer);
   3.211 +    pgp_key_pair_free (keypair);
   3.212 +    pgp_tpk_key_iter_free (iter);
   3.213 +    pgp_tpk_free(tpk);
   3.214  
   3.215      T("(%s) -> %s", fpr, pEp_status_to_string(status));
   3.216      return status;
   3.217 @@ -2479,14 +2446,10 @@
   3.218      ERROR_OUT(NULL, status, "Saving %s", fpr);
   3.219  
   3.220   out:
   3.221 -    if (signer)
   3.222 -        pgp_signer_free (signer);
   3.223 -    if (keypair)
   3.224 -        pgp_key_pair_free (keypair);
   3.225 -    if (iter)
   3.226 -        pgp_tpk_key_iter_free (iter);
   3.227 -    if (tpk)
   3.228 -        pgp_tpk_free(tpk);
   3.229 +    pgp_signer_free (signer);
   3.230 +    pgp_key_pair_free (keypair);
   3.231 +    pgp_tpk_key_iter_free (iter);
   3.232 +    pgp_tpk_free(tpk);
   3.233  
   3.234      T("(%s) -> %s", fpr, pEp_status_to_string(status));
   3.235      return status;
   3.236 @@ -2543,8 +2506,7 @@
   3.237      *expired = !(can_encrypt && can_sign && can_certify);
   3.238  
   3.239   out:
   3.240 -    if (tpk)
   3.241 -        pgp_tpk_free(tpk);
   3.242 +    pgp_tpk_free(tpk);
   3.243      T("(%s) -> %s", fpr, pEp_status_to_string(status));
   3.244      return status;
   3.245  }
     4.1 --- a/sync/cond_act_sync.yml2	Sat Jun 01 13:18:11 2019 +0200
     4.2 +++ b/sync/cond_act_sync.yml2	Thu Jun 13 17:25:55 2019 +0200
     4.3 @@ -18,7 +18,7 @@
     4.4      |> *result = _result > 0;
     4.5  }
     4.6  
     4.7 -condition weAreFirst
     4.8 +condition weAreOfferer
     4.9  ||
    4.10      TID_t *t1 = &session->sync_state.keysync.challenge;
    4.11      TID_t *t2 = &session->sync_state.own.challenge;
    4.12 @@ -416,7 +416,7 @@
    4.13      free_identity_list(il);
    4.14  ||
    4.15  
    4.16 -action ownKeysAreGroupKeys
    4.17 +action ownKeysAreDefaultKeys
    4.18  ||
    4.19      PEP_STATUS status = PEP_STATUS_OK;
    4.20  
    4.21 @@ -430,7 +430,7 @@
    4.22      }
    4.23  ||
    4.24  
    4.25 -action receivedKeysAreGroupKeys
    4.26 +action receivedKeysAreDefaultKeys
    4.27  ||
    4.28      PEP_STATUS status = PEP_STATUS_OK;
    4.29      // set flag for all keys
     5.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.2 +++ b/sync/gen_doc.yml2	Thu Jun 13 17:25:55 2019 +0200
     5.3 @@ -0,0 +1,44 @@
     5.4 +include yslt.yml2
     5.5 +
     5.6 +tstylesheet {
     5.7 +    template "/protocol" {
     5.8 +        | protocol «@name»
     5.9 +        apply "fsm" sort "@name";
    5.10 +    }
    5.11 +    template "fsm" {
    5.12 +        | fsm «@name»
    5.13 +        apply "state" sort "@name";
    5.14 +        |
    5.15 +        for "set:distinct(*/event/@name)" {
    5.16 +            sort ".";
    5.17 +            |> event «.»
    5.18 +        }
    5.19 +        |
    5.20 +        for "set:distinct(descendant::action/@name)" {
    5.21 +            sort ".";
    5.22 +            |> action «.»
    5.23 +        }
    5.24 +        |
    5.25 +        for "set:distinct(descendant::condition/@name)" {
    5.26 +            sort ".";
    5.27 +            |> condition «.»
    5.28 +        }
    5.29 +        |
    5.30 +        apply "descendant::transition"
    5.31 +            sort "ancestor::state/@name",
    5.32 +            sort "@target",
    5.33 +            sort "ancestor::event/@name";
    5.34 +        |
    5.35 +        apply "message" sort "@name";
    5.36 +        |
    5.37 +        for "set:distinct(message/field/@name)" {
    5.38 +            sort ".";
    5.39 +            |> field «.»
    5.40 +        }
    5.41 +    }
    5.42 +    template "state" | state «@name»
    5.43 +    template "transition"
    5.44 +        | transition from «ancestor::state/@name» to «@target» on event «ancestor::event/@name»
    5.45 +    template "message" | message «@name»
    5.46 +}
    5.47 +
     6.1 --- a/sync/gen_dot.ysl2	Sat Jun 01 13:18:11 2019 +0200
     6.2 +++ b/sync/gen_dot.ysl2	Thu Jun 13 17:25:55 2019 +0200
     6.3 @@ -8,34 +8,24 @@
     6.4      ||
     6.5      digraph finite_state_machine {
     6.6          rankdir=LR;
     6.7 -        node [shape = doublecircle];
     6.8 -        `apply "state[@end='1']" mode=end`;
     6.9 -        node [shape = circle];
    6.10 +        node [shape=point];
    6.11 +        InitState;
    6.12 +        node [shape=doublecircle];
    6.13 +        End;
    6.14 +        node [shape=circle, style=filled, fillcolor=lightgrey];
    6.15  
    6.16 -        `` apply "state" mode=do
    6.17 +        `` apply "descendant::transition";
    6.18      }
    6.19      ||
    6.20  
    6.21 -    template "state", mode=end > «@name» 
    6.22 -    template "state", mode=do apply "event", 0 with "state", "@name";
    6.23 -
    6.24 -    template "event" {
    6.25 -        param "state";
    6.26 +    template "transition" {
    6.27 +        const "event", "ancestor::event/@name";
    6.28          choose {
    6.29 -            when "count(transition) > 0"
    6.30 -                apply "transition|condition/transition", 0
    6.31 -                    with "state", "$state", with "event", "@name";
    6.32 +            when "ancestor::fsm/message[@name=$event]"
    6.33 +                | «ancestor::state/@name» -> «@target» [ label="«ancestor::event/@name»", color=blue ];
    6.34              otherwise
    6.35 -                if "@name != 'Init'"
    6.36 -                    | «$state» -> «$state» [ label="«@name»" ];
    6.37 +                | «ancestor::state/@name» -> «@target» [ label="«ancestor::event/@name»", color=red ];
    6.38          }
    6.39      }
    6.40 -
    6.41 -    template "transition" {
    6.42 -        param "state";
    6.43 -        param "event";
    6.44 -
    6.45 -        | «$state» -> «@target» [ label ="«$event»" ];
    6.46 -    }
    6.47  }
    6.48  
     7.1 --- a/sync/sync.fsm	Sat Jun 01 13:18:11 2019 +0200
     7.2 +++ b/sync/sync.fsm	Thu Jun 13 17:25:55 2019 +0200
     7.3 @@ -41,14 +41,14 @@
     7.4                      // this is our own Beacon; ignore
     7.5                  }
     7.6                  else {
     7.7 -                    if weAreFirst {
     7.8 +                    if weAreOfferer {
     7.9                          do useOwnChallenge;
    7.10                          send Beacon;
    7.11                      }
    7.12 -                    else /* we are second */ {
    7.13 +                    else /* we are requester */ {
    7.14                          do openNegotiation;
    7.15                          do tellWeAreNotGrouped;
    7.16 -                        // second is sending NegotiationRequest
    7.17 +                        // requester is sending NegotiationRequest
    7.18                          send NegotiationRequest;
    7.19                          do useOwnChallenge;
    7.20                      }
    7.21 @@ -62,25 +62,25 @@
    7.22                      }
    7.23                      else {
    7.24                          do storeNegotiation;
    7.25 -                        // first is sending NegotiationOpen
    7.26 +                        // offerer is accepting by confirming NegotiationOpen
    7.27                          send NegotiationOpen;
    7.28                          if partnerIsGrouped
    7.29 -                            go HandshakingWithGroup;
    7.30 +                            go HandshakingToJoin;
    7.31                          else
    7.32 -                            go HandshakingNewFirst;
    7.33 +                            go HandshakingOfferer;
    7.34                      }
    7.35                  }
    7.36              }
    7.37  
    7.38              on NegotiationOpen if sameNegotiationAndPartner {
    7.39 -                // second is receiving NegotiationOpen
    7.40 +                // requester is receiving NegotiationOpen
    7.41                  do storeNegotiation;
    7.42 -                go HandshakingNewSecond;
    7.43 +                go HandshakingRequester;
    7.44              }
    7.45          }
    7.46  
    7.47          // handshaking without existing Device group
    7.48 -        state HandshakingNewFirst timeout=600 {
    7.49 +        state HandshakingOfferer timeout=600 {
    7.50              on Init
    7.51                  do showSoleHandshake;
    7.52  
    7.53 @@ -108,17 +108,17 @@
    7.54              // Accept means init Phase1Commit
    7.55              on Accept {
    7.56                  do trustThisKey;
    7.57 -                send CommitAcceptFirst;
    7.58 -                go HandshakingNewPhase1First;
    7.59 +                send CommitAcceptOfferer;
    7.60 +                go HandshakingPhase1Offerer;
    7.61              }
    7.62  
    7.63 -            // got a CommitAccept from second
    7.64 -            on CommitAcceptSecond if sameNegotiationAndPartner
    7.65 -                go HandshakingNewPhase2First;
    7.66 +            // got a CommitAccept from requester
    7.67 +            on CommitAcceptRequester if sameNegotiationAndPartner
    7.68 +                go HandshakingPhase2Offerer;
    7.69          }
    7.70  
    7.71          // handshaking without existing Device group
    7.72 -        state HandshakingNewSecond timeout=600 {
    7.73 +        state HandshakingRequester timeout=600 {
    7.74              on Init
    7.75                  do showSoleHandshake;
    7.76  
    7.77 @@ -146,16 +146,16 @@
    7.78              // Accept means init Phase1Commit
    7.79              on Accept {
    7.80                  do trustThisKey;
    7.81 -                send CommitAcceptSecond;
    7.82 -                go HandshakingNewPhase1Second;
    7.83 +                send CommitAcceptRequester;
    7.84 +                go HandshakingPhase1Requester;
    7.85              }
    7.86  
    7.87 -            // got a CommitAccept from first
    7.88 -            on CommitAcceptFirst if sameNegotiationAndPartner
    7.89 -                go HandshakingNewPhase2Second;
    7.90 +            // got a CommitAccept from offerer
    7.91 +            on CommitAcceptOfferer if sameNegotiationAndPartner
    7.92 +                go HandshakingPhase2Requester;
    7.93          }
    7.94  
    7.95 -        state HandshakingNewPhase1First {
    7.96 +        state HandshakingPhase1Offerer {
    7.97              on Rollback if sameNegotiationAndPartner {
    7.98                  do untrustThisKey;
    7.99                  go Sole;
   7.100 @@ -167,12 +167,12 @@
   7.101                  go End;
   7.102              }
   7.103  
   7.104 -            on CommitAcceptSecond if sameNegotiationAndPartner {
   7.105 -                go NewGroupFirst;
   7.106 +            on CommitAcceptRequester if sameNegotiationAndPartner {
   7.107 +                go FormingGroupOfferer;
   7.108              }
   7.109          }
   7.110  
   7.111 -        state HandshakingNewPhase1Second {
   7.112 +        state HandshakingPhase1Requester {
   7.113              on Rollback if sameNegotiationAndPartner {
   7.114                  do untrustThisKey;
   7.115                  go Sole;
   7.116 @@ -184,12 +184,12 @@
   7.117                  go End;
   7.118              }
   7.119  
   7.120 -            on CommitAcceptFirst if sameNegotiationAndPartner {
   7.121 -                go NewGroupSecond;
   7.122 +            on CommitAcceptOfferer if sameNegotiationAndPartner {
   7.123 +                go FormingGroupRequester;
   7.124              }
   7.125          }
   7.126  
   7.127 -        state HandshakingNewPhase2First {
   7.128 +        state HandshakingPhase2Offerer {
   7.129              on Cancel {
   7.130                  send Rollback;
   7.131                  go Sole;
   7.132 @@ -202,13 +202,13 @@
   7.133              }
   7.134  
   7.135              on Accept {
   7.136 -                send CommitAcceptFirst;
   7.137 +                send CommitAcceptOfferer;
   7.138                  do trustThisKey;
   7.139 -                go NewGroupFirst;
   7.140 +                go FormingGroupOfferer;
   7.141              }
   7.142          }
   7.143  
   7.144 -        state HandshakingNewPhase2Second {
   7.145 +        state HandshakingPhase2Requester {
   7.146              on Cancel {
   7.147                  send Rollback;
   7.148                  go Sole;
   7.149 @@ -221,36 +221,35 @@
   7.150              }
   7.151  
   7.152              on Accept {
   7.153 -                send CommitAcceptSecond;
   7.154 +                send CommitAcceptRequester;
   7.155                  do trustThisKey;
   7.156 -                go NewGroupSecond;
   7.157 +                go FormingGroupRequester;
   7.158              }
   7.159          }
   7.160  
   7.161 -        state NewGroupFirst {
   7.162 +        state FormingGroupOfferer {
   7.163              on Init {
   7.164                  do prepareOwnKeys;
   7.165 -                send OwnKeysFirst; // we're not grouped yet, this is our own keys
   7.166 +                send OwnKeysOfferer; // we're not grouped yet, this is our own keys
   7.167              }
   7.168  
   7.169 -            on OwnKeysSecond {
   7.170 +            on OwnKeysRequester {
   7.171                  do saveGroupKeys;
   7.172 -                do receivedKeysAreGroupKeys;
   7.173 +                do receivedKeysAreDefaultKeys;
   7.174                  do showGroupCreated;
   7.175                  go Grouped;
   7.176              }
   7.177          }
   7.178  
   7.179 -        state NewGroupSecond {
   7.180 +        state FormingGroupRequester {
   7.181              on Init {
   7.182                  do prepareOwnKeys;
   7.183 -                send OwnKeysSecond; // we're not grouped yet, this is our own keys
   7.184 +                send OwnKeysRequester; // we're not grouped yet, this is our own keys
   7.185              }
   7.186  
   7.187 -            on OwnKeysFirst {
   7.188 +            on OwnKeysOfferer {
   7.189                  do saveGroupKeys;
   7.190 -
   7.191 -                do ownKeysAreGroupKeys;
   7.192 +                do ownKeysAreDefaultKeys;
   7.193                  do showGroupCreated;
   7.194                  go Grouped;
   7.195              }
   7.196 @@ -282,13 +281,12 @@
   7.197                  go HandshakingGrouped;
   7.198              }
   7.199  
   7.200 -            on GroupTrustThisKey {
   7.201 +            on GroupTrustThisKey
   7.202                  do trustThisKey;
   7.203 -            }
   7.204          }
   7.205  
   7.206          // sole device handshaking with group
   7.207 -        state HandshakingWithGroup {
   7.208 +        state HandshakingToJoin {
   7.209              on Init
   7.210                  do showJoinGroupHandshake;
   7.211  
   7.212 @@ -308,6 +306,9 @@
   7.213                  go End;
   7.214              }
   7.215  
   7.216 +            on CommitAcceptForGroup if sameNegotiationAndPartner
   7.217 +                go HandshakingToJoinPhase2;
   7.218 +
   7.219              on CommitReject if sameNegotiationAndPartner {
   7.220                  do disable;
   7.221                  go End;
   7.222 @@ -317,11 +318,11 @@
   7.223              on Accept {
   7.224                  do trustThisKey;
   7.225                  send CommitAccept;
   7.226 -                go HandshakingJoinPhase1;
   7.227 +                go HandshakingToJoinPhase1;
   7.228              }
   7.229          }
   7.230  
   7.231 -        state HandshakingJoinPhase1 {
   7.232 +        state HandshakingToJoinPhase1 {
   7.233              on Rollback if sameNegotiationAndPartner
   7.234                  go Sole;
   7.235              
   7.236 @@ -330,12 +331,11 @@
   7.237                  go End;
   7.238              }
   7.239  
   7.240 -            on CommitAcceptForGroup if sameNegotiationAndPartner {
   7.241 -                go JoinGroup;
   7.242 -            }
   7.243 +            on CommitAcceptForGroup if sameNegotiationAndPartner
   7.244 +                go JoiningGroup;
   7.245          }
   7.246  
   7.247 -        state HandshakingJoinPhase2 {
   7.248 +        state HandshakingToJoinPhase2 {
   7.249              on Cancel {
   7.250                  send Rollback;
   7.251                  go Sole;
   7.252 @@ -349,19 +349,16 @@
   7.253  
   7.254              on Accept {
   7.255                  do trustThisKey;
   7.256 -                go JoinGroup;
   7.257 +                go JoiningGroup;
   7.258              }
   7.259          }
   7.260  
   7.261 -        state JoinGroup {
   7.262 -            on Init {
   7.263 -                do prepareOwnKeys;
   7.264 -                send OwnKeys;
   7.265 -            }
   7.266 -
   7.267 +        state JoiningGroup {
   7.268              on GroupKeys {
   7.269                  do saveGroupKeys;
   7.270 -                do receivedKeysAreGroupKeys;
   7.271 +                do receivedKeysAreDefaultKeys;
   7.272 +                do prepareOwnKeys;
   7.273 +                send GroupKeys;
   7.274                  do showDeviceAdded;
   7.275                  go Grouped;
   7.276              }
   7.277 @@ -405,14 +402,6 @@
   7.278                  do trustThisKey;
   7.279              }
   7.280  
   7.281 -            on CommitAcceptForGroup {
   7.282 -                do showDeviceAdded;
   7.283 -                if sameNegotiationAndPartner {
   7.284 -                    do hideHandshakeDialog;
   7.285 -                    go Grouped;
   7.286 -                }
   7.287 -            }
   7.288 -
   7.289              on GroupKeys
   7.290                  do saveGroupKeys;
   7.291          }
   7.292 @@ -496,11 +485,11 @@
   7.293              field TID negotiation;
   7.294          }
   7.295  
   7.296 -        message CommitAcceptFirst 7, security=untrusted {
   7.297 +        message CommitAcceptOfferer 7, security=untrusted {
   7.298              field TID negotiation;
   7.299          }
   7.300  
   7.301 -        message CommitAcceptSecond 8, security=untrusted {
   7.302 +        message CommitAcceptRequester 8, security=untrusted {
   7.303              field TID negotiation;
   7.304          }
   7.305  
   7.306 @@ -522,15 +511,11 @@
   7.307              field IdentityList ownIdentities;
   7.308          }
   7.309  
   7.310 -        message OwnKeys 13, security=attach_own_keys {
   7.311 +        message OwnKeysOfferer 13, security=attach_own_keys {
   7.312              field IdentityList ownIdentities;
   7.313          }
   7.314  
   7.315 -        message OwnKeysFirst 14, security=attach_own_keys {
   7.316 -            field IdentityList ownIdentities;
   7.317 -        }
   7.318 -
   7.319 -        message OwnKeysSecond 15, security=attach_own_keys {
   7.320 +        message OwnKeysRequester 14, security=attach_own_keys {
   7.321              field IdentityList ownIdentities;
   7.322          }
   7.323      }
     8.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.2 +++ b/test/include/LotsOfKeysTests.h	Thu Jun 13 17:25:55 2019 +0200
     8.3 @@ -0,0 +1,19 @@
     8.4 +// This file is under GNU General Public License 3.0
     8.5 +// see LICENSE.txt
     8.6 +
     8.7 +#ifndef LOTS_OF_KEYS_H
     8.8 +#define LOTS_OF_KEYS_H
     8.9 +
    8.10 +#include <string>
    8.11 +#include "EngineTestIndividualSuite.h"
    8.12 +
    8.13 +using namespace std;
    8.14 +
    8.15 +class LotsOfKeysTests : public EngineTestIndividualSuite {
    8.16 +    public:
    8.17 +        LotsOfKeysTests(string test_suite, string test_home_dir);
    8.18 +    private:
    8.19 +        void check();
    8.20 +};
    8.21 +
    8.22 +#endif
     9.1 --- a/test/src/SuiteMaker.cc	Sat Jun 01 13:18:11 2019 +0200
     9.2 +++ b/test/src/SuiteMaker.cc	Thu Jun 13 17:25:55 2019 +0200
     9.3 @@ -39,6 +39,7 @@
     9.4  #include "KeyringImportTests.h"
     9.5  #include "EnterLeaveDeviceGroupTests.h"
     9.6  #include "SignOnlyTests.h"
     9.7 +#include "LotsOfKeysTests.h"
     9.8  #include "BCCTests.h"
     9.9  #include "Engine358Tests.h"
    9.10  #include "BlacklistAcceptNewKeyTests.h"
    9.11 @@ -102,6 +103,7 @@
    9.12      "KeyringImportTests",
    9.13      "EnterLeaveDeviceGroupTests",
    9.14      "SignOnlyTests",
    9.15 +    "LotsOfKeysTests",
    9.16      "BCCTests",
    9.17      "Engine358Tests",
    9.18      "BlacklistAcceptNewKeyTests",
    9.19 @@ -138,7 +140,7 @@
    9.20  };
    9.21  
    9.22  // This file is generated, so magic constants are ok.
    9.23 -int SuiteMaker::num_suites = 60;
    9.24 +int SuiteMaker::num_suites = 61;
    9.25  
    9.26  void SuiteMaker::suitemaker_build(const char* test_class_name, const char* test_home, Test::Suite** test_suite) {
    9.27      if (strcmp(test_class_name, "URIAddressTests") == 0)
    9.28 @@ -195,6 +197,8 @@
    9.29          *test_suite = new EnterLeaveDeviceGroupTests(test_class_name, test_home);
    9.30      else if (strcmp(test_class_name, "SignOnlyTests") == 0)
    9.31          *test_suite = new SignOnlyTests(test_class_name, test_home);
    9.32 +    else if (strcmp(test_class_name, "LotsOfKeysTests") == 0)
    9.33 +        *test_suite = new LotsOfKeysTests(test_class_name, test_home);
    9.34      else if (strcmp(test_class_name, "BCCTests") == 0)
    9.35          *test_suite = new BCCTests(test_class_name, test_home);
    9.36      else if (strcmp(test_class_name, "Engine358Tests") == 0)
    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/test/src/engine_tests/LotsOfKeysTests.cc	Thu Jun 13 17:25:55 2019 +0200
    10.3 @@ -0,0 +1,275 @@
    10.4 +// This file is under GNU General Public License 3.0
    10.5 +// see LICENSE.txt
    10.6 +
    10.7 +#include <stdlib.h>
    10.8 +#include <string>
    10.9 +#include <cstring>
   10.10 +#include <cpptest.h>
   10.11 +#include <fstream>
   10.12 +#include <sys/time.h>
   10.13 +#include <assert.h>
   10.14 +#include <sys/types.h>
   10.15 +#include <unistd.h>
   10.16 +#include <time.h>
   10.17 +
   10.18 +#include "pEpEngine.h"
   10.19 +
   10.20 +#include "test_util.h"
   10.21 +#include "EngineTestIndividualSuite.h"
   10.22 +#include "LotsOfKeysTests.h"
   10.23 +
   10.24 +// Whether to log to the file system (in /tmp).
   10.25 +#define LOG 0
   10.26 +// How to plot (in R).  You need to enable LOG above.
   10.27 +#if 0
   10.28 +
   10.29 +library(ggplot2);
   10.30 +
   10.31 +t = "2019XXXX-XXXXXX";
   10.32 +find_keys = read.csv(sprintf("/tmp/pep-benchmark-find-keys-%s.csv", t),
   10.33 +                     header=T, sep=",")
   10.34 +get_ids = read.csv(sprintf("/tmp/pep-benchmark-get-identity-%s.csv", t),
   10.35 +                   header=T, sep=",")
   10.36 +ggplot(find_keys, aes(x=Keys, y=run0_us_per_op), tag="FOO") +
   10.37 +    geom_point() +
   10.38 +    geom_point(data = get_ids, aes(Keys, run0_us_per_op), colour = 'red') +
   10.39 +    scale_x_log10() + guides(fill = guide_legend()) +
   10.40 +    labs(title="Microseconds/Op for a Key/Management DB with x keys",
   10.41 +         x="Keys in DB", y="usec")
   10.42 +
   10.43 +ggsave("/tmp/pep-benchmark.pdf", width = 4, height = 4)
   10.44 +#endif
   10.45 +
   10.46 +
   10.47 +using namespace std;
   10.48 +
   10.49 +LotsOfKeysTests::LotsOfKeysTests(string suitename, string test_home_dir) :
   10.50 +    EngineTestIndividualSuite::EngineTestIndividualSuite(suitename, test_home_dir) {
   10.51 +    add_test_to_suite(std::pair<std::string, void (Test::Suite::*)()>(string("LotsOfKeysTests::check"),
   10.52 +                                                                      static_cast<Func>(&LotsOfKeysTests::check)));
   10.53 +}
   10.54 +
   10.55 +// Only really run the bench mark if logging is enabled.
   10.56 +#if LOG > 0
   10.57 +// Maximum number of keys.
   10.58 +#  define N 31622
   10.59 +// Amount of time to run each benchmark, in usecs.
   10.60 +#  define TIME 2 * 1000000
   10.61 +#else
   10.62 +// Don't actually run the benchmark.
   10.63 +#  define N 3
   10.64 +#  define TIME 0
   10.65 +#endif
   10.66 +
   10.67 +// Number of times to run each benchmark.
   10.68 +#define REPITIONS 3
   10.69 +
   10.70 +// 10^x, x=0.5.. step 0.5.
   10.71 +const int exp[] = { 3, 10, 31, 100, 316, 1000, 3162, 10000, 31622,
   10.72 +                    100000, 316227, 1000000, 3162277, 10000000,
   10.73 +                    31622776, 100000000, 316227766, 1000000000 };
   10.74 +#define BENCHMARKS (sizeof(exp) / sizeof(exp[0]))
   10.75 +
   10.76 +struct stats {
   10.77 +    FILE *fp;
   10.78 +    const char *name;
   10.79 +    struct {
   10.80 +        // Number of keys in the db.
   10.81 +        int keys;
   10.82 +        unsigned long long time[REPITIONS];
   10.83 +        unsigned long long ops[REPITIONS];
   10.84 +    } benchmarks[BENCHMARKS];
   10.85 +};
   10.86 +
   10.87 +struct stats *stats_new(const char *name)
   10.88 +{
   10.89 +    struct stats *stats = (struct stats *) calloc(1, sizeof(*stats));
   10.90 +    stats->name = name;
   10.91 +    return stats;
   10.92 +}
   10.93 +
   10.94 +unsigned long long time() {
   10.95 +    struct timeval tv;
   10.96 +    int err = gettimeofday(&tv, NULL);
   10.97 +    assert(err == 0);
   10.98 +
   10.99 +    return (unsigned long long) (tv.tv_sec * 1000000 + tv.tv_usec);
  10.100 +}
  10.101 +
  10.102 +static char start_time[100];
  10.103 +
  10.104 +static void dump_stats(struct stats *stats, int benchmark) {
  10.105 +    if (LOG && ! stats->fp) {
  10.106 +        char fn[1024];
  10.107 +        sprintf(fn, "/tmp/pep-benchmark-%s-%s.csv", stats->name, start_time);
  10.108 +        stats->fp = fopen(fn , "w");
  10.109 +        if (! stats->fp) {
  10.110 +            printf("Opening %s failed.\n", fn);
  10.111 +            abort();
  10.112 +        }
  10.113 +
  10.114 +        fprintf(stats->fp, "Keys");
  10.115 +        for (int iter = 0; iter < REPITIONS; iter ++) {
  10.116 +            fprintf(stats->fp, ", run%d_us_per_op, run%d_ops_per_sec", iter, iter);
  10.117 +        }
  10.118 +        fprintf(stats->fp, "\n");
  10.119 +    }
  10.120 +
  10.121 +    printf("%-8d keys", stats->benchmarks[benchmark].keys);
  10.122 +    if (stats->fp)
  10.123 +        fprintf(stats->fp, "%d", stats->benchmarks[benchmark].keys);
  10.124 +
  10.125 +    for (int iter = 0; iter < REPITIONS; iter ++) {
  10.126 +        double us_per_lookup = (double) stats->benchmarks[benchmark].time[iter]
  10.127 +            / (double) stats->benchmarks[benchmark].ops[iter];
  10.128 +        double ops_per_second = (double) stats->benchmarks[benchmark].ops[iter]
  10.129 +            / ((double) stats->benchmarks[benchmark].time[iter] / 1000000);
  10.130 +
  10.131 +        printf("\t%.02f us/op (%.1f ops/s)", us_per_lookup, ops_per_second);
  10.132 +        if (stats->fp)
  10.133 +            fprintf(stats->fp, ", %f, %f", us_per_lookup, ops_per_second);
  10.134 +    }
  10.135 +
  10.136 +    printf("\n");
  10.137 +    if (stats->fp) {
  10.138 +        fprintf(stats->fp, "\n");
  10.139 +        fflush(stats->fp);
  10.140 +    }
  10.141 +}
  10.142 +
  10.143 +void LotsOfKeysTests::check() {
  10.144 +    struct tm tm;
  10.145 +    time_t t = time((time_t *) NULL);
  10.146 +    localtime_r(&t, &tm);
  10.147 +    strftime(start_time, sizeof(start_time), "%Y%m%d-%H%M%S", &tm);
  10.148 +
  10.149 +    struct stats *find_keys_stats = stats_new("find-keys");
  10.150 +    struct stats *get_identity_stats = stats_new("get-identity");
  10.151 +
  10.152 +    int benchmark = 0;
  10.153 +    PEP_STATUS status;
  10.154 +    pEp_identity **ids = (pEp_identity **) calloc(N, sizeof(*ids));
  10.155 +    assert(ids);
  10.156 +
  10.157 +    status = config_cipher_suite(session, PEP_CIPHER_SUITE_CV25519);
  10.158 +    TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
  10.159 +
  10.160 +    for (int key = 0; key < N; key ++) {
  10.161 +        // Create key
  10.162 +        char email[1024];
  10.163 +        sprintf(email, "%09d@example.org", key);
  10.164 +
  10.165 +        ids[key] = new_identity(strdup(email), NULL, NULL, "Test User");
  10.166 +        status = update_identity(session, ids[key]);
  10.167 +        TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
  10.168 +
  10.169 +        if (key == 0) {
  10.170 +            printf("\nRaw identity:\n");
  10.171 +            printf("   address = %s\n", ids[0]->address);
  10.172 +            printf("       fpr = %s\n", ids[0]->fpr);
  10.173 +            printf("   user_id = %s\n", ids[0]->user_id);
  10.174 +            printf("  username = %s\n", ids[0]->username);
  10.175 +        }
  10.176 +
  10.177 +        status = generate_keypair(session, ids[key]);
  10.178 +        TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
  10.179 +
  10.180 +        if (key == 0) {
  10.181 +            printf("\nAfter generating a key:\n");
  10.182 +            printf("   address = %s\n", ids[0]->address);
  10.183 +            printf("       fpr = %s\n", ids[0]->fpr);
  10.184 +            printf("   user_id = %s\n", ids[0]->user_id);
  10.185 +            printf("  username = %s\n", ids[0]->username);
  10.186 +        }
  10.187 +
  10.188 +        status = set_identity(session, ids[key]);
  10.189 +        TEST_ASSERT_MSG(status == PEP_STATUS_OK, tl_status_string(status));
  10.190 +
  10.191 +        if (key == 0) {
  10.192 +            printf("\nSetting identity:\n");
  10.193 +            printf("   address = %s\n", ids[0]->address);
  10.194 +            printf("       fpr = %s\n", ids[0]->fpr);
  10.195 +            printf("   user_id = %s\n", ids[0]->user_id);
  10.196 +            printf("  username = %s\n", ids[0]->username);
  10.197 +        }
  10.198 +
  10.199 +        bool do_benchmark = false;
  10.200 +        for (int i = 0; i < sizeof(exp) / sizeof(exp[0]); i ++) {
  10.201 +            if (key + 1 == exp[i]) {
  10.202 +                do_benchmark = true;
  10.203 +            } else if (key + 1 < exp[i]) {
  10.204 +                break;
  10.205 +            }
  10.206 +        }
  10.207 +
  10.208 +        if (! do_benchmark)
  10.209 +            continue;
  10.210 +
  10.211 +
  10.212 +        // Look up a random key by its email address.
  10.213 +        //
  10.214 +        // This doesn't use the engine, only the pgp
  10.215 +        // implementation.  For Sequoia, this should run in O(log
  10.216 +        // N).
  10.217 +        find_keys_stats->benchmarks[benchmark].keys = key + 1;
  10.218 +        for (int iter = 0; iter < REPITIONS; iter ++) {
  10.219 +            unsigned long long start = time();
  10.220 +            unsigned long long elapsed;
  10.221 +            int ops = 0;
  10.222 +
  10.223 +            do {
  10.224 +                int i = random() % (key + 1);
  10.225 +                assert(i <= key);
  10.226 +
  10.227 +                stringlist_t* keylist = NULL;
  10.228 +                status = find_keys(session, ids[i]->address, &keylist);
  10.229 +                free_stringlist(keylist);
  10.230 +                TEST_ASSERT_MSG(status == PEP_STATUS_OK,
  10.231 +                                tl_status_string(status));
  10.232 +                ops ++;
  10.233 +                elapsed = time() - start;
  10.234 +            } while (elapsed < TIME);
  10.235 +
  10.236 +            find_keys_stats->benchmarks[benchmark].ops[iter] = ops;
  10.237 +            find_keys_stats->benchmarks[benchmark].time[iter] = elapsed;
  10.238 +        }
  10.239 +
  10.240 +        dump_stats(find_keys_stats, benchmark);
  10.241 +
  10.242 +
  10.243 +        // Look up a random key by its pep user id.
  10.244 +        //
  10.245 +        // This uses the engine's management databank and doesn't
  10.246 +        // touch the pgp engine's DB.
  10.247 +        get_identity_stats->benchmarks[benchmark].keys = key + 1;
  10.248 +        for (int iter = 0; iter < REPITIONS; iter ++) {
  10.249 +            unsigned long long start = time();
  10.250 +            unsigned long long elapsed;
  10.251 +            int ops = 0;
  10.252 +
  10.253 +            do {
  10.254 +                int i = random() % (key + 1);
  10.255 +                pEp_identity *id = NULL;
  10.256 +                status = get_identity(session, ids[i]->address,
  10.257 +                                      ids[i]->user_id, &id);
  10.258 +                TEST_ASSERT_MSG(status == PEP_STATUS_OK,
  10.259 +                                tl_status_string(status));
  10.260 +                TEST_ASSERT_MSG(id->fpr, "no fingerprint");
  10.261 +                TEST_ASSERT_MSG(strcmp(ids[i]->fpr, id->fpr) == 0,
  10.262 +                                "fingerprint mismatch");
  10.263 +                free_identity(id);
  10.264 +
  10.265 +                ops ++;
  10.266 +                elapsed = time() - start;
  10.267 +            } while (elapsed < TIME);
  10.268 +
  10.269 +            get_identity_stats->benchmarks[benchmark].ops[iter] = ops;
  10.270 +            get_identity_stats->benchmarks[benchmark].time[iter] = elapsed;
  10.271 +        }
  10.272 +
  10.273 +        dump_stats(get_identity_stats, benchmark);
  10.274 +
  10.275 +
  10.276 +        benchmark++;
  10.277 +    }
  10.278 +}