committing to shelve changes, still debugging import_key_2.1
authorKrista 'DarthMama' Bennett <krista@pep.foundation>
Wed, 13 May 2020 19:14:18 +0200
branchimport_key_2.1
changeset 4712e01853b3f59d
parent 4711 25f5fbbc59ca
child 4717 333a32f6f7b3
committing to shelve changes, still debugging
src/pEp_internal.h
src/pgp_sequoia.c
     1.1 --- a/src/pEp_internal.h	Tue May 12 14:28:24 2020 +0200
     1.2 +++ b/src/pEp_internal.h	Wed May 13 19:14:18 2020 +0200
     1.3 @@ -134,6 +134,9 @@
     1.4          sqlite3_stmt *cert_save_insert_subkeys;
     1.5          sqlite3_stmt *cert_save_insert_userids;
     1.6          sqlite3_stmt *delete_keypair;
     1.7 +        // engine convenience hacks
     1.8 +        sqlite3_stmt *insert_ascii_import_hash;
     1.9 +        sqlite3_stmt *get_ascii_import_hash_for_fpr;
    1.10      } sq_sql;
    1.11  
    1.12      pgp_policy_t policy;
     2.1 --- a/src/pgp_sequoia.c	Tue May 12 14:28:24 2020 +0200
     2.2 +++ b/src/pgp_sequoia.c	Wed May 13 19:14:18 2020 +0200
     2.3 @@ -361,11 +361,25 @@
     2.4                                   "CREATE INDEX IF NOT EXISTS userids_index\n"
     2.5                                   "  ON userids (userid COLLATE EMAIL, primary_key)\n",
     2.6                                   NULL, NULL, NULL);
     2.7 +
     2.8      if (sqlite_result != SQLITE_OK)
     2.9          ERROR_OUT(NULL, PEP_INIT_CANNOT_OPEN_DB,
    2.10                    "creating userids table: %s",
    2.11                    sqlite3_errmsg(session->key_db));
    2.12  
    2.13 +    sqlite_result = sqlite3_exec(session->key_db,
    2.14 +                                 "CREATE TABLE IF NOT EXISTS ascii_import_hashes (\n"
    2.15 +                                 "   fpr TEXT PRIMARY KEY NOT NULL,\n"
    2.16 +                                 "   last_import_hash INT,\n"
    2.17 +                                 "   UNIQUE(fpr, last_import_hash)\n"
    2.18 +                                 ");\n",
    2.19 +                                 NULL, NULL, NULL);
    2.20 +                                 
    2.21 +    if (sqlite_result != SQLITE_OK)
    2.22 +        ERROR_OUT(NULL, PEP_INIT_CANNOT_OPEN_DB,
    2.23 +                  "creating ascii import hashes table: %s",
    2.24 +                  sqlite3_errmsg(session->key_db));
    2.25 +
    2.26      sqlite_result
    2.27          = sqlite3_prepare_v2(session->key_db, "begin transaction",
    2.28                               -1, &session->sq_sql.begin_transaction, NULL);
    2.29 @@ -482,6 +496,24 @@
    2.30                               -1, &session->sq_sql.delete_keypair, NULL);
    2.31      assert(sqlite_result == SQLITE_OK);
    2.32  
    2.33 +
    2.34 +    // Extra stuff to make the engine happy for the purposes of detecting 
    2.35 +    // possibly-changed key *files*. This is ONLY a heuristic
    2.36 +    sqlite_result
    2.37 +        = sqlite3_prepare_v2(session->key_db,
    2.38 +                             "INSERT OR REPLACE INTO ascii_import_hashes"
    2.39 +                             "    (fpr, last_import_hash)"
    2.40 +                             " VALUES (?, ?)",
    2.41 +                             -1, &session->sq_sql.insert_ascii_import_hash, NULL);
    2.42 +    assert(sqlite_result == SQLITE_OK);
    2.43 +
    2.44 +    sqlite_result
    2.45 +        = sqlite3_prepare_v2(session->key_db,
    2.46 +                             "select last_import_hash from ascii_import_hashes"
    2.47 +                             "    where fpr = ?1",
    2.48 +                             -1, &session->sq_sql.get_ascii_import_hash_for_fpr, NULL);
    2.49 +    assert(sqlite_result == SQLITE_OK);
    2.50 +    
    2.51      session->policy = pgp_null_policy ();
    2.52      if (! session->policy)
    2.53          ERROR_OUT(NULL, PEP_OUT_OF_MEMORY,
    2.54 @@ -2232,6 +2264,135 @@
    2.55      return status;
    2.56  }
    2.57  
    2.58 +// Fun stuff for import key and heuristics follows...
    2.59 +//
    2.60 +
    2.61 +// start detect possibly changed key stuff
    2.62 +
    2.63 +// NON-CRYPTOGRAPHIC HASH: take note - this is djb2 with XOR and is NEVER used 
    2.64 +// by sequoia or for any cryptographic purpose. We are using this to determine 
    2.65 +// if two ascii-armored keys for a given primary key are different, and that 
    2.66 +// is all. This is engine internal and our way of heuristically and cheaply 
    2.67 +// determining if a given key has *possibly* changed - we don't mind false 
    2.68 +// positives.
    2.69 +static uint32_t _non_crypto_hash_djb2(const char* str, const char* end) {
    2.70 +    uint32_t hash = 5381; // will also be returned if !str. That's OK.
    2.71 +    int c;
    2.72 +
    2.73 +    while ((str < end) && (c = *str++))
    2.74 +        hash = ((hash << 5) + hash) ^ c; // hash(i - 1) * 33 ^ str[i]
    2.75 +
    2.76 +    return hash;
    2.77 +} 
    2.78 +
    2.79 +static void _trim_ascii_key(const char* keystring, const char** start, 
    2.80 +                            const char** stop) {
    2.81 +    *start = NULL;
    2.82 +    *stop = NULL;
    2.83 +    
    2.84 +    const char* header = "-----BEGIN PGP PUBLIC KEY BLOCK-----";
    2.85 +    const char* footer = "-----END PGP PUBLIC KEY BLOCK-----";    
    2.86 +    const int _KEY_HEADER_LEN = 36; // strlen(header)
    2.87 +    const char* begin = strstr(keystring, header);
    2.88 +    if (!begin)
    2.89 +        return;
    2.90 +    begin = header + _KEY_HEADER_LEN;
    2.91 +    const char* end = strstr(begin, footer);
    2.92 +    if (!end)
    2.93 +        return;
    2.94 +    while (isspace(*begin) && begin < end) {
    2.95 +        begin++;
    2.96 +    }
    2.97 +    if (begin == end)
    2.98 +        return;
    2.99 +    while (isspace(*end) && end > begin) {
   2.100 +        end--;
   2.101 +    }
   2.102 +    if (end == begin)
   2.103 +        return;
   2.104 +    *start = begin;
   2.105 +    *stop = end + 1;        
   2.106 +}
   2.107 +
   2.108 +static PEP_STATUS _has_hash_changed(PEP_SESSION session,
   2.109 +                                    uint32_t test_hash, 
   2.110 +                                    const char* fpr, 
   2.111 +                                    bool* differs) {
   2.112 +
   2.113 +    bool changed = false;
   2.114 +    sqlite3_stmt* stmt = session->sq_sql.get_ascii_import_hash_for_fpr;
   2.115 +    sqlite3_bind_text(stmt, 1, 
   2.116 +                      fpr, -1, SQLITE_STATIC);
   2.117 +    int sqlite_result = sqlite3_step(stmt);                  
   2.118 +
   2.119 +    uint32_t old_hash = 0;
   2.120 +    
   2.121 +    PEP_STATUS status = PEP_STATUS_OK;
   2.122 +    
   2.123 +    switch (sqlite_result) {
   2.124 +        case SQLITE_ROW:
   2.125 +            old_hash = (uint32_t) sqlite3_column_int(stmt, 0);
   2.126 +            changed = (old_hash != test_hash);
   2.127 +            break;
   2.128 +        case SQLITE_DONE:
   2.129 +            // Not yet in database. Whee!
   2.130 +            changed = true;
   2.131 +            break;
   2.132 +        default:
   2.133 +           ERROR_OUT(NULL, PEP_UNKNOWN_ERROR,
   2.134 +                     "stepping: %s", sqlite3_errmsg(session->key_db));
   2.135 +    }
   2.136 +                    
   2.137 +    *differs = changed;
   2.138 +                                        
   2.139 +out:
   2.140 +    sqlite3_reset(stmt);
   2.141 +    T(" -> %s", pEp_status_to_string(status));
   2.142 +    return status;
   2.143 +}
   2.144 +
   2.145 +static PEP_STATUS _update_ascii_hash_for_key(PEP_SESSION session, 
   2.146 +                                             uint32_t new_hash, 
   2.147 +                                             const char* fpr) {
   2.148 +    PEP_STATUS status = PEP_STATUS_OK;
   2.149 +    sqlite3_stmt* stmt = session->sq_sql.insert_ascii_import_hash;
   2.150 +    sqlite3_bind_text(stmt, 1, fpr, -1, SQLITE_STATIC);
   2.151 +    sqlite3_bind_int(stmt, 2, new_hash);                      
   2.152 +    int sqlite_result = sqlite3_step(stmt);
   2.153 +    sqlite3_reset(stmt);    
   2.154 +    if (sqlite_result != SQLITE_DONE)
   2.155 +        ERROR_OUT(NULL, PEP_UNKNOWN_ERROR,
   2.156 +                  "Updating hash: %s", sqlite3_errmsg(session->key_db));
   2.157 +
   2.158 +out:
   2.159 +    T(" -> %s", pEp_status_to_string(status));
   2.160 +    return status;    
   2.161 +}
   2.162 +
   2.163 +static PEP_STATUS _detect_and_update_changed_ascii_key(PEP_SESSION session,
   2.164 +                                                       const char* keydata,
   2.165 +                                                       const char* fpr,
   2.166 +                                                       bool* changed) {
   2.167 +    const char* keydata_begin = NULL;
   2.168 +    const char* keydata_end = NULL;
   2.169 +    
   2.170 +    _trim_ascii_key(keydata, &keydata_begin, &keydata_end);
   2.171 +    if (!(keydata_begin && keydata_end))
   2.172 +        return PEP_ILLEGAL_VALUE;
   2.173 +    
   2.174 +    uint32_t hashval = _non_crypto_hash_djb2(keydata_begin, keydata_end);    
   2.175 +    PEP_STATUS status = _has_hash_changed(session, hashval, fpr, changed);
   2.176 +
   2.177 +    if (status != PEP_STATUS_OK)
   2.178 +        return status;
   2.179 +    
   2.180 +    if (*changed)
   2.181 +        status = _update_ascii_hash_for_key(session, hashval, fpr);    
   2.182 +    
   2.183 +    return status;    
   2.184 +}
   2.185 +// end detect possibly changed key stuff
   2.186 +
   2.187  static unsigned int count_keydata_parts(const char* key_data, size_t size) {
   2.188      unsigned int retval = 0;
   2.189  
   2.190 @@ -2252,14 +2413,23 @@
   2.191      return retval;
   2.192   }
   2.193  
   2.194 +// This is for single keys, which is why we're using a boolean here.
   2.195  PEP_STATUS _pgp_import_keydata(PEP_SESSION session, const char *key_data,
   2.196                                 size_t size, identity_list **private_idents,
   2.197                                 stringlist_t** imported_keys,
   2.198 -                               uint64_t* changed_key_index)
   2.199 +                               bool* changed)
   2.200  {
   2.201      PEP_STATUS status = PEP_NO_KEY_IMPORTED;
   2.202      pgp_error_t err;
   2.203      pgp_cert_parser_t parser = NULL;
   2.204 +    stringlist_t* keylist = NULL;
   2.205 +    *changed = false;
   2.206 +    
   2.207 +    if (imported_keys) {
   2.208 +        keylist = ((*imported_keys) ? *imported_keys : new_stringlist(NULL));
   2.209 +        if (!keylist)
   2.210 +            ERROR_OUT(NULL, PEP_OUT_OF_MEMORY, "setting retval keylist");
   2.211 +    }    
   2.212  
   2.213      if (private_idents)
   2.214          *private_idents = NULL;
   2.215 @@ -2353,8 +2523,21 @@
   2.216              // If private_idents is not NULL and there is any private key
   2.217              // material, it will be saved.
   2.218              status = cert_save(session, cert, private_idents);
   2.219 -            if (status == PEP_STATUS_OK)
   2.220 +            if (status == PEP_STATUS_OK) {
   2.221                  status = PEP_KEY_IMPORTED;
   2.222 +                if (imported_keys) {
   2.223 +                    char* fpr = pgp_fingerprint_to_hex(pgp_cert_fingerprint(cert));
   2.224 +                    if (!fpr) {
   2.225 +                        status = PEP_UNKNOWN_ERROR; // KB: Should we do this?
   2.226 +                        ERROR_OUT(NULL, status, "getting cert fingerprint hex");                    
   2.227 +                    }
   2.228 +                    stringlist_add(keylist, fpr); 
   2.229 +                    
   2.230 +                    status = _detect_and_update_changed_ascii_key(session, key_data, fpr, changed);  
   2.231 +                    if (status != PEP_STATUS_OK)
   2.232 +                        ERROR_OUT(NULL, status, "detecting / changing ascii key in DB");                    
   2.233 +                }    
   2.234 +            }    
   2.235              else
   2.236                  ERROR_OUT(NULL, status, "saving certificate");
   2.237          }
   2.238 @@ -2391,9 +2574,12 @@
   2.239                                stringlist_t** imported_keys,
   2.240                                uint64_t* changed_key_index)
   2.241  {
   2.242 -
   2.243 +    *changed_key_index = 0;
   2.244 +    
   2.245      const char* pgp_begin = "-----BEGIN PGP";
   2.246      size_t prefix_len = strlen(pgp_begin);
   2.247 +    
   2.248 +    PEP_STATUS retval = PEP_STATUS_OK;
   2.249  
   2.250      // Because we also import binary keys we have to be careful with this.
   2.251      // 
   2.252 @@ -2407,9 +2593,15 @@
   2.253      }
   2.254  
   2.255      unsigned int keycount = count_keydata_parts(key_data, size);
   2.256 -    if (keycount < 2)
   2.257 -        return(_pgp_import_keydata(session, key_data, size, private_idents,
   2.258 -                                   imported_keys, changed_key_index));
   2.259 +    if (keycount < 2) {
   2.260 +        bool changed = false;
   2.261 +        retval = _pgp_import_keydata(session, key_data, size, private_idents,
   2.262 +                                     imported_keys, &changed);
   2.263 +        if (retval == PEP_KEY_IMPORTED)
   2.264 +            *changed_key_index = 1; // only one key 
   2.265 +        
   2.266 +        return retval;    
   2.267 +    }        
   2.268  
   2.269      unsigned int i;
   2.270      const char* curr_begin;
   2.271 @@ -2417,7 +2609,9 @@
   2.272  
   2.273      identity_list* collected_idents = NULL;
   2.274  
   2.275 -    PEP_STATUS retval = PEP_KEY_IMPORTED;
   2.276 +    retval = PEP_KEY_IMPORTED;
   2.277 +    
   2.278 +    int imported_key_index = 0;
   2.279              
   2.280      for (i = 0, curr_begin = key_data; i < keycount; i++) {
   2.281          const char* next_begin = NULL;
   2.282 @@ -2433,12 +2627,13 @@
   2.283          else
   2.284              curr_size = (key_data + size) - curr_begin;
   2.285  
   2.286 +        bool changed = false;    
   2.287          PEP_STATUS curr_status = _pgp_import_keydata(session, 
   2.288                                                       curr_begin, 
   2.289                                                       curr_size, 
   2.290                                                       private_idents,
   2.291                                                       imported_keys,
   2.292 -                                                     changed_key_index);
   2.293 +                                                     &changed);
   2.294          if (private_idents && *private_idents) {
   2.295              if (!collected_idents)
   2.296                  collected_idents = *private_idents;
   2.297 @@ -2447,6 +2642,10 @@
   2.298              *private_idents = NULL;
   2.299          }
   2.300  
   2.301 +        // Set changed bit for appropriately-indexed key
   2.302 +        if (changed && curr_status == PEP_KEY_IMPORTED)
   2.303 +            *changed_key_index |= (1 << imported_key_index++);
   2.304 +
   2.305          if (curr_status != retval) {
   2.306              switch (curr_status) {
   2.307                  case PEP_NO_KEY_IMPORTED: