Merge branch 'feature/credential-manager' into feature/multi-directory-cred

Author: adorton-adobe

Makefile

@@ -15,7 +15,7 @@ endif
 
 standalone:
 	python -m pip install --upgrade pip
-	python -m pip install --upgrade pyinstaller
+	python -m pip install --upgrade pyinstaller==4.0
 	python -m pip install --upgrade setuptools
 	-$(RM) $(output_dir)
 	python .build/pre_build.py

docs/en/user-manual/additional_tools.md

@@ -104,6 +104,16 @@ in OS Secure storage.
 Successful console output after running the ```store``` command:
 
 ```
+To store the priv_key_data when it's too long
+In order to save the priv_key_data which is present in the  connector-umapi or connector-adobe-console 
+file, Credential Store command will attempt to store the key data in the OS.
+Window credential store generally can't store data as large as Private Key. To store the key, Sync Tool 
+asks the user to encrypt the key. 
+If yes then user will be prompted for a password which will be saved as priv_key_pass and the encrypted key itself will be saved as priv_key_data in the file. 
+If no, the key data will remain in the file in an unencrypted state.
+
+Refer to the (URL)
+
 (venv) C:\Program Files\Adobe\Adobe User Sync Tool>python user-sync.pex credentials store
    <output from store goes here>
 ```

examples/config files - basic/connector-adobe-console.yml

@@ -44,7 +44,7 @@ integration:
   # data directly in this file.  To do this, remove the priv_key_path entry above
   # and uncomment the following entry.  Replace the sample data with the data
   # from your private key file (which will be much longer).
-  #priv_key_data: |
+  # priv_key_data: |
   #   -----BEGIN RSA PRIVATE KEY-----
   #   MIIf74jfd84oAgEA6brj4uZ2f1Nkf84j843jfjjJGHYJ8756GHHGGz7jLyZWSscH
   #   CoifurKJY763GHKL98mJGYxWSBvhlWskdjdatagoeshere986fKFUNGd74kdfuEH
@@ -70,7 +70,6 @@ integration:
   # Having done this, you can use the setting priv_key_pass to specify the passphrase needed
   # by User Sync to decrypt the private key file (or private key data), as in:
   # priv_key_pass: "my passphrase for my private key"
-  
   # Private key pass can also be stored using credentials store command
 
 # (optional) identity_type_filter (default value is all)

examples/config files - basic/connector-umapi.yml

@@ -43,17 +43,33 @@ server:
 # the correct settings for your enterprise organization.
 # [NOTE: the priv_key_path setting can be an absolute or relative pathname;
 # if relative, it is interpreted relative to this configuration file.]
+
+# (optional) You can store credentials in the operating system credential store
+# (Windows Credential Manager, Mac Keychain, Linux Freedesktop Secret Service
+# or KWallet - these will be built into the Linux distribution).
+# To use this feature, You can secure the credentials by using the credentials store command in the command line
+# Refer to the (URL to Additional Tools)
+
+# (optional) You can encrypt the private key by running `user-sync.exe encrypt'.  
+# This will encrypt the private key as RSA 256. you should uncomment 
+# priv_key_pass key so that User Sync is knows to decrypt the key.  priv_key_pass should have the plaintext value
+# of the password (or, see credential storage above).  The decryption also works for keys encrypted
+# externally with OpenSSL or other libraries, provided the format is correct.
+
 enterprise:
   org_id: "Org ID goes here"
   client_id: "Client ID goes here"
   client_secret: "Client secret goes here"
   tech_acct_id: "Tech account ID goes here"
   priv_key_path: "private.key"
+  #priv_key_pass: "my passphrase for my private key"
 
   # (optional) As an alternative to priv_key_path, you can place the private key 
   # data directly in this file.  To do this, remove the priv_key_path entry above 
   # and uncomment the following entry.  Replace the sample data with the data 
   # from your private key file (which will be much longer).
+  # Note: the Windows credential store can't store data as large as a private
+  # key, so Windows users will be prompted to encrypt the private key data instead.
   #priv_key_data: |
   #   -----BEGIN RSA PRIVATE KEY-----
   #   MIIf74jfd84oAgEA6brj4uZ2f1Nkf84j843jfjjJGHYJ8756GHHGGz7jLyZWSscH

tests/test_config.py

@@ -302,7 +302,7 @@ def test_get_credential_old_format(tmp_config_files):
     with pytest.raises(AssertionException):
         ldap_dict_config.get_credential('password', 'user_sync')
     username = ldap_config['username']
-    credman.set('ldap_secure_identifier', 'test_password', username)
+    credman.set('ldap_secure_identifier', 'test_password', username=username)
     # set the plain key to None so get_credential will look for the secure_password_key format
     ldap_config['password'] = None
     assert ldap_dict_config.get_credential('password', username) == 'test_password'

tests/test_credentials.py

@@ -1,10 +1,13 @@
 import os
 import uuid
 
+import keyring
 import pytest
+import yaml
 
-from user_sync.credentials import *
-from user_sync.error import AssertionException
+from tests.util import update_dict
+from user_sync import encryption
+from user_sync.credentials import CredentialConfig, CredentialManager, Key, LdapCredentialConfig, UmapiCredentialConfig
 
 
 @pytest.fixture
@@ -17,6 +20,19 @@ def ldap_config_file(fixture_dir):
     return os.path.join(fixture_dir, 'connector-ldap.yml')
 
 
+@pytest.fixture
+def modify_umapi_config(tmp_config_files):
+    (_, _, umapi_config_file) = tmp_config_files
+
+    def _modify_umapi_config(keys, val):
+        conf = yaml.safe_load(open(umapi_config_file))
+        conf = update_dict(conf, keys, val)
+        yaml.dump(conf, open(umapi_config_file, 'w'))
+
+        return umapi_config_file
+    return _modify_umapi_config
+
+
 def test_nested_set(ldap_config_file):
     c = CredentialConfig(ldap_config_file)
     c.set_nested_key(['password'], {'secure': 'somethingverysecure'})
@@ -27,37 +43,37 @@ def test_nested_set(ldap_config_file):
 def test_retrieve_ldap_creds_valid(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     c = CredentialConfig(ldap_config_file)
-    key_list = ['password']
-    plaintext_cred = c.get_nested_key(key_list)
-    c.store_key(key_list)
-    retrieved_plaintext_cred = c.retrieve_key(key_list)
+    key = Key(['password'])
+    plaintext_cred = c.get_nested_key(key.key_path)
+    c.store_key(key)
+    retrieved_plaintext_cred = c.retrieve_key(key)
     assert retrieved_plaintext_cred == plaintext_cred
 
 
 def test_retrieve_ldap_creds_invalid(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     c = CredentialConfig(ldap_config_file)
-    key_list = ['password']
+    key = Key(['password'])
     # if store_key has not been called previously, retrieve_key returns None
-    assert c.retrieve_key(key_list) is None
+    assert c.retrieve_key(key) is None
 
 
 def test_revert_valid(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     c = CredentialConfig(ldap_config_file)
-    key_list = ['password']
-    plaintext_cred = c.get_nested_key(key_list)
-    c.store_key(key_list)
-    reverted_plaintext_cred = c.revert_key(key_list)
+    key = Key(['password'])
+    plaintext_cred = c.get_nested_key(key.key_path)
+    c.store_key(key)
+    reverted_plaintext_cred = c.revert_key(key)
     assert reverted_plaintext_cred == plaintext_cred
 
 
 def test_revert_invalid(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     c = CredentialConfig(ldap_config_file)
-    key_list = ['password']
+    key = Key(['password'])
     # assume store_key has not been called
-    assert c.revert_key(key_list) is None
+    assert c.revert_key(key) is None
 
 
 def test_retrieve_revert_ldap_valid(tmp_config_files):
@@ -84,20 +100,18 @@ def test_retrieve_revert_ldap_invalid(tmp_config_files):
     # if store has not been previously called before retrieve and revert we can expect the following
     retrieved_key_dict = ldap.retrieve()
     assert retrieved_key_dict == {}
-
     creds = ldap.revert()
     assert creds == {}
 
 
-
-def test_retrieve_revert_umapi_valid(tmp_config_files):
-    (_, _, umapi_config_file) = tmp_config_files
-    umapi = UmapiCredentialConfig(umapi_config_file)
+def test_retrieve_revert_umapi_valid(private_key, modify_umapi_config):
+    umapi_config_file = modify_umapi_config(['enterprise', 'priv_key_path'], private_key)
+    umapi = UmapiCredentialConfig(umapi_config_file, auto=True)
     # Using the api_key for assertions. The rest can be added in later if deemed necessary
     assert not umapi.parse_secure_key(umapi.get_nested_key(['enterprise', 'client_id']))
     unsecured_api_key = umapi.get_nested_key(['enterprise', 'client_id'])
     umapi.store()
-    with open(umapi_config_file) as f:
+    with open(umapi_config_file, 'r') as f:
         data = yaml.load(f)
         assert umapi.parse_secure_key(data['enterprise']['client_id'])
     retrieved_key_dict = umapi.retrieve()
@@ -108,9 +122,10 @@ def test_retrieve_revert_umapi_valid(tmp_config_files):
         assert data['enterprise']['client_id'] == unsecured_api_key
 
 
-def test_credman_retrieve_revert_valid(tmp_config_files):
-    (root_config_file, ldap_config_file, umapi_config_file) = tmp_config_files
-    credman = CredentialManager(root_config_file)
+def test_credman_retrieve_revert_valid(tmp_config_files, private_key, modify_umapi_config):
+    (root_config_file, ldap_config_file, _) = tmp_config_files
+    umapi_config_file = modify_umapi_config(['enterprise', 'priv_key_path'], private_key)
+    credman = CredentialManager(root_config_file, auto=True)
     with open(ldap_config_file) as f:
         data = yaml.load(f)
         plaintext_ldap_password = data['password']
@@ -137,8 +152,9 @@ def test_credman_retrieve_revert_valid(tmp_config_files):
         assert data['enterprise']['client_id'] == plaintext_umapi_api_key
 
 
-def test_credman_retrieve_revert_invalid(tmp_config_files):
-    (root_config_file, ldap_config_file, umapi_config_file) = tmp_config_files
+def test_credman_retrieve_revert_invalid(tmp_config_files, private_key, modify_umapi_config):
+    (root_config_file, ldap_config_file, _) = tmp_config_files
+    umapi_config_file = modify_umapi_config(['enterprise', 'priv_key_path'], private_key)
     credman = CredentialManager(root_config_file)
     # if credman.store() has not been called first then we can expect the following
     retrieved_creds = credman.retrieve()
@@ -147,7 +163,6 @@ def test_credman_retrieve_revert_invalid(tmp_config_files):
     assert creds == {}
 
 
-
 def test_set():
     identifier = 'TestId'
     value = 'TestValue'
@@ -170,7 +185,7 @@ def test_set_long():
     value = "".join([str(uuid.uuid4()) for x in range(500)])
 
     if isinstance(keyring.get_keyring(), keyring.backends.Windows.WinVaultKeyring):
-        with pytest.raises(AssertionException):
+        with pytest.raises(Exception):
             cm.set(identifier, value)
     else:
         cm.set(identifier, value)
@@ -189,7 +204,8 @@ def test_get_not_valid():
 def test_config_store(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     ldap = LdapCredentialConfig(ldap_config_file)
-    assert not ldap.parse_secure_key(ldap.get_nested_key(['password']))
+    key = Key(['password'])
+    assert not ldap.parse_secure_key(ldap.get_nested_key(key.key_path))
     ldap.store()
     with open(ldap_config_file) as f:
         data = yaml.load(f)
@@ -199,14 +215,52 @@ def test_config_store(tmp_config_files):
 def test_config_store_key(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     ldap = LdapCredentialConfig(ldap_config_file)
-    assert not ldap.parse_secure_key(ldap.get_nested_key(['password']))
-    ldap.store_key(['password'])
-    assert ldap.parse_secure_key(ldap.get_nested_key(['password']))
+    key = Key(['password'])
+    assert not ldap.parse_secure_key(ldap.get_nested_key(key.key_path))
+    ldap.store_key(key)
+    assert ldap.parse_secure_key(ldap.get_nested_key(key.key_path))
 
 
 def test_config_store_key_none(tmp_config_files):
     (_, ldap_config_file, _) = tmp_config_files
     ldap = LdapCredentialConfig(ldap_config_file)
-    ldap.set_nested_key(['password'], [])
-    with pytest.raises(AssertionException):
-        ldap.store_key(['password'])
+    key = Key(['password'])
+    ldap.set_nested_key(key.key_path, [])
+    assert ldap.store_key(key) is None
+
+
+def test_credman_encrypt_decrypt_key_path(tmp_config_files, private_key, modify_umapi_config):
+    (root_config_file, ldap_config_file, _) = tmp_config_files
+    umapi_config_file = modify_umapi_config(['enterprise', 'priv_key_path'], private_key)
+    credman = CredentialManager(root_config_file, auto=True)
+    with open(private_key) as f:
+        key_data = f.read()
+        assert encryption.is_encryptable(key_data)
+    credman.store()
+    with open(private_key) as f:
+        key_data = f.read()
+        assert not encryption.is_encryptable(key_data)
+    credman.revert()
+    with open(private_key) as f:
+        key_data = f.read()
+        assert encryption.is_encryptable(key_data)
+
+
+def test_credman_encrypt_decrypt_key_data(tmp_config_files, private_key, modify_umapi_config):
+    (root_config_file, ldap_config_file, _) = tmp_config_files
+    umapi_config_file = modify_umapi_config(['enterprise', 'priv_key_path'], None)
+    with open(private_key) as f:
+        key_data = f.read()
+        umapi_config_file = modify_umapi_config(['enterprise', 'priv_key_data'], key_data)
+    credman = CredentialManager(root_config_file, auto=True)
+    with open(umapi_config_file) as f:
+        umapi_dict = yaml.load(f)
+        assert encryption.is_encryptable(umapi_dict['enterprise']['priv_key_data'])
+    credman.store()
+    with open(umapi_config_file) as f:
+        umapi_dict = yaml.load(f)
+        assert not encryption.is_encryptable(umapi_dict['enterprise']['priv_key_data'])
+    credman.revert()
+    with open(umapi_config_file) as f:
+        umapi_dict = yaml.load(f)
+        assert encryption.is_encryptable(umapi_dict['enterprise']['priv_key_data'])

tests/test_umapi_util.py

@@ -41,7 +41,15 @@ def test_make_auth_dict(umapi_config_file, private_key):
         invalid_auth_dict = make_auth_dict(name, umapi_dict_config, org_id_from_file, tech_acct_from_file, logger)
     # and if there is only the secure format it should work as long as the credential has been set
     credman = CredentialManager()
-    credman.set('make_auth_identifier', 'keydata', org_id_from_file)
+    credman.set('make_auth_identifier', 'keydata', username=org_id_from_file)
     umapi_config['enterprise']['priv_key_data'] = None
     auth_dict = make_auth_dict(name, umapi_dict_config, org_id_from_file, tech_acct_from_file, logger)
     assert auth_dict['private_key_data'] == 'keydata'
+    # clear out old format (already tested for the exception if both are provided)
+    umapi_config['enterprise']['secure_priv_key_data_key'] = None
+    # put the regular priv_key_data back in the file and store
+    # the first 256 chars so it'll work without cryptfile on Windows
+    umapi_config['enterprise']['priv_key_data'] = {'secure': 'truncated_key_data'}
+    credman.set('truncated_key_data', key_data_from_file[:255])
+    auth_dict = make_auth_dict(name, umapi_dict_config, org_id_from_file, tech_acct_from_file, logger)
+    assert auth_dict['private_key_data'] == key_data_from_file[:255]