From c17775b715fa9094dbd95a909a6626e2a03cd610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomek=20W=C3=B3jcik?= Date: Thu, 8 Feb 2024 20:52:08 +0000 Subject: [PATCH] v1.2.0 --- CHANGELOG.md | 4 + docs/source/conf.py | 4 +- docs/source/ext.rst | 20 +++ keep_it_secret/__init__.py | 2 +- keep_it_secret/ext/vault.py | 177 +++++++++++++++++++++ poetry.lock | 22 ++- pyproject.toml | 5 +- setup.cfg | 5 + tests/ext/vault/__init__.py | 0 tests/ext/vault/conftest.py | 12 ++ tests/ext/vault/fixtures.py | 13 ++ tests/ext/vault/test_VaultKV2Field.py | 212 ++++++++++++++++++++++++++ tests/ext/vault/test_VaultSecrets.py | 109 +++++++++++++ 13 files changed, 579 insertions(+), 6 deletions(-) create mode 100644 keep_it_secret/ext/vault.py create mode 100644 tests/ext/vault/__init__.py create mode 100644 tests/ext/vault/conftest.py create mode 100644 tests/ext/vault/fixtures.py create mode 100644 tests/ext/vault/test_VaultKV2Field.py create mode 100644 tests/ext/vault/test_VaultSecrets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 02a4ce4..11d04f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Keep It Secret Changelog +#### v1.2.0 (2024-02-08) + +* Hashicorp Vault integration. + #### v1.1.0 (2024-01-18) * `Secrets.resolve_dependency()` API for resolving field dependencies. diff --git a/docs/source/conf.py b/docs/source/conf.py index ed912f2..6e52e26 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,10 +11,10 @@ project = 'Keep It Secret' copyright = '2023-present Tomek Wójcik' author = 'Tomek Wójcik' -version = '1.1.0' +version = '1.2.0' # The full version, including alpha/beta/rc tags -release = '1.1.0' +release = '1.2.0' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/source/ext.rst b/docs/source/ext.rst index 6e7384c..331360b 100644 --- a/docs/source/ext.rst +++ b/docs/source/ext.rst @@ -25,6 +25,26 @@ to be installed: .. autoclass:: keep_it_secret.ext.aws.AWSSecretsManagerField :members: +Hashicorp Vault Wrapper +----------------------- + +**Installation** + +Since Vault extension has external dependencies it needs to be explicitly named +to be installed: + +.. code-block:: shell + + $ pip install keep_it_secret[vault] + +**API** + +.. autoclass:: keep_it_secret.ext.vault.VaultSecrets + :members: + +.. autoclass:: keep_it_secret.ext.vault.VaultKV2Field + :members: + Basic secrets loader -------------------- diff --git a/keep_it_secret/__init__.py b/keep_it_secret/__init__.py index 08bc0c5..50ffe5f 100644 --- a/keep_it_secret/__init__.py +++ b/keep_it_secret/__init__.py @@ -6,7 +6,7 @@ from .fields import ( # noqa: F401 ) from .secrets import Secrets # noqa: F401 -__version__ = '1.1.0' +__version__ = '1.2.0' __all__ = [ 'AbstractField', diff --git a/keep_it_secret/ext/vault.py b/keep_it_secret/ext/vault.py new file mode 100644 index 0000000..f6e3b82 --- /dev/null +++ b/keep_it_secret/ext/vault.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +import typing + +import hvac + +from keep_it_secret.fields import EnvField, Field +from keep_it_secret.secrets import Secrets + + +class VaultSecrets(Secrets): + """ + Concrete :py:class:`keep_it_secret.Secrets` subclass that maps environment + variables to Vault credentials. + """ + + url: str = EnvField.new('VAULT_URL', required=True) + """ + Maps ``VAULT_URL`` environment variable. + + :type: ``str`` + """ + + token: str = EnvField.new('VAULT_TOKEN', required=True) + """ + Maps ``VAULT_TOKEN`` environment variable. + + :type: ``str`` + """ + + client_cert_path: str | None = EnvField.new('VAULT_CLIENT_CERT_PATH', required=False) + """ + Maps ``VAULT_CLIENT_CERT_PATH`` environment variable. + + :type: ``str | None`` + """ + + client_key_path: str | None = EnvField.new('VAULT_CLIENT_KEY_PATH', required=False) + """ + Maps ``VAULT_CLIENT_KEY_PATH`` environment variable. + + :type: ``str | None`` + """ + + server_cert_path: str | None = EnvField.new('VAULT_SERVER_CERT_PATH', required=False) + """ + Maps ``VAULT_SERVER_CERT_PATH`` environment variable. + + :type: ``str | None`` + """ + + def __init__(self, parent: Secrets | None = None): + super().__init__(parent) + + self.client: hvac.Client | None = None + + def as_hvac_client_kwargs(self) -> dict[str, typing.Any]: + """ + Return representation of the mapped variables for use in + ``hvac.Client`` constructor. + """ + result: dict[str, typing.Any] = { + 'url': self.url, + 'token': self.token, + } + + if self.client_cert_path is not None and self.client_key_path is not None: + result['cert'] = (self.client_cert_path, self.client_key_path) + + if self.server_cert_path is not None: + result['verify'] = self.server_cert_path + + return result + + def get_client(self) -> hvac.Client: + """ + Return the ``hvac.Client`` instance configured using the credentials. + """ + if self.client is None: + self.client = hvac.Client( + **self.as_hvac_client_kwargs(), + ) + + return self.client + + +class VaultKV2Field(Field): + """ + Concrete :py:class:`keep_it_secret.Field` subclass that uses Hashicorp + Vault KV V2 secrets engine to resolve the value. + + If ``as_type`` isn't provided, the fetched value will be returned as-is ( + i.e. as a dict). Otherwise, ``as_type`` should be a type which accepts + kwargs representing the value's keys in its constructor. + + :param mount_point: Mount path for the secret engine. + :param path: Path to the secret to fetch. + :param version: Version identifier. Defaults to ``None`` (aka the newest + version). + :param default: Default value. Defaults to ``None``. + """ + + def __init__(self, + mount_point: str, + path: str, + version: str | None = None, + default: typing.Any = None, + **field_options): + super().__init__(**field_options) + + as_type: type | None = field_options.pop('as_type', None) + self.as_type = self.cast if as_type is not None else None # type: ignore[assignment] + + self.mount_point = mount_point + self.path = path + self.version = version + self.default = default + self.cast_to = as_type + + @classmethod + def new(cls: type[VaultKV2Field], # type: ignore[override] + mount_point: str, + path: str, + version: str | None = None, + default: typing.Any = None, + **field_options): + return cls(mount_point, path, version=version, default=default, **field_options) + + def cast(self, data: typing.Any) -> typing.Any: + """ + Cast ``data`` to the type specified in ``as_type`` argument of the + constructor. + """ + if isinstance(data, dict) is False: + return data + + if self.cast_to is None: + return data + + return self.cast_to(**data) + + def get_value(self, secrets: Secrets) -> typing.Any: + """ + Retrieve, decode and return the secret stored in a KV V2 secrets engine + mounted at *mount_path* under the path *path*. + + Depends on :py:class:`VaultSecrets` to be declared in ``vault`` field + on ``secrets`` or one of its parents. + + :raises DependencyMissing: Signal that ``secrets.aws`` field is + missing. + :raises RequiredValueMissing: Signal the field's value is required but + *secret_id* is not present in the secrets engine. + """ + vault_secrets: VaultSecrets = secrets.resolve_dependency( + 'vault', include_parents=True, + ) + if vault_secrets is secrets.UNRESOLVED_DEPENDENCY: + raise self.DependencyMissing('vault') + + client = vault_secrets.get_client() + + try: + secret_response = client.secrets.kv.v2.read_secret_version( + self.path, + version=self.version, + mount_point=self.mount_point, + raise_on_deleted_version=True, + ) + + return secret_response['data']['data'] + except hvac.exceptions.InvalidPath as exception: + if self.required is True: + raise self.RequiredValueMissing(f'{self.mount_point}{self.path}') from exception + else: + return self.default diff --git a/poetry.lock b/poetry.lock index 7f2203f..5faf70e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -429,6 +429,23 @@ pygments = ">=2.7" sphinx = ">=6.0,<8.0" sphinx-basic-ng = "*" +[[package]] +name = "hvac" +version = "2.1.0" +description = "HashiCorp Vault API client" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "hvac-2.1.0-py3-none-any.whl", hash = "sha256:73bc91e58c3fc7c6b8107cdaca9cb71fa0a893dfd80ffbc1c14e20f24c0c29d7"}, + {file = "hvac-2.1.0.tar.gz", hash = "sha256:b48bcda11a4ab0a7b6c47232c7ba7c87fda318ae2d4a7662800c465a78742894"}, +] + +[package.dependencies] +requests = ">=2.27.1,<3.0.0" + +[package.extras] +parser = ["pyhcl (>=0.4.4,<0.5.0)"] + [[package]] name = "idna" version = "3.6" @@ -1692,8 +1709,9 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [extras] aws = ["boto3"] +vault = ["hvac"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "e0cabda70067d490af2f9c9e9251384fb43454c545da6a071e25b1b74cc9f951" +content-hash = "e4959d95b206c17013e548f015fbf288d9bc754584bfc128458cadead137a863" diff --git a/pyproject.toml b/pyproject.toml index 237dd3e..23f024a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep-it-secret" -version = "1.1.0" +version = "1.2.0" description = "Keep It Secret by BTHLabs" authors = ["Tomek Wójcik "] maintainers = ["BTHLabs "] @@ -13,12 +13,14 @@ documentation = "https://projects.bthlabs.pl/keep-it-secret/" [tool.poetry.dependencies] python = "^3.10" boto3 = {version = ">=1.34.0", optional = true} +hvac = {version = ">=2.1.0", optional = true} [tool.poetry.group.dev.dependencies] boto3 = "1.34.8" flake8 = "6.1.0" flake8-commas = "2.1.0" furo = "2023.9.10" +hvac = "2.1.0" invoke = "1.7.3" ipdb = "0.13.13" ipython = "8.19.0" @@ -32,6 +34,7 @@ twine = "4.0.2" [tool.poetry.extras] aws = ["boto3"] +vault = ["hvac"] [build-system] requires = ["poetry-core"] diff --git a/setup.cfg b/setup.cfg index 1947b42..001cbc1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,8 @@ env = AWS_SECRET_ACCESS_KEY=thisisntright AWS_SESSION_TOKEN=thisisntright AWS_DEFAULT_REGION=eu-central-1 + VAULT_URL=http://thisisntright:8200/ + VAULT_TOKEN=thisisntright [mypy] @@ -21,5 +23,8 @@ ignore_missing_imports = True [mypy-boto3.*] ignore_missing_imports = True +[mypy-hvac.*] +ignore_missing_imports = True + [mypy-moto.*] ignore_missing_imports = True diff --git a/tests/ext/vault/__init__.py b/tests/ext/vault/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ext/vault/conftest.py b/tests/ext/vault/conftest.py new file mode 100644 index 0000000..aaec631 --- /dev/null +++ b/tests/ext/vault/conftest.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# type: ignore +from __future__ import annotations + +import pytest + +from .fixtures import TestingVaultSecrets + + +@pytest.fixture +def testing_vault_secrets() -> TestingVaultSecrets: + return TestingVaultSecrets() diff --git a/tests/ext/vault/fixtures.py b/tests/ext/vault/fixtures.py new file mode 100644 index 0000000..270f4af --- /dev/null +++ b/tests/ext/vault/fixtures.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +# -*- coding: utf-8 -*- +from __future__ import annotations + +from keep_it_secret.ext.vault import VaultSecrets +from keep_it_secret.fields import SecretsField +from keep_it_secret.secrets import Secrets + + +class TestingVaultSecrets(Secrets): + vault = SecretsField.new(VaultSecrets) diff --git a/tests/ext/vault/test_VaultKV2Field.py b/tests/ext/vault/test_VaultKV2Field.py new file mode 100644 index 0000000..f4dd327 --- /dev/null +++ b/tests/ext/vault/test_VaultKV2Field.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +# type: ignore +from __future__ import annotations + +from unittest import mock + +import hvac +import pytest +from pytest_mock import MockerFixture + +from keep_it_secret import Secrets +from keep_it_secret.ext import vault + + +@pytest.fixture +def field() -> vault.VaultKV2Field: + return vault.VaultKV2Field('keep_it_secret/', 'tests/spam') + + +@pytest.fixture +def as_type() -> mock.Mock: + return mock.Mock() + + +@pytest.fixture +def field_with_as_type(as_type: mock.Mock) -> vault.VaultKV2Field: + return vault.VaultKV2Field('keep_it_secret/', 'tests/spam', as_type=as_type) + + +@pytest.fixture +def hvac_kv_v2_client(mocker: MockerFixture) -> mock.Mock: + result = mock.Mock() + result.secrets.kv.v2 = mock.Mock(spec=['read_secret_version']) + + return result + + +def test_init(): + # When + result = vault.VaultKV2Field('keep_it_secret/', 'tests/spam') + + # Then + assert result.mount_point == 'keep_it_secret/' + assert result.path == 'tests/spam' + assert result.version is None + assert result.default is None + assert result.cast_to is None + assert result.as_type is None + + +def test_init_with_version(): + # When + result = vault.VaultKV2Field( + 'keep_it_secret/', 'tests/spam', version='1', + ) + + # Then + assert result.version == '1' + + +def test_init_with_default(): + # When + result = vault.VaultKV2Field( + 'keep_it_secret/', 'tests/spam', default='eggs', + ) + + # Then + assert result.default == 'eggs' + + +def test_init_with_as_type(): + # When + result = vault.VaultKV2Field( + 'keep_it_secret/', 'tests/spam', as_type=dict, + ) + + # Then + assert result.as_type == result.cast + assert result.cast_to == dict + + +def test_init_with_field_options(): + # When + result = vault.VaultKV2Field( + 'keep_it_secret/', 'tests/spam', required=False, description='eggs', + ) + + # Then + assert result.required is False + assert result.description == 'eggs' + + +def test_new(mocker: MockerFixture): + # Given + mock_init = mocker.patch.object( + vault.VaultKV2Field, '__init__', return_value=None, + ) + + # When + _ = vault.VaultKV2Field.new( + 'keep_it_secret/', + 'tests/spam', + version='1', + default=dict, + as_type=dict, + required=False, + description='eggs', + ) + + # Then + mock_init.assert_called_once_with( + 'keep_it_secret/', + 'tests/spam', + version='1', + default=dict, + as_type=dict, + required=False, + description='eggs', + ) + + +def test_cast_not_dict(field: vault.VaultKV2Field): + # When + result = field.cast('spam') + + # Then + assert result == 'spam' + + +def test_cast(field_with_as_type: vault.VaultKV2Field, as_type: mock.Mock): + # Given + as_type.return_value = 'spam' + + # When + result = field_with_as_type.cast({ + 'spam': True, + 'eggs': False, + }) + + # Then + assert result == 'spam' + + as_type.assert_called_once_with( + spam=True, + eggs=False, + ) + + +def test_get_value_vault_dependency_missing(field: vault.VaultKV2Field, + testing_secrets: Secrets): + # Given + with pytest.raises(field.DependencyMissing) as exception_info: + # When + _ = field(testing_secrets) + + # Then + assert exception_info.value.args[0] == 'vault' + + +def test_get_value_required_value_not_found(testing_vault_secrets: Secrets, + hvac_kv_v2_client: mock.Mock, + field: vault.VaultKV2Field): + # Given + testing_vault_secrets.vault.client = hvac_kv_v2_client + + hvac_kv_v2_client.secrets.kv.v2.read_secret_version.side_effect = hvac.exceptions.InvalidPath() + + with pytest.raises(field.RequiredValueMissing) as exception_info: + # When + _ = field(testing_vault_secrets) + + # Then + assert exception_info.value.args[0] == 'keep_it_secret/tests/spam' + + +def test_get_value_not_found_not_required(testing_vault_secrets: Secrets, + hvac_kv_v2_client: mock.Mock, + field: vault.VaultKV2Field): + # Given + testing_vault_secrets.vault.client = hvac_kv_v2_client + + hvac_kv_v2_client.secrets.kv.v2.read_secret_version.side_effect = hvac.exceptions.InvalidPath() + + field = vault.VaultKV2Field( + 'keep_it_secret/', 'tests/spam', required=False, + ) + + result = field(testing_vault_secrets) + + # Then + assert result is None + + +def test_get_value(testing_vault_secrets: Secrets, + hvac_kv_v2_client: mock.Mock, + field: vault.VaultKV2Field): + # Given + testing_vault_secrets.vault.client = hvac_kv_v2_client + + hvac_kv_v2_client.secrets.kv.v2.read_secret_version.return_value = { + 'data': { + 'data': { + 'spam': True, + }, + }, + } + + # When + result = field(testing_vault_secrets) + + # Then + assert result == {'spam': True} diff --git a/tests/ext/vault/test_VaultSecrets.py b/tests/ext/vault/test_VaultSecrets.py new file mode 100644 index 0000000..3f81c3d --- /dev/null +++ b/tests/ext/vault/test_VaultSecrets.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# type: ignore +from __future__ import annotations + +import os +from unittest import mock + +import pytest +from pytest_mock import MockerFixture + +from keep_it_secret.ext import vault + + +@pytest.fixture +def mock_hvac_client(mocker: MockerFixture) -> mock.Mock: + return mocker.patch.object(vault.hvac, 'Client') + + +@pytest.fixture +def hvac_client() -> mock.Mock: + return mock.Mock() + + +def test_init(): + # When + result = vault.VaultSecrets() + + # Then + assert result.client is None + + +@mock.patch.dict( + os.environ, + { + 'VAULT_URL': 'https://vault.work/', + 'VAULT_TOKEN': 'test_vault_token', + 'VAULT_CLIENT_CERT_PATH': '/tmp/vault_client_cert.pem', + 'VAULT_CLIENT_KEY_PATH': '/tmp/vault_client_key.pem', + 'VAULT_SERVER_CERT_PATH': '/tmp/vault_server_cert.pem', + }, +) +def test_as_hvac_client_kwargs(): + # Given + secrets = vault.VaultSecrets() + + # When + result = secrets.as_hvac_client_kwargs() + + # Then + assert result == { + 'url': 'https://vault.work/', + 'token': 'test_vault_token', + 'cert': ('/tmp/vault_client_cert.pem', '/tmp/vault_client_key.pem'), + 'verify': '/tmp/vault_server_cert.pem', + } + + +@mock.patch.dict( + os.environ, + { + 'VAULT_URL': 'https://vault.work/', + 'VAULT_TOKEN': 'test_vault_token', + }, +) +def test_as_hvac_client_kwargs_without_optional_fields(): + # Given + secrets = vault.VaultSecrets() + + # When + result = secrets.as_hvac_client_kwargs() + + # Then + assert result == { + 'url': 'https://vault.work/', + 'token': 'test_vault_token', + } + + +def test_get_client_cache_miss(mock_hvac_client: mock.Mock, + hvac_client: mock.Mock): + # Given + mock_hvac_client.return_value = hvac_client + + secrets = vault.VaultSecrets() + + # When + result = secrets.get_client() + + # Then + assert result == hvac_client + + assert secrets.client == hvac_client + + mock_hvac_client.assert_called_once_with(**secrets.as_hvac_client_kwargs()) + + +def test_get_client_cache_hit(mock_hvac_client: mock.Mock, + hvac_client: mock.Mock): + # Given + secrets = vault.VaultSecrets() + secrets.client = hvac_client + + # When + result = secrets.get_client() + + # Then + assert result == hvac_client + + mock_hvac_client.assert_not_called()