diff --git a/.gitignore b/.gitignore index acc8f26d31..84aeb25c05 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ client/python/poetry.lock /pyproject.toml client/python/.openapi-generator/ client/python/docs/ -client/python/polaris/ +client/python/apache_polaris/sdk/ client/python/test/ !client/python/test/test_cli_parsing.py diff --git a/client/python/apache_polaris/__init__.py b/client/python/apache_polaris/__init__.py new file mode 100644 index 0000000000..d8a500d9d8 --- /dev/null +++ b/client/python/apache_polaris/__init__.py @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + diff --git a/client/python/cli/__init__.py b/client/python/apache_polaris/cli/__init__.py similarity index 100% rename from client/python/cli/__init__.py rename to client/python/apache_polaris/cli/__init__.py diff --git a/client/python/cli/api_client_builder.py b/client/python/apache_polaris/cli/api_client_builder.py similarity index 97% rename from client/python/cli/api_client_builder.py rename to client/python/apache_polaris/cli/api_client_builder.py index 4f2f2c5cf6..7b00a332bd 100644 --- a/client/python/cli/api_client_builder.py +++ b/client/python/apache_polaris/cli/api_client_builder.py @@ -22,7 +22,7 @@ from functools import cached_property from typing import Optional, Dict -from cli.constants import ( +from apache_polaris.cli.constants import ( CONFIG_FILE, CLIENT_PROFILE_ENV, CLIENT_ID_ENV, @@ -33,8 +33,8 @@ DEFAULT_HOSTNAME, DEFAULT_PORT ) -from cli.options.option_tree import Argument -from polaris.management import ApiClient, Configuration +from apache_polaris.cli.options.option_tree import Argument +from apache_polaris.sdk.management import ApiClient, Configuration def _load_profiles() -> Dict[str, str]: diff --git a/client/python/cli/command/__init__.py b/client/python/apache_polaris/cli/command/__init__.py similarity index 92% rename from client/python/cli/command/__init__.py rename to client/python/apache_polaris/cli/command/__init__.py index 8a7d3d84c7..3db3b3ca39 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/apache_polaris/cli/command/__init__.py @@ -19,9 +19,9 @@ import argparse from abc import ABC -from cli.constants import Commands, Arguments -from cli.options.parser import Parser -from polaris.management import PolarisDefaultApi +from apache_polaris.cli.constants import Commands, Arguments +from apache_polaris.cli.options.parser import Parser +from apache_polaris.sdk.management import PolarisDefaultApi class Command(ABC): @@ -44,7 +44,7 @@ def options_get(key, f=lambda x: x): command = None if options.command == Commands.CATALOGS: - from cli.command.catalogs import CatalogsCommand + from apache_polaris.cli.command.catalogs import CatalogsCommand command = CatalogsCommand( options_get(f"{Commands.CATALOGS}_subcommand"), @@ -88,7 +88,7 @@ def options_get(key, f=lambda x: x): catalog_signing_name=options_get(Arguments.CATALOG_SIGNING_NAME) ) elif options.command == Commands.PRINCIPALS: - from cli.command.principals import PrincipalsCommand + from apache_polaris.cli.command.principals import PrincipalsCommand command = PrincipalsCommand( options_get(f"{Commands.PRINCIPALS}_subcommand"), @@ -105,7 +105,7 @@ def options_get(key, f=lambda x: x): new_client_secret=options_get(Arguments.NEW_CLIENT_SECRET), ) elif options.command == Commands.PRINCIPAL_ROLES: - from cli.command.principal_roles import PrincipalRolesCommand + from apache_polaris.cli.command.principal_roles import PrincipalRolesCommand command = PrincipalRolesCommand( options_get(f"{Commands.PRINCIPAL_ROLES}_subcommand"), @@ -120,7 +120,7 @@ def options_get(key, f=lambda x: x): else remove_properties, ) elif options.command == Commands.CATALOG_ROLES: - from cli.command.catalog_roles import CatalogRolesCommand + from apache_polaris.cli.command.catalog_roles import CatalogRolesCommand command = CatalogRolesCommand( options_get(f"{Commands.CATALOG_ROLES}_subcommand"), @@ -134,7 +134,7 @@ def options_get(key, f=lambda x: x): else remove_properties, ) elif options.command == Commands.PRIVILEGES: - from cli.command.privileges import PrivilegesCommand + from apache_polaris.cli.command.privileges import PrivilegesCommand subcommand = options_get(f"{Commands.PRIVILEGES}_subcommand") command = PrivilegesCommand( @@ -151,7 +151,7 @@ def options_get(key, f=lambda x: x): cascade=options_get(Arguments.CASCADE), ) elif options.command == Commands.NAMESPACES: - from cli.command.namespaces import NamespacesCommand + from apache_polaris.cli.command.namespaces import NamespacesCommand subcommand = options_get(f"{Commands.NAMESPACES}_subcommand") command = NamespacesCommand( @@ -165,14 +165,14 @@ def options_get(key, f=lambda x: x): properties=properties, ) elif options.command == Commands.PROFILES: - from cli.command.profiles import ProfilesCommand + from apache_polaris.cli.command.profiles import ProfilesCommand subcommand = options_get(f"{Commands.PROFILES}_subcommand") command = ProfilesCommand( subcommand, profile_name=options_get(Arguments.PROFILE) ) elif options.command == Commands.POLICIES: - from cli.command.policies import PoliciesCommand + from apache_polaris.cli.command.policies import PoliciesCommand subcommand = options_get(f"{Commands.POLICIES}_subcommand") command = PoliciesCommand( diff --git a/client/python/cli/command/catalog_roles.py b/client/python/apache_polaris/cli/command/catalog_roles.py similarity index 96% rename from client/python/cli/command/catalog_roles.py rename to client/python/apache_polaris/cli/command/catalog_roles.py index 3fa7cebb5c..9c96f004dc 100644 --- a/client/python/cli/command/catalog_roles.py +++ b/client/python/apache_polaris/cli/command/catalog_roles.py @@ -21,10 +21,10 @@ from pydantic import StrictStr -from cli.command import Command -from cli.constants import Subcommands, Arguments -from cli.options.option_tree import Argument -from polaris.management import ( +from apache_polaris.cli.command import Command +from apache_polaris.cli.constants import Subcommands, Arguments +from apache_polaris.cli.options.option_tree import Argument +from apache_polaris.sdk.management import ( PolarisDefaultApi, CreateCatalogRoleRequest, CatalogRole, diff --git a/client/python/cli/command/catalogs.py b/client/python/apache_polaris/cli/command/catalogs.py similarity index 96% rename from client/python/cli/command/catalogs.py rename to client/python/apache_polaris/cli/command/catalogs.py index 3af2af608c..0abc9c58ee 100644 --- a/client/python/cli/command/catalogs.py +++ b/client/python/apache_polaris/cli/command/catalogs.py @@ -16,20 +16,27 @@ # specific language governing permissions and limitations # under the License. # -from dataclasses import dataclass -from typing import Dict, List +from apache_polaris.cli.command import Command +from apache_polaris.cli.constants import StorageType, CatalogType, \ + CatalogConnectionType, Subcommands, Arguments, AuthenticationType, \ + ServiceIdentityType +from apache_polaris.cli.options.option_tree import Argument + +from dataclasses import dataclass from pydantic import StrictStr, SecretStr +from typing import Dict, List -from cli.command import Command -from cli.constants import StorageType, CatalogType, CatalogConnectionType, Subcommands, Arguments, AuthenticationType, \ - ServiceIdentityType -from cli.options.option_tree import Argument -from polaris.management import PolarisDefaultApi, CreateCatalogRequest, UpdateCatalogRequest, \ - StorageConfigInfo, ExternalCatalog, AwsStorageConfigInfo, AzureStorageConfigInfo, GcpStorageConfigInfo, \ - PolarisCatalog, CatalogProperties, BearerAuthenticationParameters, ImplicitAuthenticationParameters, \ - OAuthClientCredentialsParameters, SigV4AuthenticationParameters, HadoopConnectionConfigInfo, \ - IcebergRestConnectionConfigInfo, AwsIamServiceIdentityInfo +from apache_polaris.sdk.management import PolarisDefaultApi, \ + CreateCatalogRequest, \ + UpdateCatalogRequest, \ + StorageConfigInfo, ExternalCatalog, AwsStorageConfigInfo, \ + AzureStorageConfigInfo, GcpStorageConfigInfo, \ + PolarisCatalog, CatalogProperties, BearerAuthenticationParameters, \ + ImplicitAuthenticationParameters, \ + OAuthClientCredentialsParameters, SigV4AuthenticationParameters, \ + HadoopConnectionConfigInfo, \ + IcebergRestConnectionConfigInfo, AwsIamServiceIdentityInfo @dataclass diff --git a/client/python/cli/command/namespaces.py b/client/python/apache_polaris/cli/command/namespaces.py similarity index 89% rename from client/python/cli/command/namespaces.py rename to client/python/apache_polaris/cli/command/namespaces.py index e3bf193f56..dbc20fae84 100644 --- a/client/python/cli/command/namespaces.py +++ b/client/python/apache_polaris/cli/command/namespaces.py @@ -18,16 +18,15 @@ # import json from dataclasses import dataclass -from typing import Dict, Optional, List - from pydantic import StrictStr +from typing import Dict, Optional, List -from cli.command import Command -from cli.command.utils import get_catalog_api_client -from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR -from cli.options.option_tree import Argument -from polaris.catalog import IcebergCatalogAPI, CreateNamespaceRequest -from polaris.management import PolarisDefaultApi +from apache_polaris.cli.command import Command +from apache_polaris.cli.command.utils import get_catalog_api_client +from apache_polaris.cli.constants import Subcommands, Arguments, UNIT_SEPARATOR +from apache_polaris.cli.options.option_tree import Argument +from apache_polaris.sdk.catalog import IcebergCatalogAPI, CreateNamespaceRequest +from apache_polaris.sdk.management import PolarisDefaultApi @dataclass diff --git a/client/python/cli/command/policies.py b/client/python/apache_polaris/cli/command/policies.py similarity index 91% rename from client/python/cli/command/policies.py rename to client/python/apache_polaris/cli/command/policies.py index 78ada1768a..437ed08fa3 100644 --- a/client/python/cli/command/policies.py +++ b/client/python/apache_polaris/cli/command/policies.py @@ -16,22 +16,29 @@ # specific language governing permissions and limitations # under the License. # -import os + import json + +import os from dataclasses import dataclass from typing import Optional, Dict -from cli.command import Command -from cli.command.utils import get_catalog_api_client -from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR -from cli.options.option_tree import Argument -from polaris.management import PolarisDefaultApi -from polaris.catalog.api.policy_api import PolicyAPI -from polaris.catalog.models.create_policy_request import CreatePolicyRequest -from polaris.catalog.models.update_policy_request import UpdatePolicyRequest -from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget -from polaris.catalog.models.attach_policy_request import AttachPolicyRequest -from polaris.catalog.models.detach_policy_request import DetachPolicyRequest +from apache_polaris.cli.command import Command +from apache_polaris.cli.command.utils import get_catalog_api_client +from apache_polaris.cli.constants import Subcommands, Arguments, UNIT_SEPARATOR +from apache_polaris.cli.options.option_tree import Argument +from apache_polaris.sdk.catalog.api.policy_api import PolicyAPI +from apache_polaris.sdk.catalog.models.attach_policy_request import \ + AttachPolicyRequest +from apache_polaris.sdk.catalog.models.create_policy_request import \ + CreatePolicyRequest +from apache_polaris.sdk.catalog.models.detach_policy_request import \ + DetachPolicyRequest +from apache_polaris.sdk.catalog.models.policy_attachment_target import \ + PolicyAttachmentTarget +from apache_polaris.sdk.catalog.models.update_policy_request import \ + UpdatePolicyRequest +from apache_polaris.sdk.management import PolarisDefaultApi @dataclass diff --git a/client/python/cli/command/principal_roles.py b/client/python/apache_polaris/cli/command/principal_roles.py similarity index 93% rename from client/python/cli/command/principal_roles.py rename to client/python/apache_polaris/cli/command/principal_roles.py index f9685e38eb..e6e3ee0024 100644 --- a/client/python/cli/command/principal_roles.py +++ b/client/python/apache_polaris/cli/command/principal_roles.py @@ -17,15 +17,15 @@ # under the License. # from dataclasses import dataclass -from typing import Dict, Optional, List - from pydantic import StrictStr +from typing import Dict, Optional, List -from cli.command import Command -from cli.constants import Subcommands, Arguments -from cli.options.option_tree import Argument -from polaris.management import PolarisDefaultApi, CreatePrincipalRoleRequest, PrincipalRole, UpdatePrincipalRoleRequest, \ - GrantPrincipalRoleRequest +from apache_polaris.cli.command import Command +from apache_polaris.cli.constants import Subcommands, Arguments +from apache_polaris.cli.options.option_tree import Argument +from apache_polaris.sdk.management import PolarisDefaultApi, \ + CreatePrincipalRoleRequest, PrincipalRole, UpdatePrincipalRoleRequest, \ + GrantPrincipalRoleRequest @dataclass diff --git a/client/python/cli/command/principals.py b/client/python/apache_polaris/cli/command/principals.py similarity index 98% rename from client/python/cli/command/principals.py rename to client/python/apache_polaris/cli/command/principals.py index c39c221c08..a2a15b3fa4 100644 --- a/client/python/cli/command/principals.py +++ b/client/python/apache_polaris/cli/command/principals.py @@ -22,9 +22,9 @@ from pydantic import StrictStr -from cli.command import Command -from cli.constants import Subcommands -from polaris.management import ( +from apache_polaris.cli.command import Command +from apache_polaris.cli.constants import Subcommands +from apache_polaris.sdk.management import ( PolarisDefaultApi, CreatePrincipalRequest, Principal, diff --git a/client/python/cli/command/privileges.py b/client/python/apache_polaris/cli/command/privileges.py similarity index 96% rename from client/python/cli/command/privileges.py rename to client/python/apache_polaris/cli/command/privileges.py index 6cd19913ff..60d579d5f3 100644 --- a/client/python/cli/command/privileges.py +++ b/client/python/apache_polaris/cli/command/privileges.py @@ -21,10 +21,10 @@ from pydantic import StrictStr -from cli.command import Command -from cli.constants import Subcommands, Actions, Arguments -from cli.options.option_tree import Argument -from polaris.management import ( +from apache_polaris.cli.command import Command +from apache_polaris.cli.constants import Subcommands, Actions, Arguments +from apache_polaris.cli.options.option_tree import Argument +from apache_polaris.sdk.management import ( PolarisDefaultApi, AddGrantRequest, NamespaceGrant, diff --git a/client/python/cli/command/profiles.py b/client/python/apache_polaris/cli/command/profiles.py similarity index 97% rename from client/python/cli/command/profiles.py rename to client/python/apache_polaris/cli/command/profiles.py index 9d40dced51..e005a8300e 100644 --- a/client/python/cli/command/profiles.py +++ b/client/python/apache_polaris/cli/command/profiles.py @@ -16,15 +16,16 @@ # specific language governing permissions and limitations # under the License. # + +import json + import os import sys -import json from dataclasses import dataclass from typing import Dict, Optional, List - -from cli.command import Command -from cli.constants import ( +from apache_polaris.cli.command import Command +from apache_polaris.cli.constants import ( Subcommands, Arguments, DEFAULT_HEADER, @@ -33,7 +34,7 @@ CONFIG_DIR, CONFIG_FILE, ) -from polaris.management import PolarisDefaultApi +from apache_polaris.sdk.management import PolarisDefaultApi @dataclass diff --git a/client/python/cli/command/utils.py b/client/python/apache_polaris/cli/command/utils.py similarity index 88% rename from client/python/cli/command/utils.py rename to client/python/apache_polaris/cli/command/utils.py index b19b562829..259566c403 100644 --- a/client/python/cli/command/utils.py +++ b/client/python/apache_polaris/cli/command/utils.py @@ -18,9 +18,9 @@ # import re -from polaris.catalog.api_client import ApiClient -from polaris.catalog.configuration import Configuration -from polaris.management import PolarisDefaultApi +from apache_polaris.sdk.catalog.api_client import ApiClient +from apache_polaris.sdk.catalog.configuration import Configuration +from apache_polaris.sdk.management import PolarisDefaultApi def get_catalog_api_client(api: PolarisDefaultApi) -> ApiClient: diff --git a/client/python/cli/constants.py b/client/python/apache_polaris/cli/constants.py similarity index 100% rename from client/python/cli/constants.py rename to client/python/apache_polaris/cli/constants.py diff --git a/client/python/cli/options/__init__.py b/client/python/apache_polaris/cli/options/__init__.py similarity index 100% rename from client/python/cli/options/__init__.py rename to client/python/apache_polaris/cli/options/__init__.py diff --git a/client/python/cli/options/option_tree.py b/client/python/apache_polaris/cli/options/option_tree.py similarity index 99% rename from client/python/cli/options/option_tree.py rename to client/python/apache_polaris/cli/options/option_tree.py index 4ab221b14d..4ffe0f7fc2 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/apache_polaris/cli/options/option_tree.py @@ -19,8 +19,9 @@ from dataclasses import dataclass, field from typing import List -from cli.constants import StorageType, CatalogType, PrincipalType, Hints, Commands, Arguments, Subcommands, Actions, \ - CatalogConnectionType, AuthenticationType, ServiceIdentityType +from apache_polaris.cli.constants import StorageType, CatalogType, \ + PrincipalType, Hints, Commands, Arguments, Subcommands, Actions, \ + CatalogConnectionType, AuthenticationType, ServiceIdentityType @dataclass diff --git a/client/python/cli/options/parser.py b/client/python/apache_polaris/cli/options/parser.py similarity index 98% rename from client/python/cli/options/parser.py rename to client/python/apache_polaris/cli/options/parser.py index 3a22440335..40ae46529f 100644 --- a/client/python/cli/options/parser.py +++ b/client/python/apache_polaris/cli/options/parser.py @@ -20,8 +20,8 @@ import sys from typing import List, Optional, Dict -from cli.constants import Arguments, DEFAULT_HEADER -from cli.options.option_tree import OptionTree, Option, Argument +from apache_polaris.cli.constants import Arguments, DEFAULT_HEADER +from apache_polaris.cli.options.option_tree import OptionTree, Option, Argument class Parser(object): diff --git a/client/python/cli/polaris_cli.py b/client/python/apache_polaris/cli/polaris_cli.py similarity index 78% rename from client/python/cli/polaris_cli.py rename to client/python/apache_polaris/cli/polaris_cli.py index 78d8728e34..8bd49d63dd 100644 --- a/client/python/cli/polaris_cli.py +++ b/client/python/apache_polaris/cli/polaris_cli.py @@ -24,10 +24,10 @@ import urllib3 -from cli.api_client_builder import ApiClientBuilder -from cli.constants import Commands -from cli.options.parser import Parser -from polaris.management import PolarisDefaultApi +from apache_polaris.cli.api_client_builder import ApiClientBuilder +from apache_polaris.cli.constants import Commands +from apache_polaris.cli.options.parser import Parser +from apache_polaris.sdk.management import PolarisDefaultApi class PolarisCli: @@ -50,29 +50,29 @@ class PolarisCli: def _patch_generated_models() -> None: """ The OpenAPI generator creates an `api_client` that dynamically looks up - model classes from the `polaris.catalog.models` module using `getattr()`. + model classes from the `apache_polaris.sdk.catalog.models` module using `getattr()`. For example, when a response for a `create_policy` call is received, the deserializer tries to find the `LoadPolicyResponse` class by looking for - `polaris.catalog.models.LoadPolicyResponse`. + `apache_polaris.sdk.catalog.models.LoadPolicyResponse`. However, the generator fails to add the necessary `import` statements - to the `polaris/catalog/models/__init__.py` file. This means that even + to the `apache_polaris/sdk/catalog/models/__init__.py` file. This means that even though the model files exist (e.g., `load_policy_response.py`), the classes - are not part of the `polaris.catalog.models` namespace. + are not part of the `apache_polaris.sdk.catalog.models` namespace. This method works around the bug in the generated code without modifying the source files. It runs once per CLI execution, before any commands, and manually injects the missing response-side model classes into the - `polaris.catalog.models` namespace, allowing the deserializer to find them. + `apache_polaris.sdk.catalog.models` namespace, allowing the deserializer to find them. """ - import polaris.catalog.models - from polaris.catalog.models.applicable_policy import ApplicablePolicy - from polaris.catalog.models.get_applicable_policies_response import GetApplicablePoliciesResponse - from polaris.catalog.models.list_policies_response import ListPoliciesResponse - from polaris.catalog.models.load_policy_response import LoadPolicyResponse - from polaris.catalog.models.policy import Policy - from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget - from polaris.catalog.models.policy_identifier import PolicyIdentifier + import apache_polaris.sdk.catalog.models + from apache_polaris.sdk.catalog.models.applicable_policy import ApplicablePolicy + from apache_polaris.sdk.catalog.models.get_applicable_policies_response import GetApplicablePoliciesResponse + from apache_polaris.sdk.catalog.models.list_policies_response import ListPoliciesResponse + from apache_polaris.sdk.catalog.models.load_policy_response import LoadPolicyResponse + from apache_polaris.sdk.catalog.models.policy import Policy + from apache_polaris.sdk.catalog.models.policy_attachment_target import PolicyAttachmentTarget + from apache_polaris.sdk.catalog.models.policy_identifier import PolicyIdentifier models_to_patch = { "ApplicablePolicy": ApplicablePolicy, @@ -85,14 +85,14 @@ def _patch_generated_models() -> None: } for name, model_class in models_to_patch.items(): - setattr(polaris.catalog.models, name, model_class) + setattr(apache_polaris.sdk.catalog.models, name, model_class) @staticmethod def execute(args=None): PolarisCli._patch_generated_models() options = Parser.parse(args) if options.command == Commands.PROFILES: - from cli.command import Command + from apache_polaris.cli.command import Command command = Command.from_options(options) command.execute() @@ -101,7 +101,7 @@ def execute(args=None): options, direct_authentication=PolarisCli.DIRECT_AUTHENTICATION_ENABLED ).get_api_client() try: - from cli.command import Command + from apache_polaris.cli.command import Command admin_api = PolarisDefaultApi(api_client) command = Command.from_options(options) diff --git a/client/python/apache_polaris/sdk/__init__.py b/client/python/apache_polaris/sdk/__init__.py new file mode 100644 index 0000000000..d8a500d9d8 --- /dev/null +++ b/client/python/apache_polaris/sdk/__init__.py @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + diff --git a/client/python/polaris/.keep b/client/python/apache_polaris/sdk/catalog/.keep similarity index 100% rename from client/python/polaris/.keep rename to client/python/apache_polaris/sdk/catalog/.keep diff --git a/client/python/polaris/catalog/.keep b/client/python/apache_polaris/sdk/catalog/api/.keep similarity index 100% rename from client/python/polaris/catalog/.keep rename to client/python/apache_polaris/sdk/catalog/api/.keep diff --git a/client/python/polaris/catalog/api/.keep b/client/python/apache_polaris/sdk/catalog/models/.keep similarity index 100% rename from client/python/polaris/catalog/api/.keep rename to client/python/apache_polaris/sdk/catalog/models/.keep diff --git a/client/python/polaris/catalog/models/.keep b/client/python/apache_polaris/sdk/management/.keep similarity index 100% rename from client/python/polaris/catalog/models/.keep rename to client/python/apache_polaris/sdk/management/.keep diff --git a/client/python/polaris/management/.keep b/client/python/apache_polaris/sdk/management/api/.keep similarity index 100% rename from client/python/polaris/management/.keep rename to client/python/apache_polaris/sdk/management/api/.keep diff --git a/client/python/polaris/management/api/.keep b/client/python/apache_polaris/sdk/management/models/.keep similarity index 100% rename from client/python/polaris/management/api/.keep rename to client/python/apache_polaris/sdk/management/models/.keep diff --git a/client/python/generate_clients.py b/client/python/generate_clients.py index 0763c494bb..9aaa07fd87 100644 --- a/client/python/generate_clients.py +++ b/client/python/generate_clients.py @@ -44,9 +44,11 @@ # Open API Generator Configs PACKAGE_NAME_POLARIS_MANAGEMENT = ( - "--additional-properties=packageName=polaris.management" + "--additional-properties=packageName=apache_polaris.sdk.management" +) +PACKAGE_NAME_POLARIS_CATALOG = ( + "--additional-properties=packageName=apache_polaris.sdk.catalog" ) -PACKAGE_NAME_POLARIS_CATALOG = "--additional-properties=packageName=polaris.catalog" PYTHON_VERSION = "--additional-properties=pythonVersion=3.10" # Cleanup @@ -59,14 +61,15 @@ Path(".openapi-generator-ignore"), Path(".pytest_cache/"), Path("test/test_cli_parsing.py"), - Path("cli/"), - Path("polaris/__pycache__/"), - Path("polaris/catalog/__pycache__/"), - Path("polaris/catalog/models/__pycache__/"), - Path("polaris/catalog/api/__pycache__/"), - Path("polaris/management/__pycache__/"), - Path("polaris/management/models/__pycache__/"), - Path("polaris/management/api/__pycache__/"), + Path("apache_polaris/__pycache__/"), + Path("apache_polaris/cli/"), + Path("apache_polaris/sdk/__pycache__/"), + Path("apache_polaris/sdk/catalog/__pycache__/"), + Path("apache_polaris/sdk/catalog/models/__pycache__/"), + Path("apache_polaris/sdk/catalog/api/__pycache__/"), + Path("apache_polaris/sdk/management/__pycache__/"), + Path("apache_polaris/sdk/management/models/__pycache__/"), + Path("apache_polaris/sdk/management/api/__pycache__/"), Path("integration_tests/"), Path(".github/workflows/python.yml"), Path(".gitlab-ci.yml"), diff --git a/client/python/integration_tests/conftest.py b/client/python/integration_tests/conftest.py index adfb9d6cd7..d448fb50e1 100644 --- a/client/python/integration_tests/conftest.py +++ b/client/python/integration_tests/conftest.py @@ -27,14 +27,16 @@ from pyiceberg.schema import Schema from pyiceberg.types import NestedField, StringType, IntegerType, BooleanType -from polaris.catalog import OAuthTokenResponse -from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI -from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API -from polaris.catalog.api.policy_api import PolicyAPI -from polaris.catalog.api_client import ApiClient as CatalogApiClient -from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration - -from polaris.management import ( +from apache_polaris.sdk.catalog import OAuthTokenResponse +from apache_polaris.sdk.catalog.api.iceberg_catalog_api import IcebergCatalogAPI +from apache_polaris.sdk.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API +from apache_polaris.sdk.catalog.api.policy_api import PolicyAPI +from apache_polaris.sdk.catalog.api_client import ApiClient as CatalogApiClient +from apache_polaris.sdk.catalog.api_client import ( + Configuration as CatalogApiClientConfiguration, +) + +from apache_polaris.sdk.management import ( Catalog, ApiClient, PolarisDefaultApi, @@ -379,31 +381,37 @@ def format_namespace(namespace: List[str]) -> str: def _patch_generated_models() -> None: """ The OpenAPI generator creates an `api_client` that dynamically looks up - model classes from the `polaris.catalog.models` module using `getattr()`. + model classes from the `apache_polaris.sdk.catalog.models` module using `getattr()`. For example, when a response for a `create_policy` call is received, the deserializer tries to find the `LoadPolicyResponse` class by looking for - `polaris.catalog.models.LoadPolicyResponse`. + `apache_polaris.sdk.catalog.models.LoadPolicyResponse`. However, the generator fails to add the necessary `import` statements - to the `polaris/catalog/models/__init__.py` file. This means that even + to the `apache_polaris/sdk/catalog/models/__init__.py` file. This means that even though the model files exist (e.g., `load_policy_response.py`), the classes - are not part of the `polaris.catalog.models` namespace. + are not part of the `apache_polaris.sdk.catalog.models` namespace. This fixture works around the bug in the generated code without modifying the source files. It runs once per test session, before any tests, and manually injects the missing response-side model classes into the - `polaris.catalog.models` namespace, allowing the deserializer to find them. + `apache_polaris.sdk.catalog.models` namespace, allowing the deserializer to find them. """ - import polaris.catalog.models - from polaris.catalog.models.applicable_policy import ApplicablePolicy - from polaris.catalog.models.get_applicable_policies_response import ( + import apache_polaris.sdk.catalog.models + from apache_polaris.sdk.catalog.models.applicable_policy import ApplicablePolicy + from apache_polaris.sdk.catalog.models.get_applicable_policies_response import ( GetApplicablePoliciesResponse, ) - from polaris.catalog.models.list_policies_response import ListPoliciesResponse - from polaris.catalog.models.load_policy_response import LoadPolicyResponse - from polaris.catalog.models.policy import Policy - from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget - from polaris.catalog.models.policy_identifier import PolicyIdentifier + from apache_polaris.sdk.catalog.models.list_policies_response import ( + ListPoliciesResponse, + ) + from apache_polaris.sdk.catalog.models.load_policy_response import ( + LoadPolicyResponse, + ) + from apache_polaris.sdk.catalog.models.policy import Policy + from apache_polaris.sdk.catalog.models.policy_attachment_target import ( + PolicyAttachmentTarget, + ) + from apache_polaris.sdk.catalog.models.policy_identifier import PolicyIdentifier models_to_patch = { "ApplicablePolicy": ApplicablePolicy, @@ -416,4 +424,4 @@ def _patch_generated_models() -> None: } for name, model_class in models_to_patch.items(): - setattr(polaris.catalog.models, name, model_class) + setattr(apache_polaris.sdk.catalog.models, name, model_class) diff --git a/client/python/integration_tests/test_catalog_apis.py b/client/python/integration_tests/test_catalog_apis.py index 25695b147b..079b2b0618 100644 --- a/client/python/integration_tests/test_catalog_apis.py +++ b/client/python/integration_tests/test_catalog_apis.py @@ -19,28 +19,29 @@ import json import os.path -import time - import pytest +import time +from pyiceberg.catalog import Catalog as PyIcebergCatalog +from pyiceberg.schema import Schema -from integration_tests.conftest import format_namespace -from polaris.catalog import ( +from apache_polaris.sdk.catalog import ( CreateNamespaceRequest, CreateTableRequest, ModelSchema, TableIdentifier, IcebergCatalogAPI, ) -from polaris.catalog.models.attach_policy_request import AttachPolicyRequest -from polaris.catalog.models.create_policy_request import CreatePolicyRequest -from polaris.catalog.models.detach_policy_request import DetachPolicyRequest -from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget -from polaris.catalog.models.update_policy_request import UpdatePolicyRequest -from polaris.catalog.exceptions import NotFoundException -from polaris.catalog.api.policy_api import PolicyAPI -from pyiceberg.schema import Schema -from polaris.management import Catalog -from pyiceberg.catalog import Catalog as PyIcebergCatalog +from apache_polaris.sdk.catalog.api.policy_api import PolicyAPI +from apache_polaris.sdk.catalog.exceptions import NotFoundException +from apache_polaris.sdk.catalog.models.attach_policy_request import AttachPolicyRequest +from apache_polaris.sdk.catalog.models.create_policy_request import CreatePolicyRequest +from apache_polaris.sdk.catalog.models.detach_policy_request import DetachPolicyRequest +from apache_polaris.sdk.catalog.models.policy_attachment_target import ( + PolicyAttachmentTarget, +) +from apache_polaris.sdk.catalog.models.update_policy_request import UpdatePolicyRequest +from apache_polaris.sdk.management import Catalog +from integration_tests.conftest import format_namespace def test_create_namespace( diff --git a/client/python/integration_tests/test_management_apis.py b/client/python/integration_tests/test_management_apis.py index 4256408e5a..d9003f5ff6 100644 --- a/client/python/integration_tests/test_management_apis.py +++ b/client/python/integration_tests/test_management_apis.py @@ -17,12 +17,7 @@ # under the License. # -from integration_tests.conftest import ( - create_principal, - create_principal_role, - create_catalog_role, -) -from polaris.management import ( +from apache_polaris.sdk.management import ( GrantPrincipalRoleRequest, GrantCatalogRoleRequest, AddGrantRequest, @@ -33,6 +28,11 @@ Catalog, ResetPrincipalRequest, ) +from integration_tests.conftest import ( + create_principal, + create_principal_role, + create_catalog_role, +) def test_principals(management_client: PolarisDefaultApi) -> None: diff --git a/client/python/polaris/management/models/.keep b/client/python/polaris/management/models/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml index d5f7011899..02bbb0b5ea 100644 --- a/client/python/pyproject.toml +++ b/client/python/pyproject.toml @@ -18,7 +18,7 @@ # [project] -name = "polaris" +name = "apache-polaris" version = "1.2.0" description = "Apache Polaris" authors = [ @@ -42,13 +42,16 @@ homepage = "https://polaris.apache.org/" repository = "https://github.com/apache/polaris/" [project.scripts] -polaris = "cli.polaris_cli:main" +polaris = "apache_polaris.cli.polaris_cli:main" [tool.poetry] requires-poetry = "==2.2.1" +packages = [ + { include = "apache_polaris" } +] include = [ - { path = "polaris/**", format = ["sdist", "wheel"] }, - { path = "cli/**", format = ["sdist", "wheel"] }, + { path = "apache_polaris/cli", format = ["sdist", "wheel"] }, + { path = "apache_polaris/sdk", format = ["sdist", "wheel"] }, { path = "templates/**", format = "sdist" }, { path = "spec/**", format = "sdist" }, ] diff --git a/client/python/test/test_cli_parsing.py b/client/python/test/test_cli_parsing.py index af5f8e28b3..168b8e17da 100644 --- a/client/python/test/test_cli_parsing.py +++ b/client/python/test/test_cli_parsing.py @@ -17,16 +17,15 @@ # under the License. # -import unittest import io +import unittest from functools import reduce from typing import List from unittest.mock import patch, MagicMock -from cli.command import Command -from cli.options.parser import Parser -from polaris.catalog import ApiClient -from polaris.management import PolarisDefaultApi +from apache_polaris.cli.command import Command +from apache_polaris.cli.options.parser import Parser +from apache_polaris.sdk.management import PolarisDefaultApi INVALID_ARGS = 2 diff --git a/getting-started/spark/notebooks/SparkPolaris.ipynb b/getting-started/spark/notebooks/SparkPolaris.ipynb index 11d190f941..0d51a1c0ea 100644 --- a/getting-started/spark/notebooks/SparkPolaris.ipynb +++ b/getting-started/spark/notebooks/SparkPolaris.ipynb @@ -16,10 +16,10 @@ "metadata": {}, "outputs": [], "source": [ - "from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI\n", - "from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API\n", - "from polaris.catalog.api_client import ApiClient as CatalogApiClient\n", - "from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration\n", + "from apache_polaris.sdk.catalog.api.iceberg_catalog_api import IcebergCatalogAPI\n", + "from apache_polaris.sdk.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API\n", + "from apache_polaris.sdk.catalog.api_client import ApiClient as CatalogApiClient\n", + "from apache_polaris.sdk.catalog.api_client import Configuration as CatalogApiClientConfiguration\n", "\n", "polaris_credential = 'root:s3cr3t' # pragma: allowlist secret\n", "\n", @@ -47,13 +47,12 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "0f7a311a-9a55-4ff7-a40e-db3c74c53b9b", "metadata": {}, + "cell_type": "code", "outputs": [], + "execution_count": null, "source": [ - "from polaris.management import *\n", + "from apache_polaris.sdk.management import *\n", "\n", "client = ApiClient(Configuration(access_token=token.access_token,\n", " host='http://polaris:8181/api/management/v1'))\n", @@ -67,7 +66,8 @@ "root_client.create_catalog(create_catalog_request=CreateCatalogRequest(catalog=catalog))\n", "resp = root_client.get_catalog(catalog_name=catalog.name)\n", "resp" - ] + ], + "id": "df281b1e1a131eb0" }, { "cell_type": "markdown", diff --git a/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb b/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb index 69e7f34316..f4e4a00bb4 100644 --- a/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb +++ b/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb @@ -16,9 +16,9 @@ "metadata": {}, "outputs": [], "source": [ - "from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API\n", - "from polaris.catalog.api_client import ApiClient as CatalogApiClient\n", - "from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration\n", + "from apache_polaris.sdk.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API\n", + "from apache_polaris.sdk.catalog.api_client import ApiClient as CatalogApiClient\n", + "from apache_polaris.sdk.catalog.api_client import Configuration as CatalogApiClientConfiguration\n", "\n", "polaris_credential = 'root:s3cr3t' # pragma: allowlist secret\n", "\n", @@ -63,7 +63,7 @@ } ], "source": [ - "from polaris.management import *\n", + "from apache_polaris.sdk.management import *\n", "\n", "client = ApiClient(Configuration(access_token=token.access_token,\n", " host='http://polaris:8181/api/management/v1'))\n", diff --git a/regtests/.dockerignore b/regtests/.dockerignore index 81993676bf..03365d249b 100644 --- a/regtests/.dockerignore +++ b/regtests/.dockerignore @@ -20,5 +20,4 @@ .python-version .pytest_cache t_pyspark/src/spark-warehouse -t_pyspark/src/.pytest_cache -polaris/polaris.management.egg-info \ No newline at end of file +t_pyspark/src/.pytest_cache \ No newline at end of file diff --git a/regtests/t_pyspark/src/conftest.py b/regtests/t_pyspark/src/conftest.py index d45363f6c5..ea93267315 100644 --- a/regtests/t_pyspark/src/conftest.py +++ b/regtests/t_pyspark/src/conftest.py @@ -24,9 +24,9 @@ import pytest -from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI -from polaris.catalog.api_client import ApiClient as CatalogApiClient -from polaris.management import Catalog, AwsStorageConfigInfo, ApiClient, PolarisDefaultApi, Configuration, \ +from apache_polaris.sdk.catalog.api.iceberg_catalog_api import IcebergCatalogAPI +from apache_polaris.sdk.catalog.api_client import ApiClient as CatalogApiClient +from apache_polaris.sdk.management import Catalog, AwsStorageConfigInfo, ApiClient, PolarisDefaultApi, Configuration, \ CreateCatalogRequest, GrantCatalogRoleRequest, CatalogRole, ApiException, AddGrantRequest, CatalogGrant, \ CatalogPrivilege, CreateCatalogRoleRequest, CatalogProperties @@ -192,7 +192,7 @@ def _create_catalog_with_storage(root_client, catalog_client, catalog_name, stor storage_config_info: Storage configuration (S3 or FILE) base_location: Base location for the catalog """ - from polaris.management import AwsStorageConfigInfo + from apache_polaris.sdk.management import AwsStorageConfigInfo # Build properties dict catalog_properties = { @@ -256,7 +256,7 @@ def file_catalog(root_client, catalog_client): Catalog that always uses FILE storage for local testing. This fixture runs in any environment without external dependencies. """ - from polaris.management import FileStorageConfigInfo + from apache_polaris.sdk.management import FileStorageConfigInfo catalog_name = f'file_catalog_{str(uuid.uuid4())[-10:]}' storage_config = FileStorageConfigInfo(storage_type="FILE", allowed_locations=["file:///tmp"]) @@ -273,7 +273,7 @@ def s3_catalog(root_client, catalog_client, test_bucket, aws_role_arn, aws_bucke Catalog that always uses S3 storage for AWS testing. Tests using this fixture should include @pytest.mark.skipif for AWS_TEST_ENABLED. """ - from polaris.management import AwsStorageConfigInfo + from apache_polaris.sdk.management import AwsStorageConfigInfo catalog_name = f's3_catalog_{str(uuid.uuid4())[-10:]}' storage_config = AwsStorageConfigInfo( diff --git a/regtests/t_pyspark/src/test_spark_sql_fine_grained_authz.py b/regtests/t_pyspark/src/test_spark_sql_fine_grained_authz.py index b6b47203c7..cdedc7ee04 100644 --- a/regtests/t_pyspark/src/test_spark_sql_fine_grained_authz.py +++ b/regtests/t_pyspark/src/test_spark_sql_fine_grained_authz.py @@ -31,7 +31,7 @@ from py4j.protocol import Py4JJavaError from iceberg_spark import IcebergSparkSession -from polaris.management import PrincipalRole, CatalogRole, CatalogGrant, CatalogPrivilege, \ +from apache_polaris.sdk.management import PrincipalRole, CatalogRole, CatalogGrant, CatalogPrivilege, \ AddGrantRequest, GrantCatalogRoleRequest, GrantPrincipalRoleRequest # Import existing helper functions instead of copying them @@ -45,7 +45,7 @@ def fine_grained_authz_test_catalog(root_client, catalog_client): Catalog specifically for fine-grained authorization testing. Does NOT assign catalog_admin to service_admin to avoid privilege inheritance issues. """ - from polaris.management import FileStorageConfigInfo, Catalog, CatalogProperties, CreateCatalogRequest + from apache_polaris.sdk.management import FileStorageConfigInfo, Catalog, CatalogProperties, CreateCatalogRequest from conftest import create_catalog_role catalog_name = f'fine_grained_authz_test_catalog_{str(uuid.uuid4())[-10:]}' diff --git a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py index ae1f71c8b3..d26041cd6e 100644 --- a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py +++ b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py @@ -30,13 +30,13 @@ from iceberg_spark import IcebergSparkSession -from polaris.catalog import CreateNamespaceRequest, CreateTableRequest, ModelSchema -from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI -from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API -from polaris.catalog.api_client import ApiClient as CatalogApiClient -from polaris.catalog.configuration import Configuration -from polaris.management import ApiClient as ManagementApiClient -from polaris.management import PolarisDefaultApi, Principal, PrincipalRole, CatalogRole, \ +from apache_polaris.sdk.catalog import CreateNamespaceRequest, CreateTableRequest, ModelSchema +from apache_polaris.sdk.catalog.api.iceberg_catalog_api import IcebergCatalogAPI +from apache_polaris.sdk.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API +from apache_polaris.sdk.catalog.api_client import ApiClient as CatalogApiClient +from apache_polaris.sdk.catalog.configuration import Configuration +from apache_polaris.sdk.management import ApiClient as ManagementApiClient +from apache_polaris.sdk.management import PolarisDefaultApi, Principal, PrincipalRole, CatalogRole, \ CatalogGrant, CatalogPrivilege, ApiException, CreateCatalogRoleRequest, CreatePrincipalRoleRequest, \ CreatePrincipalRequest, AddGrantRequest, GrantCatalogRoleRequest, GrantPrincipalRoleRequest, UpdateCatalogRequest