Merge pull request #283 from dbt-msft/v1.2.0

first beta version for v1.2.0

Author: sdebruyn

.github/workflows/integration-tests-azure.yml

@@ -4,6 +4,7 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - master
+      - v*
   pull_request_target:
     types: [labeled]
 

.github/workflows/integration-tests-sqlserver.yml

@@ -4,9 +4,11 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - master
+      - v*
   pull_request:
     branches:
       - master
+      - v*
 
 jobs:
   integration-tests-sql-server:

.github/workflows/unit-tests.yml

@@ -4,9 +4,11 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - master
+      - v*
   pull_request:
     branches:
       - master
+      - v*
 
 jobs:
   unit-tests:

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
       - id: mixed-line-ending
       - id: check-docstring-first
   - repo: 'https://github.com/adrienverge/yamllint'
-    rev: v1.26.3
+    rev: v1.27.1
     hooks:
       - id: yamllint
         args:
@@ -32,7 +32,7 @@ repos:
     hooks:
       - id: absolufy-imports
   - repo: 'https://github.com/hadialqattan/pycln'
-    rev: v1.3.5
+    rev: v2.1.1
     hooks:
       - id: pycln
         args:
@@ -50,7 +50,7 @@ repos:
           - '--python-version'
           - '39'
   - repo: 'https://github.com/psf/black'
-    rev: 22.6.0
+    rev: 22.8.0
     hooks:
       - id: black
         args:
@@ -78,7 +78,7 @@ repos:
         stages:
           - manual
   - repo: 'https://github.com/pre-commit/mirrors-mypy'
-    rev: v0.961
+    rev: v0.971
     hooks:
       - id: mypy
         args:

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+### v1.2.0 (betas)
+
+#### Features
+
+* Support for [dbt-core 1.2](https://github.com/dbt-labs/dbt-core/releases/tag/v1.2.0)
+* Support automatic retries with new `retries` setting introduced in core
+* The correct owner of a table/view is now visible in generated documentation (and in catalog.json)
+* A lot of features of dbt-utils & T-SQL utils are now available out-of-the-box in dbt-core and this adapter. A new release of T-SQL utils will follow.
+  *  Support for all `type_*` macros
+  *  Support for all [cross-database macros](https://docs.getdbt.com/reference/dbt-jinja-functions/cross-database-macros), except:
+    *  `bool_or`
+    *  `listagg` will only work in SQL Server 2017 or newer or the cloud versions. The `limit_num` option is unsupported. `DISTINCT` cannot be used in the measure.
+
+#### Fixes
+
+* In some cases the `TIMESTAMP` would be used as data type instead of `DATETIMEOFFSET`, fixed that
+
+#### Chores
+
+* Update adapter testing framework to 1.2.1
+* Update pre-commit, tox, pytest and pre-commit hooks
+* Type hinting in connection class
+
+#### Outstanding work for next beta release
+
+* Native dbt support for GRANTs as configs
+* Add documentation about new features to official dbt docs pages
+
 ### v1.1.0
 
 See changes included in v1.1.0rc1 below as well

README.md

@@ -24,7 +24,7 @@ This adapter requires the Microsoft ODBC driver to be installed:
 <details><summary>Debian/Ubuntu</summary>
 <p>
 
-Make sure to install the ODBC headers:
+Make sure to install the ODBC headers as well as the driver linked above:
 
 ```shell
 sudo apt-get install -y unixodbc-dev

dbt/adapters/sqlserver/__init__.py

@@ -1,7 +1,8 @@
 from dbt.adapters.base import AdapterPlugin
 
 from dbt.adapters.sqlserver.connections import SQLServerConnectionManager, SQLServerCredentials
-from dbt.adapters.sqlserver.impl import SQLServerAdapter
+from dbt.adapters.sqlserver.sql_server_adapter import SQLServerAdapter
+from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn
 from dbt.include import sqlserver
 
 Plugin = AdapterPlugin(
@@ -10,4 +11,10 @@
     include_path=sqlserver.PACKAGE_PATH,
 )
 
-__all__ = ["Plugin", "SQLServerConnectionManager", "SQLServerAdapter", "SQLServerCredentials"]
+__all__ = [
+    "Plugin",
+    "SQLServerConnectionManager",
+    "SQLServerColumn",
+    "SQLServerAdapter",
+    "SQLServerCredentials",
+]

dbt/adapters/sqlserver/__version__.py

@@ -1 +1 @@
-version = "1.1.0"
+version = "1.2.0b1"

dbt/adapters/sqlserver/connections.py

@@ -3,8 +3,9 @@
 from contextlib import contextmanager
 from dataclasses import dataclass
 from itertools import chain, repeat
-from typing import Callable, Dict, Mapping, Optional
+from typing import Any, Callable, Dict, Mapping, Optional, Tuple
 
+import agate
 import dbt.exceptions
 import pyodbc
 from azure.core.credentials import AccessToken
@@ -17,7 +18,8 @@
 )
 from dbt.adapters.base import Credentials
 from dbt.adapters.sql import SQLConnectionManager
-from dbt.contracts.connection import AdapterResponse
+from dbt.clients.agate_helper import empty_table
+from dbt.contracts.connection import AdapterResponse, Connection, ConnectionState
 from dbt.events import AdapterLogger
 
 from dbt.adapters.sqlserver import __version__
@@ -46,6 +48,7 @@ class SQLServerCredentials(Credentials):
     authentication: Optional[str] = "sql"
     encrypt: Optional[bool] = False
     trust_cert: Optional[bool] = False
+    retries: int = 1
 
     _ALIASES = {
         "user": "UID",
@@ -287,7 +290,6 @@ def exception_handler(self, sql):
                 self.release()
             except pyodbc.Error:
                 logger.debug("Failed to release connection!")
-                pass
 
             raise dbt.exceptions.DatabaseException(str(e).strip()) from e
 
@@ -304,69 +306,73 @@ def exception_handler(self, sql):
             raise dbt.exceptions.RuntimeException(e)
 
     @classmethod
-    def open(cls, connection):
+    def open(cls, connection: Connection) -> Connection:
 
-        if connection.state == "open":
+        if connection.state == ConnectionState.OPEN:
             logger.debug("Connection is already open, skipping open.")
             return connection
 
-        credentials = connection.credentials
+        credentials = cls.get_credentials(connection.credentials)
 
-        try:
-            con_str = []
-            con_str.append(f"DRIVER={{{credentials.driver}}}")
-
-            if "\\" in credentials.host:
+        con_str = [f"DRIVER={{{credentials.driver}}}"]
 
-                # If there is a backslash \ in the host name, the host is a
-                # SQL Server named instance. In this case then port number has to be omitted.
-                con_str.append(f"SERVER={credentials.host}")
-            else:
-                con_str.append(f"SERVER={credentials.host},{credentials.port}")
+        if "\\" in credentials.host:
 
-            con_str.append(f"Database={credentials.database}")
+            # If there is a backslash \ in the host name, the host is a
+            # SQL Server named instance. In this case then port number has to be omitted.
+            con_str.append(f"SERVER={credentials.host}")
+        else:
+            con_str.append(f"SERVER={credentials.host},{credentials.port}")
 
-            type_auth = getattr(credentials, "authentication", "sql")
+        con_str.append(f"Database={credentials.database}")
 
-            if "ActiveDirectory" in type_auth:
-                con_str.append(f"Authentication={credentials.authentication}")
+        type_auth = getattr(credentials, "authentication", "sql")
 
-                if type_auth == "ActiveDirectoryPassword":
-                    con_str.append(f"UID={{{credentials.UID}}}")
-                    con_str.append(f"PWD={{{credentials.PWD}}}")
-                elif type_auth == "ActiveDirectoryInteractive":
-                    con_str.append(f"UID={{{credentials.UID}}}")
+        if "ActiveDirectory" in type_auth:
+            con_str.append(f"Authentication={credentials.authentication}")
 
-            elif getattr(credentials, "windows_login", False):
-                con_str.append("trusted_connection=yes")
-            elif type_auth == "sql":
+            if type_auth == "ActiveDirectoryPassword":
                 con_str.append(f"UID={{{credentials.UID}}}")
                 con_str.append(f"PWD={{{credentials.PWD}}}")
+            elif type_auth == "ActiveDirectoryInteractive":
+                con_str.append(f"UID={{{credentials.UID}}}")
+
+        elif getattr(credentials, "windows_login", False):
+            con_str.append("trusted_connection=yes")
+        elif type_auth == "sql":
+            con_str.append(f"UID={{{credentials.UID}}}")
+            con_str.append(f"PWD={{{credentials.PWD}}}")
+
+        # still confused whether to use "Yes", "yes", "True", or "true"
+        # to learn more visit
+        # https://docs.microsoft.com/en-us/sql/relational-databases/native-client/features/using-encryption-without-validation?view=sql-server-ver15
+        if getattr(credentials, "encrypt", False) is True:
+            con_str.append("Encrypt=Yes")
+            if getattr(credentials, "trust_cert", False) is True:
+                con_str.append("TrustServerCertificate=Yes")
 
-            # still confused whether to use "Yes", "yes", "True", or "true"
-            # to learn more visit
-            # https://docs.microsoft.com/en-us/sql/relational-databases/native-client/features/using-encryption-without-validation?view=sql-server-ver15
-            if getattr(credentials, "encrypt", False) is True:
-                con_str.append("Encrypt=Yes")
-                if getattr(credentials, "trust_cert", False) is True:
-                    con_str.append("TrustServerCertificate=Yes")
+        plugin_version = __version__.version
+        application_name = f"dbt-{credentials.type}/{plugin_version}"
+        con_str.append(f"Application Name={application_name}")
 
-            plugin_version = __version__.version
-            application_name = f"dbt-{credentials.type}/{plugin_version}"
-            con_str.append(f"Application Name={application_name}")
+        con_str_concat = ";".join(con_str)
 
-            con_str_concat = ";".join(con_str)
+        index = []
+        for i, elem in enumerate(con_str):
+            if "pwd=" in elem.lower():
+                index.append(i)
 
-            index = []
-            for i, elem in enumerate(con_str):
-                if "pwd=" in elem.lower():
-                    index.append(i)
+        if len(index) != 0:
+            con_str[index[0]] = "PWD=***"
 
-            if len(index) != 0:
-                con_str[index[0]] = "PWD=***"
+        con_str_display = ";".join(con_str)
 
-            con_str_display = ";".join(con_str)
+        retryable_exceptions = [  # https://github.com/mkleehammer/pyodbc/wiki/Exceptions
+            pyodbc.InternalError,  # not used according to docs, but defined in PEP-249
+            pyodbc.OperationalError,
+        ]
 
+        def connect():
             logger.debug(f"Using connection string: {con_str_display}")
 
             attrs_before = get_pyodbc_attrs_before(credentials)
@@ -375,24 +381,19 @@ def open(cls, connection):
                 attrs_before=attrs_before,
                 autocommit=True,
             )
-
-            connection.state = "open"
-            connection.handle = handle
             logger.debug(f"Connected to db: {credentials.database}")
+            return handle
+
+        return cls.retry_connection(
+            connection,
+            connect=connect,
+            logger=logger,
+            retry_limit=credentials.retries,
+            retryable_exceptions=retryable_exceptions,
+        )
 
-        except pyodbc.Error as e:
-            logger.debug(f"Could not connect to db: {e}")
-
-            connection.handle = None
-            connection.state = "fail"
-
-            raise dbt.exceptions.FailedToConnectException(str(e))
-
-        return connection
-
-    def cancel(self, connection):
+    def cancel(self, connection: Connection):
         logger.debug("Cancel query")
-        pass
 
     def add_begin_query(self):
         # return self.add_query('BEGIN TRANSACTION', auto_begin=False)
@@ -402,7 +403,13 @@ def add_commit_query(self):
         # return self.add_query('COMMIT TRANSACTION', auto_begin=False)
         pass
 
-    def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False):
+    def add_query(
+        self,
+        sql: str,
+        auto_begin: bool = True,
+        bindings: Optional[Any] = None,
+        abridge_sql_log: bool = False,
+    ) -> Tuple[Connection, Any]:
 
         connection = self.get_thread_connection()
 
@@ -435,11 +442,11 @@ def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False):
             return connection, cursor
 
     @classmethod
-    def get_credentials(cls, credentials):
+    def get_credentials(cls, credentials: SQLServerCredentials) -> SQLServerCredentials:
         return credentials
 
     @classmethod
-    def get_response(cls, cursor) -> AdapterResponse:
+    def get_response(cls, cursor: Any) -> AdapterResponse:
         # message = str(cursor.statusmessage)
         message = "OK"
         rows = cursor.rowcount
@@ -456,7 +463,9 @@ def get_response(cls, cursor) -> AdapterResponse:
             rows_affected=rows,
         )
 
-    def execute(self, sql, auto_begin=True, fetch=False):
+    def execute(
+        self, sql: str, auto_begin: bool = True, fetch: bool = False
+    ) -> Tuple[AdapterResponse, agate.Table]:
         _, cursor = self.add_query(sql, auto_begin)
         response = self.get_response(cursor)
         if fetch:
@@ -466,7 +475,7 @@ def execute(self, sql, auto_begin=True, fetch=False):
                     break
             table = self.get_result_from_cursor(cursor)
         else:
-            table = dbt.clients.agate_helper.empty_table()
+            table = empty_table()
         # Step through all result sets so we process all errors
         while cursor.nextset():
             pass

dbt/adapters/sqlserver/impl.py renamed to dbt/adapters/sqlserver/sql_server_adapter.py

@@ -4,11 +4,13 @@
 from dbt.adapters.base.relation import BaseRelation
 from dbt.adapters.sql import SQLAdapter
 
-from dbt.adapters.sqlserver import SQLServerConnectionManager
+from dbt.adapters.sqlserver.connections import SQLServerConnectionManager
+from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn
 
 
 class SQLServerAdapter(SQLAdapter):
     ConnectionManager = SQLServerConnectionManager
+    Column = SQLServerColumn
 
     @classmethod
     def date_function(cls):