fix(kernel): Thrift-parity fixes + GetArrowBatches + U2M scopes (TIMESTAMP, DDL, User-Agent, VOID, Arrow batches, U2M scopes, auth logging)

auth/oauth/m2m/m2m.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/databricks/databricks-sql-go/auth"
 	"github.com/databricks/databricks-sql-go/auth/oauth"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 )
@@ -72,12 +72,14 @@ func (c *authClient) Authenticate(r *http.Request) error {
 
 	c.tokenSource = GetTokenSource(config)
 	token, err := c.tokenSource.Token()
-	log.Debug().Msgf("databricks OAuth token fetched successfully")
 	if err != nil {
-		log.Err(err).Msg("failed to get token")
+		logger.Err(err).Msg("failed to get token")
 
 		return err
 	}
+	// Log via the driver's configurable logger (defaults to Warn) rather than the
+	// global zerolog logger, so this line honors SetLogLevel like every other.
+	logger.Debug().Msgf("databricks OAuth token fetched successfully")
 	token.SetAuthHeader(r)
 
 	return nil

auth/oauth/oauth.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-oidc/v3/oidc"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 	"golang.org/x/oauth2"
 )
 
@@ -99,21 +99,21 @@ func resolveOIDCIssuer(ctx context.Context, client *http.Client, hostName string
 	cfgURL := fmt.Sprintf("https://%s/.well-known/databricks-config", hostName)
 	meta, ok := fetchHostMetadata(ctx, client, cfgURL)
 	if !ok || meta.OIDCEndpoint == "" {
-		log.Debug().Msgf("oauth: no usable databricks-config for %q; using bare-host OIDC issuer", hostName)
+		logger.Debug().Msgf("oauth: no usable databricks-config for %q; using bare-host OIDC issuer", hostName)
 		return fallback
 	}
 
 	// An account-rooted endpoint needs a non-empty account_id; otherwise the
 	// placeholder would resolve to a malformed ".../accounts/" issuer. Fall back
 	// rather than emit it (the function's documented contract).
 	if strings.Contains(meta.OIDCEndpoint, accountIDPlaceholder) && meta.AccountID == "" {
-		log.Warn().Msgf("oauth: databricks-config for %q has an %s placeholder but empty account_id; using bare-host OIDC issuer", hostName, accountIDPlaceholder)
+		logger.Warn().Msgf("oauth: databricks-config for %q has an %s placeholder but empty account_id; using bare-host OIDC issuer", hostName, accountIDPlaceholder)
 		return fallback
 	}
 
 	issuer := substituteAccountID(meta)
 	if !isValidDatabricksIssuer(issuer) {
-		log.Warn().Msgf("oauth: databricks-config for %q advertised an unusable oidc_endpoint %q; using bare-host OIDC issuer", hostName, issuer)
+		logger.Warn().Msgf("oauth: databricks-config for %q advertised an unusable oidc_endpoint %q; using bare-host OIDC issuer", hostName, issuer)
 		return fallback
 	}
 	return issuer

auth/oauth/oauth_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -62,6 +63,28 @@ func TestGetAzureDnsZone(t *testing.T) {
 	}
 }
 
+// TestGetScopes pins the literal cloud-specific scope sets (not GetScopes's own
+// output) so a regression in the AWS/GCP-vs-Azure branching — the source of the
+// kernel U2M access_denied incident — is caught rather than asserted tautologically.
+func TestGetScopes(t *testing.T) {
+	cases := []struct {
+		name string
+		host string
+		want []string
+	}{
+		{"AWS", "dbc-1234.cloud.databricks.com", []string{"offline_access", "sql"}},
+		{"GCP", "x.gcp.databricks.com", []string{"offline_access", "sql"}},
+		{"Azure", "adb-123.azuredatabricks.net", []string{"offline_access", "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/user_impersonation"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := GetScopes(tc.host, nil); !reflect.DeepEqual(got, tc.want) {
+				t.Fatalf("GetScopes(%q, nil) = %v, want %v", tc.host, got, tc.want)
+			}
+		})
+	}
+}
+
 // TestResolveOIDCIssuer drives resolveOIDCIssuer end-to-end against an httptest
 // server: the server stands in for the connection host, so a 200 with a given
 // databricks-config body exercises the real fetch + substitution + validation +

auth/oauth/u2m/authenticator.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/databricks/databricks-sql-go/auth"
 	"github.com/databricks/databricks-sql-go/auth/oauth"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 	"golang.org/x/oauth2"
 )
 
@@ -84,15 +84,9 @@ type u2mAuthenticator struct {
 // mode. It structurally satisfies the U2MCredentialsProvider interface the kernel
 // backend asserts (defined in internal/backend/kernel, off the public API).
 //
-// Only the client id crosses to the kernel — NOT the scopes. The Thrift path
-// requests offline_access + sql (AWS/GCP) or offline_access + <tenant>/
-// user_impersonation (Azure) via oauth.GetScopes, while the kernel's U2M flow
-// applies its own default set (all-apis + offline_access). Neither backend exposes
-// a user-facing U2M-scopes option, so this is a fixed-vs-fixed difference, not a
-// dropped caller choice; against the built-in databricks-sql-connector public
-// client (which all-apis targets) both authorize successfully. If a U2M-scopes
-// option is ever added, forward it via kernel.Auth.Scopes (already wired through
-// set_auth_u2m) so the two paths request the same set.
+// Only the client id is read here; resolveKernelAuth pairs it with the same
+// oauth.GetScopes set the Thrift path requests, so both backends authorize against
+// the built-in databricks-sql-connector client identically (not the kernel default).
 func (c *u2mAuthenticator) U2MClientID() string { return c.clientID }
 
 // Auth will start the OAuth Authorization Flow to authenticate the cli client
@@ -160,7 +154,7 @@ func (tsp *tokenSourceProvider) GetTokenSource() (oauth2.TokenSource, error) {
 	loginURL := tsp.config.AuthCodeURL(state, challenge, challengeMethod)
 	tsp.state = state
 
-	log.Info().Msgf("listening on %s://%s/", tsp.redirectURL.Scheme, tsp.redirectURL.Host)
+	logger.Info().Msgf("listening on %s://%s/", tsp.redirectURL.Scheme, tsp.redirectURL.Host)
 	listener, err := net.Listen("tcp", tsp.redirectURL.Host)
 	if err != nil {
 		return nil, err
@@ -225,28 +219,28 @@ func (tsp *tokenSourceProvider) ServeHTTP(w http.ResponseWriter, r *http.Request
 
 	// Do some checking of the response here to show more relevant content
 	if resp.err != "" {
-		log.Error().Msg(resp.err)
+		logger.Error().Msg(resp.err)
 		w.WriteHeader(http.StatusBadRequest)
 		_, err := w.Write([]byte(errorHTML("Identity Provider returned an error: " + resp.err))) //nolint:gosec // XSS not a concern for local OAuth callback
 		if err != nil {
-			log.Error().Err(err).Msg("unable to write error response")
+			logger.Error().Err(err).Msg("unable to write error response")
 		}
 		return
 	}
 	if resp.state != tsp.state && r.URL.String() != "/favicon.ico" {
 		msg := "Authentication state received did not match original request. Please try to login again."
-		log.Error().Msg(msg)
+		logger.Error().Msg(msg)
 		w.WriteHeader(http.StatusBadRequest)
 		_, err := w.Write([]byte(errorHTML(msg)))
 		if err != nil {
-			log.Error().Err(err).Msg("unable to write error response")
+			logger.Error().Err(err).Msg("unable to write error response")
 		}
 		return
 	}
 
 	_, err := w.Write([]byte(infoHTML("CLI Login Success", "You may close this window anytime now and go back to terminal")))
 	if err != nil {
-		log.Error().Err(err).Msg("unable to write success response")
+		logger.Error().Err(err).Msg("unable to write success response")
 	}
 }
 

auth/tokenprovider/authenticator.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 
 	"github.com/databricks/databricks-sql-go/auth"
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 )
 
 // TokenProviderAuthenticator implements auth.Authenticator using a TokenProvider.
@@ -47,7 +47,7 @@ func (a *TokenProviderAuthenticator) Authenticate(r *http.Request) error {
 	}
 
 	token.SetAuthHeader(r)
-	log.Debug().Msgf("token provider authenticator: authenticated using provider %s", a.provider.Name())
+	logger.Debug().Msgf("token provider authenticator: authenticated using provider %s", a.provider.Name())
 
 	return nil
 }

auth/tokenprovider/cached.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rs/zerolog/log"
+	"github.com/databricks/databricks-sql-go/logger"
 )
 
 // CachedTokenProvider wraps another provider and caches tokens
@@ -43,7 +43,7 @@ func (p *CachedTokenProvider) GetToken(ctx context.Context) (*Token, error) {
 
 	// If cache is valid and not being refreshed, return a copy
 	if cached != nil && !needsRefresh {
-		log.Debug().Msgf("cached token provider: using cached token for provider %s", p.provider.Name())
+		logger.Debug().Msgf("cached token provider: using cached token for provider %s", p.provider.Name())
 		// Return a copy to avoid concurrent modification issues
 		return copyToken(cached), nil
 	}
@@ -75,7 +75,7 @@ func (p *CachedTokenProvider) GetToken(ctx context.Context) (*Token, error) {
 	p.mutex.Unlock()
 
 	// Fetch new token WITHOUT holding the lock
-	log.Debug().Msgf("cached token provider: fetching new token from provider %s", p.provider.Name())
+	logger.Debug().Msgf("cached token provider: fetching new token from provider %s", p.provider.Name())
 	token, err := p.provider.GetToken(ctx)
 
 	// Update cache with result

auth/tokenprovider/exchange.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/databricks/databricks-sql-go/logger"
 	"github.com/golang-jwt/jwt/v5"
-	"github.com/rs/zerolog/log"
 )
 
 // FederationProvider wraps another token provider and automatically handles token exchange
@@ -61,16 +61,16 @@ func (p *FederationProvider) GetToken(ctx context.Context) (*Token, error) {
 
 	// Check if token is a JWT and needs exchange
 	if p.needsTokenExchange(baseToken.AccessToken) {
-		log.Debug().Msgf("federation provider: attempting token exchange for %s", p.baseProvider.Name())
+		logger.Debug().Msgf("federation provider: attempting token exchange for %s", p.baseProvider.Name())
 
 		// Try token exchange
 		exchangedToken, err := p.tryTokenExchange(ctx, baseToken.AccessToken)
 		if err != nil {
-			log.Warn().Err(err).Msg("federation provider: token exchange failed, using original token")
+			logger.Warn().Err(err).Msg("federation provider: token exchange failed, using original token")
 			return baseToken, nil // Fall back to original token
 		}
 
-		log.Debug().Msg("federation provider: token exchange successful")
+		logger.Debug().Msg("federation provider: token exchange successful")
 		return exchangedToken, nil
 	}
 
@@ -87,7 +87,7 @@ func (p *FederationProvider) needsTokenExchange(tokenString string) bool {
 	// 3. Token validation will be done by Databricks during exchange
 	token, _, err := new(jwt.Parser).ParseUnverified(tokenString, jwt.MapClaims{})
 	if err != nil {
-		log.Debug().Err(err).Msg("federation provider: not a JWT token, skipping exchange")
+		logger.Debug().Err(err).Msg("federation provider: not a JWT token, skipping exchange")
 		return false
 	}
 
@@ -114,7 +114,7 @@ func (p *FederationProvider) tryTokenExchange(ctx context.Context, subjectToken
 		exchangeURL = "https://" + exchangeURL
 	} else if strings.HasPrefix(exchangeURL, "http://") {
 		// Warn if using insecure HTTP for token exchange
-		log.Warn().Msgf("federation provider: using insecure HTTP for token exchange: %s", exchangeURL)
+		logger.Warn().Msgf("federation provider: using insecure HTTP for token exchange: %s", exchangeURL)
 	}
 	if !strings.HasSuffix(exchangeURL, "/") {
 		exchangeURL += "/"
@@ -179,7 +179,7 @@ func (p *FederationProvider) tryTokenExchange(ctx context.Context, subjectToken
 		return nil, fmt.Errorf("token exchange returned empty access token")
 	}
 	if tokenResp.TokenType == "" {
-		log.Debug().Msg("token exchange: token_type not specified, defaulting to Bearer")
+		logger.Debug().Msg("token exchange: token_type not specified, defaulting to Bearer")
 		tokenResp.TokenType = "Bearer"
 	}
 	if tokenResp.ExpiresIn < 0 {
@@ -211,15 +211,15 @@ func (p *FederationProvider) isSameHost(url1, url2 string) bool {
 	u2, err2 := url.Parse(parsedURL2)
 
 	if err1 != nil || err2 != nil {
-		log.Debug().Msgf("federation provider: failed to parse URLs for comparison: url1=%s err1=%v, url2=%s err2=%v",
+		logger.Debug().Msgf("federation provider: failed to parse URLs for comparison: url1=%s err1=%v, url2=%s err2=%v",
 			url1, err1, parsedURL2, err2)
 		return false
 	}
 
 	// Use Hostname() instead of Host to ignore port differences
 	// This handles cases like "host.com:443" == "host.com" for HTTPS
 	isSame := u1.Hostname() == u2.Hostname()
-	log.Debug().Msgf("federation provider: host comparison: %s vs %s = %v", u1.Hostname(), u2.Hostname(), isSame)
+	logger.Debug().Msgf("federation provider: host comparison: %s vs %s = %v", u1.Hostname(), u2.Hostname(), isSame)
 	return isSame
 }
 

internal/arrowscan/arrowscan.go

@@ -173,7 +173,15 @@ func ScanCellCached(col arrow.Array, row int, loc *time.Location, keys *StructKe
 		// it. The cross-backend parity test pins both sides to µs (matching the wire
 		// reality), so it can't exercise a non-µs value; TestScanCellTimestampUnits
 		// covers this arm's unit-correctness directly instead.
-		return inLocation(c.Value(row).ToTime(dt.Unit), loc), nil
+		//
+		// Convert via the unit-specific time.Unix* constructors rather than arrow's
+		// ToTime: ToTime forms an int64-nanosecond intermediate (value*multiplier),
+		// which overflows for TIMESTAMPs outside ~1678–2262 and silently wraps a valid
+		// instant to a wrong one (e.g. TIMESTAMP '0001-01-01' → 1754). time.UnixMicro/
+		// UnixMilli/Unix split into (sec, nsec) internally, so the full 0001–9999 range
+		// round-trips. The Thrift default path never hits this — it receives TIMESTAMP
+		// as a preformatted server string — so this divergence is kernel-only.
+		return inLocation(timestampToTime(int64(c.Value(row)), dt.Unit), loc), nil
 	case *array.Decimal128:
 		dt := col.DataType().(*arrow.Decimal128Type)
 		return decimalfmt.ExactString(c.Value(row), dt.Scale), nil
@@ -185,6 +193,13 @@ func ScanCellCached(col arrow.Array, row int, loc *time.Location, keys *StructKe
 		// shared renderer to reuse, and this stays kernel-side).
 		dt := col.DataType().(*arrow.DurationType)
 		return formatDayTimeInterval(int64(c.Value(row)), dt.Unit), nil
+	case *array.DayTimeInterval:
+		// INTERVAL DAY TO SECOND can also arrive as arrow's DayTimeInterval ({Days,
+		// Milliseconds}) rather than a Duration. Fold it to total milliseconds and reuse
+		// the same renderer. int32 days × 86_400_000 stays well within int64, so the
+		// day→ms scaling cannot overflow.
+		v := c.Value(row)
+		return formatDayTimeInterval(int64(v.Days)*86_400_000+int64(v.Milliseconds), arrow.Millisecond), nil
 	case *array.MonthInterval:
 		// INTERVAL YEAR TO MONTH arrives as a month count; Thrift's server string is
 		// "years-months".
@@ -276,6 +291,24 @@ func abs64(x int64) int64 {
 	return x
 }
 
+// timestampToTime converts an arrow timestamp value in the given unit to a UTC
+// time.Time without arrow's ToTime int64-nanosecond overflow (see the *array.Timestamp
+// scan arm). Each constructor takes its argument in native units and splits into
+// (seconds, nanoseconds) internally, so the whole 0001–9999 range is representable;
+// only the nanosecond arm passes ns directly (already the finest unit, cannot overflow).
+func timestampToTime(v int64, unit arrow.TimeUnit) time.Time {
+	switch unit {
+	case arrow.Second:
+		return time.Unix(v, 0).UTC()
+	case arrow.Millisecond:
+		return time.UnixMilli(v).UTC()
+	case arrow.Microsecond:
+		return time.UnixMicro(v).UTC()
+	default: // arrow.Nanosecond
+		return time.Unix(0, v).UTC()
+	}
+}
+
 // inLocation renders t in loc, matching the Thrift path's .In(location); a nil
 // loc leaves the value in UTC (arrow's ToTime default).
 func inLocation(t time.Time, loc *time.Location) time.Time {