Skip to content

feat(bigquery): add support for MaxSlots #12958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gcf-merge-on-green merged 5 commits into from
Sep 26, 2025
Merged

feat(bigquery): add support for MaxSlots #12958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
32cefa0
feat(bigquery): add support for MaxSlots
shollyman Sep 26, 2025
35e1675
revert go.work.sum
shollyman Sep 26, 2025
702f9ad
update discovery dep
shollyman Sep 26, 2025
a119633
fastpath
shollyman Sep 26, 2025
fa361d9
Merge branch 'main' into fr-max-slots
shollyman Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions bigquery/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,15 @@ type CopyConfig struct {
// format is
// `projects/{project}/locations/{location}/reservations/{reservation}`.
Reservation string

// A target limit on the rate of slot consumption by this query. If set to a
// value > 0, BigQuery will attempt to limit the rate of slot consumption by
// this query to keep it below the configured limit, even if the query is
// eligible for more slots based on fair scheduling. The unused slots will be
// available for other jobs and queries to use.
//
// Note: This feature is not yet generally available.
MaxSlots int32
}

func (c *CopyConfig) toBQ() *bq.JobConfiguration {
Expand All @@ -99,6 +108,7 @@ func (c *CopyConfig) toBQ() *bq.JobConfiguration {
},
JobTimeoutMs: c.JobTimeout.Milliseconds(),
Reservation: c.Reservation,
MaxSlots: int64(c.MaxSlots),
}
}

Expand All @@ -112,6 +122,7 @@ func bqToCopyConfig(q *bq.JobConfiguration, c *Client) *CopyConfig {
OperationType: TableCopyOperationType(q.Copy.OperationType),
JobTimeout: time.Duration(q.JobTimeoutMs) * time.Millisecond,
Reservation: q.Reservation,
MaxSlots: int32(q.MaxSlots),
}
for _, t := range q.Copy.SourceTables {
cc.Srcs = append(cc.Srcs, bqToTable(t, c))
Expand Down
2 changes: 2 additions & 0 deletions bigquery/copy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ func TestCopy(t *testing.T) {
DestinationEncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
Labels: map[string]string{"a": "b"},
Reservation: "reservation/1",
MaxSlots: 123,
},
want: func() *bq.Job {
j := defaultCopyJob()
Expand All @@ -97,6 +98,7 @@ func TestCopy(t *testing.T) {
j.Configuration.Copy.WriteDisposition = "WRITE_TRUNCATE"
j.Configuration.Copy.DestinationEncryptionConfiguration = &bq.EncryptionConfiguration{KmsKeyName: "keyName"}
j.Configuration.Reservation = "reservation/1"
j.Configuration.MaxSlots = 123
return j
}(),
},
Expand Down
11 changes: 11 additions & 0 deletions bigquery/extract.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ type ExtractConfig struct {
// format is
// `projects/{project}/locations/{location}/reservations/{reservation}`.
Reservation string

// A target limit on the rate of slot consumption by this query. If set to a
// value > 0, BigQuery will attempt to limit the rate of slot consumption by
// this query to keep it below the configured limit, even if the query is
// eligible for more slots based on fair scheduling. The unused slots will be
// available for other jobs and queries to use.
//
// Note: This feature is not yet generally available.
MaxSlots int32
}

func (e *ExtractConfig) toBQ() *bq.JobConfiguration {
Expand All @@ -85,6 +94,7 @@ func (e *ExtractConfig) toBQ() *bq.JobConfiguration {
},
JobTimeoutMs: e.JobTimeout.Milliseconds(),
Reservation: e.Reservation,
MaxSlots: int64(e.MaxSlots),
}
if e.Src != nil {
cfg.Extract.SourceTable = e.Src.toBQ()
Expand Down Expand Up @@ -115,6 +125,7 @@ func bqToExtractConfig(q *bq.JobConfiguration, c *Client) *ExtractConfig {
UseAvroLogicalTypes: qe.UseAvroLogicalTypes,
JobTimeout: time.Duration(q.JobTimeoutMs) * time.Millisecond,
Reservation: q.Reservation,
MaxSlots: int32(q.MaxSlots),
}
}

Expand Down
2 changes: 2 additions & 0 deletions bigquery/extract_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ func TestExtract(t *testing.T) {
Labels: map[string]string{"a": "b"},
JobTimeout: 8 * time.Second,
Reservation: "reservation/1",
MaxSlots: 234,
},
want: func() *bq.Job {
j := defaultExtractJob()
Expand All @@ -94,6 +95,7 @@ func TestExtract(t *testing.T) {
f := false
j.Configuration.Extract.PrintHeader = &f
j.Configuration.Reservation = "reservation/1"
j.Configuration.MaxSlots = 234
return j
}(),
},
Expand Down
44 changes: 22 additions & 22 deletions bigquery/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,33 @@ require (
github.com/google/uuid v1.6.0
github.com/googleapis/gax-go/v2 v2.15.0
go.opencensus.io v0.24.0
go.opentelemetry.io/otel v1.36.0
go.opentelemetry.io/otel/sdk v1.36.0
golang.org/x/sync v0.16.0
go.opentelemetry.io/otel v1.37.0
go.opentelemetry.io/otel/sdk v1.37.0
golang.org/x/sync v0.17.0
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
google.golang.org/api v0.247.0
google.golang.org/api v0.250.0
google.golang.org/genproto v0.0.0-20250603155806-513f23925822
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c
google.golang.org/grpc v1.74.2
google.golang.org/protobuf v1.36.7
google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090
google.golang.org/grpc v1.75.1
google.golang.org/protobuf v1.36.9
)

require (
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.16.4 // indirect
cloud.google.com/go/auth v0.16.5 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.8.0 // indirect
cloud.google.com/go/compute/metadata v0.8.4 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/goccy/go-json v0.10.2 // indirect
Expand All @@ -58,16 +58,16 @@ require (
go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
golang.org/x/crypto v0.41.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
golang.org/x/crypto v0.42.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.26.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/time v0.12.0 // indirect
golang.org/x/tools v0.35.0 // indirect
golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.44.0 // indirect
golang.org/x/oauth2 v0.31.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/text v0.29.0 // indirect
golang.org/x/time v0.13.0 // indirect
golang.org/x/tools v0.36.0 // indirect
)
Loading
Loading