From fd3975cd60843ede55adc81d4f0e7711a4af0ea8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 15:54:13 -0400 Subject: [PATCH 01/38] Revert "feat: Add high precision TIMESTAMP values for queries (#7147)" This reverts commit bea42b2d48ef9a53395c0ad585f3baa941f5f822. # Conflicts: # handwritten/bigquery/src/bigquery.ts # handwritten/bigquery/src/job.ts # handwritten/bigquery/test/bigquery.ts --- handwritten/bigquery/src/bigquery.ts | 85 ++------- handwritten/bigquery/src/job.ts | 27 +-- handwritten/bigquery/system-test/bigquery.ts | 164 +----------------- .../system-test/timestamp_output_format.ts | 30 ++-- handwritten/bigquery/test/bigquery.ts | 68 +------- 5 files changed, 40 insertions(+), 334 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index d52d07343b5..9bafee94e23 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -1100,11 +1100,6 @@ export class BigQuery extends Service { }; }), }; - } else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') { - return { - type: 'TIMESTAMP', - timestampPrecision: '12', - }; } providedType = (providedType as string).toUpperCase(); @@ -2256,33 +2251,14 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const listParams = { - 'formatOptions.timestampOutputFormat': - queryReq.formatOptions?.timestampOutputFormat, - 'formatOptions.useInt64Timestamp': - queryReq.formatOptions?.useInt64Timestamp, - }; - if (options.skipParsing) { - rows = res.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - listParams, - }); - delete res.rows; - } - } catch (e) { - (callback as SimpleQueryRowsCallback)(e as Error, null, job); - return; + if (options.skipParsing) { + rows = res.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + }); + delete res.rows; } } this.trace_('[runJobsQuery] job complete'); @@ -2364,18 +2340,6 @@ export class BigQuery extends Service { if (options.job) { return undefined; } - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts - ? {} - : { - timestampOutputFormat: 'ISO8601_STRING', - }; - const formatOptions = extend(defaultOpts, { - timestampOutputFormat: options['formatOptions.timestampOutputFormat'], - useInt64Timestamp: options['formatOptions.useInt64Timestamp'], - }); const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, @@ -2384,7 +2348,9 @@ export class BigQuery extends Service { maximumBytesBilled: queryObj.maximumBytesBilled, timeoutMs: options.timeoutMs, location: queryObj.location || options.location, - formatOptions, + formatOptions: { + useInt64Timestamp: true, + }, maxResults: queryObj.maxResults || options.maxResults, query: queryObj.query, useLegacySql: false, @@ -2628,7 +2594,6 @@ function convertSchemaFieldValue( value = BigQueryRange.fromSchemaValue_( value, schemaField.rangeElementType!.type!, - options.listParams, // Required to convert TIMESTAMP values ); break; } @@ -2706,14 +2671,6 @@ export class BigQueryRange { }; } - /** - * This method returns start and end values for RANGE typed values returned from - * the server. It decodes the server RANGE value into start and end values so - * they can be used to construct a BigQueryRange. - * @private - * @param {string} value The range value. - * @returns {string[]} The start and end of the range. - */ private static fromStringValue_(value: string): [start: string, end: string] { let cleanedValue = value; if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { @@ -2733,24 +2690,7 @@ export class BigQueryRange { return [start, end]; } - /** - * This method is only used by convertSchemaFieldValue and only when range - * values are passed into convertSchemaFieldValue. It produces a value that is - * delivered to the user for read calls and it needs to pass along listParams - * to ensure TIMESTAMP types are converted properly. - * @private - * @param {string} value The range value. - * @param {string} elementType The element type. - * @param {bigquery.tabledata.IListParams | bigquery.jobs.IGetQueryResultsParams} [listParams] The list parameters. - * @returns {BigQueryRange} - */ - static fromSchemaValue_( - value: string, - elementType: string, - listParams?: - | bigquery.tabledata.IListParams - | bigquery.jobs.IGetQueryResultsParams, - ): BigQueryRange { + static fromSchemaValue_(value: string, elementType: string): BigQueryRange { const [start, end] = BigQueryRange.fromStringValue_(value); const convertRangeSchemaValue = (value: string) => { if (value === 'UNBOUNDED' || value === 'NULL') { @@ -2758,7 +2698,6 @@ export class BigQueryRange { } return convertSchemaFieldValue({type: elementType}, value, { wrapIntegers: false, - listParams, }); }; return BigQuery.range( diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index 22554a1d8f3..dc0b068c2ad 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -596,25 +596,14 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - try { - /* - Without this try/catch block, calls to /query endpoint will hang - indefinitely if a call to mergeSchemaWithRows_ fails because the - error never makes it to the callback. Instead, pass the error to the - callback the user provides so that the user can see the error. - */ - if (options.skipParsing) { - rows = resp.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); - delete resp.rows; - } - } catch (e) { - callback!(e as Error, null, null, resp); - return; + if (options.skipParsing) { + rows = resp.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + delete resp.rows; } } diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 92550709de1..c74f4ba959b 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1495,14 +1495,9 @@ describe('BigQuery', () => { ], }, (err, rows) => { - try { - // Without this try block the test runner silently fails - assert.ifError(err); - assert.strictEqual(rows!.length, 1); - done(); - } catch (e) { - done(e); - } + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); }, ); }); @@ -1526,159 +1521,6 @@ describe('BigQuery', () => { }, ); }); - describe('High Precision Query System Tests', () => { - let bigquery: BigQuery; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValuePicoseconds = - '2023-01-01T12:00:00.123456789123Z'; - const expectedErrorMessage = - 'Cannot specify both timestamp_as_int and timestamp_output_format.'; - - before(() => { - bigquery = new BigQuery(); - }); - - const testCases = [ - { - name: 'TOF: FLOAT64, UI64: true (error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: true, - expectedTsValue: undefined, - expectedError: expectedErrorMessage, - }, - { - name: 'TOF: omitted, UI64: omitted (default INT64)', - timestampOutputFormat: undefined, - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, - }, - { - name: 'TOF: omitted, UI64: true', - timestampOutputFormat: undefined, - useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, - }, - ]; - - testCases.forEach(testCase => { - it(`should handle ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. Queries using TIMESTAMP_ADD are another example. - */ - const query = { - query: 'SELECT ? as ts', - params: [ - bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), - ], - types: ['TIMESTAMP(12)'], - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].ts.value !== undefined); - assert.strictEqual( - rows[0].ts.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - it(`should handle nested ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. - */ - const query = { - query: 'SELECT ? obj', - params: [ - { - nested: { - a: bigquery.timestamp( - '2023-01-01T12:00:00.123456789123Z', - ), - }, - }, - ], - types: [ - { - nested: { - a: 'TIMESTAMP(12)', - }, - }, - ], - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].obj.nested.a.value !== undefined); - assert.strictEqual( - rows[0].obj.nested.a.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - }); - }); }); describe('named', () => { diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 0fe388e1e3b..96ede116075 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -34,8 +34,8 @@ describe('Timestamp Output Format System Tests', () => { const dataset = bigquery.dataset(datasetId); const table = dataset.table(tableId); const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; + const expectedTsValueMicroseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456789123Z'; before(async () => { await dataset.create(); @@ -59,13 +59,13 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', @@ -78,19 +78,19 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', timestampOutputFormat: 'FLOAT64', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp=true', timestampOutputFormat: 'INT64', useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp=false', timestampOutputFormat: 'INT64', useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', @@ -103,50 +103,50 @@ describe('Timestamp Output Format System Tests', () => { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', timestampOutputFormat: 'ISO8601_STRING', useInt64Timestamp: false, - expectedTsValue: expectedTsValuePicoseconds, + expectedTsValue: expectedTsValueNanoseconds, }, // Additional test cases for undefined combinations { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', timestampOutputFormat: undefined, useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, + expectedTsValue: expectedTsValueNanoseconds, }, { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', timestampOutputFormat: undefined, useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', timestampOutputFormat: undefined, useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', timestampOutputFormat: 'FLOAT64', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with INT64 and useInt64Timestamp undefined', timestampOutputFormat: 'INT64', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, + expectedTsValue: expectedTsValueMicroseconds, }, { name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', timestampOutputFormat: 'ISO8601_STRING', useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, + expectedTsValue: expectedTsValueNanoseconds, }, ]; diff --git a/handwritten/bigquery/test/bigquery.ts b/handwritten/bigquery/test/bigquery.ts index 74fbbedf1c7..bc006e70660 100644 --- a/handwritten/bigquery/test/bigquery.ts +++ b/handwritten/bigquery/test/bigquery.ts @@ -42,7 +42,6 @@ import { TableField, Query, QueryResultsOptions, - QueryOptions, } from '../src'; import {SinonStub} from 'sinon'; import {PreciseDate} from '@google-cloud/precise-date'; @@ -3466,75 +3465,12 @@ describe('BigQuery', () => { }, jobCreationMode: 'JOB_CREATION_REQUIRED', formatOptions: { - timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, }, }; assert.deepStrictEqual(req, expectedReq); }); - describe('timestamp format options', () => { - const testCases: { - name: string; - opts: QueryOptions; - expected?: any; - bail?: boolean; - }[] = [ - { - name: 'TOF: omitted, UI64: omitted (default ISO8601_STRING)', - opts: {}, - expected: { - timestampOutputFormat: 'ISO8601_STRING', - }, - }, - { - name: 'TOF: omitted, UI64: true', - opts: { - ['formatOptions.useInt64Timestamp']: true, - }, - expected: { - useInt64Timestamp: true, - }, - }, - { - name: 'TOF: omitted, UI64: false (default ISO8601_STRING)', - opts: { - ['formatOptions.useInt64Timestamp']: false, - }, - expected: { - useInt64Timestamp: false, - }, - }, - ]; - - testCases.forEach(testCase => { - it(`should handle ${testCase.name}`, () => { - const req = bq.buildQueryRequest_(QUERY_STRING, testCase.opts); - - const expectedReq = { - query: QUERY_STRING, - useLegacySql: false, - requestId: req.requestId, - jobCreationMode: 'JOB_CREATION_OPTIONAL', - formatOptions: testCase.expected, - connectionProperties: undefined, - continuous: undefined, - createSession: undefined, - defaultDataset: undefined, - destinationEncryptionConfiguration: undefined, - labels: undefined, - location: undefined, - maxResults: undefined, - maximumBytesBilled: undefined, - preserveNulls: undefined, - reservation: undefined, - timeoutMs: undefined, - useQueryCache: undefined, - writeIncrementalResults: undefined, - }; - assert.deepStrictEqual(req, expectedReq); - }); - }); - }); it('should create a QueryRequest from a SQL string', () => { const req = bq.buildQueryRequest_(QUERY_STRING, {}); for (const key in req) { @@ -3548,7 +3484,7 @@ describe('BigQuery', () => { requestId: req.requestId, jobCreationMode: 'JOB_CREATION_OPTIONAL', formatOptions: { - timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, }, }; assert.deepStrictEqual(req, expectedReq); From f31ccc7638afd5fcb88324015b27227f054b2ef3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 16:15:22 -0400 Subject: [PATCH 02/38] Revert "feat: support high precision timestamp strings on getRows calls (#1596)" This reverts commit b3217a39f0d85faaf606b97b85e0166d5f7fb07f. # Conflicts: # handwritten/bigquery/src/table.ts --- handwritten/bigquery/src/bigquery.ts | 56 +---- handwritten/bigquery/src/table.ts | 50 ++-- .../system-test/timestamp_output_format.ts | 216 ------------------ handwritten/bigquery/test/table.ts | 9 +- 4 files changed, 24 insertions(+), 307 deletions(-) delete mode 100644 handwritten/bigquery/system-test/timestamp_output_format.ts diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 9bafee94e23..116434f173d 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -597,9 +597,6 @@ export class BigQuery extends Service { wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; - listParams?: - | bigquery.tabledata.IListParams - | bigquery.jobs.IGetQueryResultsParams; }, ) { // deep copy schema fields to avoid mutation @@ -2480,9 +2477,6 @@ function convertSchemaFieldValue( wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; - listParams?: - | bigquery.tabledata.IListParams - | bigquery.jobs.IGetQueryResultsParams; }, ) { if (value === null) { @@ -2542,43 +2536,9 @@ function convertSchemaFieldValue( break; } case 'TIMESTAMP': { - /* - At this point, 'value' will equal the timestamp value returned from the - server. We need to parse this value differently depending on its format. - For example, value could be any of the following: - 1672574400123456 - 1672574400.123456 - 2023-01-01T12:00:00.123456789123Z - */ - const listParams = options.listParams; - const timestampOutputFormat = listParams - ? listParams['formatOptions.timestampOutputFormat'] - : undefined; - const useInt64Timestamp = listParams - ? listParams['formatOptions.useInt64Timestamp'] - : undefined; - if (timestampOutputFormat === 'ISO8601_STRING') { - // value is ISO string, create BigQueryTimestamp wrapping the string - value = BigQuery.timestamp(value); - } else if ( - useInt64Timestamp !== true && - timestampOutputFormat !== 'INT64' && - (useInt64Timestamp !== undefined || timestampOutputFormat !== undefined) - ) { - // NOTE: The additional - // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) - // check is to ensure that calls to the /query endpoint remain - // unaffected as they will not be providing any listParams. - // - // If the program reaches this point in time then - // value is float seconds so convert to BigQueryTimestamp - value = BigQuery.timestamp(Number(value)); - } else { - // Expect int64 micros (default or explicit INT64) - const pd = new PreciseDate(); - pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); - value = BigQuery.timestamp(pd); - } + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); break; } case 'GEOGRAPHY': { @@ -2773,16 +2733,6 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); - if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { - /* - TODO: - When https://github.com/googleapis/nodejs-precise-date/pull/302 - is released and we have full support for picoseconds in PreciseData - then we can remove this if block. - */ - this.value = value; - return; - } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index e92c6a6791d..62ed4521256 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -55,7 +55,6 @@ import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; import {IntegerTypeCastOptions} from './bigquery'; import {RowQueue} from './rowQueue'; -import IDataFormatOptions = bigquery.IDataFormatOptions; // This is supposed to be a @google-cloud/storage `File` type. The storage npm // module includes these types, but is current installed as a devDependency. @@ -1867,42 +1866,25 @@ class Table extends ServiceObject { callback!(err, null, null, resp); return; } - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - if (options.skipParsing) { - rows = rows || []; - } else { - rows = BigQuery.mergeSchemaWithRows_( - this.metadata.schema, - rows || [], - { - wrapIntegers, - selectedFields, - parseJSON, - listParams: qs, - }, - ); - } - } catch (err) { - callback!(err as Error | null, null, null, resp); - return; + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], { + wrapIntegers, + selectedFields, + parseJSON, + }); } callback!(null, rows, nextQuery, resp); }; - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts - ? {} - : { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', - }; - const qs = extend(defaultOpts, options); + + const qs = extend( + { + 'formatOptions.useInt64Timestamp': true, + }, + options, + ); + this.request( { uri: '/data', diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts deleted file mode 100644 index 96ede116075..00000000000 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import * as assert from 'assert'; -import {describe, it, before, after} from 'mocha'; -import {BigQuery} from '../src/bigquery'; -import {randomUUID} from 'crypto'; -import {RequestResponse} from '@google-cloud/common/build/src/service-object'; - -const bigquery = new BigQuery(); - -interface TestCase { - name: string; - timestampOutputFormat?: string; - useInt64Timestamp?: boolean; - expectedError?: string; - expectedTsValue?: string; -} - -describe('Timestamp Output Format System Tests', () => { - const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; - const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; - const dataset = bigquery.dataset(datasetId); - const table = dataset.table(tableId); - const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; - const expectedTsValueMicroseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456789123Z'; - - before(async () => { - await dataset.create(); - await table.create({ - schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], - }); - // Insert a row to test retrieval - await table.insert([{ts: insertedTsValue}]); - }); - - after(async () => { - try { - await dataset.delete({force: true}); - } catch (e) { - console.error('Error deleting dataset:', e); - } - }); - - const testCases: TestCase[] = [ - { - name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', - timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', - useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', - timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: true, - expectedError: - 'Cannot specify both use_int64_timestamp and timestamp_output_format.', - }, - { - name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with INT64 and useInt64Timestamp=true', - timestampOutputFormat: 'INT64', - useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with INT64 and useInt64Timestamp=false', - timestampOutputFormat: 'INT64', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', - timestampOutputFormat: 'ISO8601_STRING', - useInt64Timestamp: true, - expectedError: - 'Cannot specify both use_int64_timestamp and timestamp_output_format.', - }, - { - name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', - timestampOutputFormat: 'ISO8601_STRING', - useInt64Timestamp: false, - expectedTsValue: expectedTsValueNanoseconds, - }, - // Additional test cases for undefined combinations - { - name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', - timestampOutputFormat: undefined, - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, - }, - { - name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', - timestampOutputFormat: undefined, - useInt64Timestamp: true, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', - timestampOutputFormat: undefined, - useInt64Timestamp: false, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', - timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with INT64 and useInt64Timestamp undefined', - timestampOutputFormat: 'INT64', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueMicroseconds, - }, - { - name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', - timestampOutputFormat: 'ISO8601_STRING', - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValueNanoseconds, - }, - ]; - - testCases.forEach( - ({ - name, - timestampOutputFormat, - useInt64Timestamp, - expectedError, - expectedTsValue, - }) => { - it(name, async () => { - const options: {[key: string]: any} = {}; - if (timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - timestampOutputFormat; - } - if (useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = useInt64Timestamp; - } - - if (expectedError) { - try { - await table.getRows(options); - assert.fail('The call should have thrown an error.'); - } catch (e) { - assert.strictEqual((e as Error).message, expectedError); - } - } else { - const [rows] = await table.getRows(options); - assert(rows.length > 0); - assert.strictEqual(rows[0].ts.value, expectedTsValue); - } - }); - }, - ); - - it('should make a request with ISO8601_STRING when no format options are being used', done => { - void (async () => { - const originalRequest = table.request; - const requestPromise: Promise = new Promise( - (resolve, reject) => { - const innerPromise = new Promise((innerResolve, innerReject) => { - innerResolve({}); - }); - resolve(innerPromise as Promise); - }, - ); - table.request = reqOpts => { - table.request = originalRequest; - if ( - reqOpts.qs['formatOptions.timestampOutputFormat'] === 'ISO8601_STRING' - ) { - done(); - } else { - done( - new Error( - 'The default timestampOutputFormat should be ISO8601_STRING', - ), - ); - } - return requestPromise; - }; - await table.getRows({}); - })(); - }); -}); diff --git a/handwritten/bigquery/test/table.ts b/handwritten/bigquery/test/table.ts index 2e13f1572d6..bb3225459ea 100644 --- a/handwritten/bigquery/test/table.ts +++ b/handwritten/bigquery/test/table.ts @@ -2046,7 +2046,7 @@ describe('BigQuery/Table', () => { assert.strictEqual(reqOpts.uri, '/data'); assert.deepStrictEqual(reqOpts.qs, { ...options, - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, }); callback(null, {}); }; @@ -2219,6 +2219,7 @@ describe('BigQuery/Table', () => { table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { callback(null, { + 'formatOptions.useInt64Timestamp': true, pageToken, }); }; @@ -2228,7 +2229,7 @@ describe('BigQuery/Table', () => { assert.deepStrictEqual(nextQuery, { a: 'b', c: 'd', - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, pageToken, }); // Original object isn't affected. @@ -2445,7 +2446,7 @@ describe('BigQuery/Table', () => { table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, }); callback(null, {}); }; @@ -2469,7 +2470,7 @@ describe('BigQuery/Table', () => { table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + 'formatOptions.useInt64Timestamp': true, }); callback(null, {}); }; From 5544e36556739a18cc37974cb7bb1a61f72846c8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 16:18:57 -0400 Subject: [PATCH 03/38] skip a test --- handwritten/bigquery/system-test/bigquery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index c74f4ba959b..58009c5e668 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1041,7 +1041,7 @@ describe('BigQuery', () => { } }); - it('should create a table with timestampPrecision', async () => { + it.skip('should create a table with timestampPrecision', async () => { const table = dataset.table(generateName('timestamp-precision-table')); const schema = { fields: [ From 1bbd192f0fcbbc1b6eea6bac51c1f47de0aebd1b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 Mar 2026 16:28:18 -0400 Subject: [PATCH 04/38] Remove the High Precision Query System tests --- handwritten/bigquery/system-test/bigquery.ts | 155 ------------------- 1 file changed, 155 deletions(-) diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 58009c5e668..aced74c4b40 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1815,161 +1815,6 @@ describe('BigQuery', () => { }, ); }); - describe('High Precision Query System Tests', () => { - let bigquery: BigQuery; - const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; - const expectedTsValuePicoseconds = - '2023-01-01T12:00:00.123456789123Z'; - const expectedErrorMessage = - 'Cannot specify both timestamp_as_int and timestamp_output_format.'; - - before(() => { - bigquery = new BigQuery(); - }); - - const testCases = [ - { - name: 'TOF: FLOAT64, UI64: true (error)', - timestampOutputFormat: 'FLOAT64', - useInt64Timestamp: true, - expectedTsValue: undefined, - expectedError: expectedErrorMessage, - }, - { - name: 'TOF: omitted, UI64: omitted (default INT64)', - timestampOutputFormat: undefined, - useInt64Timestamp: undefined, - expectedTsValue: expectedTsValuePicoseconds, - }, - { - name: 'TOF: omitted, UI64: true', - timestampOutputFormat: undefined, - useInt64Timestamp: true, - expectedTsValue: expectedTsValueNanoseconds, - }, - ]; - - testCases.forEach(testCase => { - it(`should handle ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. Queries using TIMESTAMP_ADD are another example. - */ - const query = { - query: 'SELECT @r AS ts', - params: { - r: bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), - }, - types: { - r: 'TIMESTAMP(12)', - }, - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].ts.value !== undefined); - assert.strictEqual( - rows[0].ts.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - it(`should handle nested ${testCase.name}`, async () => { - /* - The users use the new TIMESTAMP(12) type to indicate they want to - opt in to using timestampPrecision=12. The reason is that some queries - like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set - timestampPrecision=12 and we don't want this code change to affect - existing users. - */ - const query = { - query: 'SELECT @r obj', - params: { - r: { - nested: { - a: bigquery.timestamp( - '2023-01-01T12:00:00.123456789123Z', - ), - }, - }, - }, - types: { - r: { - nested: { - a: 'TIMESTAMP(12)', - }, - }, - }, - }; - - const options: any = {}; - if (testCase.timestampOutputFormat !== undefined) { - options['formatOptions.timestampOutputFormat'] = - testCase.timestampOutputFormat; - } - if (testCase.useInt64Timestamp !== undefined) { - options['formatOptions.useInt64Timestamp'] = - testCase.useInt64Timestamp; - } - - try { - const [rows] = await bigquery.query(query, options); - if (testCase.expectedError) { - assert.fail( - `Query should have failed for ${testCase.name}, but succeeded`, - ); - } - assert.ok(rows.length > 0); - assert.ok(rows[0].obj.nested.a.value !== undefined); - assert.strictEqual( - rows[0].obj.nested.a.value, - testCase.expectedTsValue, - ); - } catch (err: any) { - if (!testCase.expectedError) { - throw err; - } - - const message = err.message; - assert.strictEqual( - message, - testCase.expectedError, - `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, - ); - } - }); - }); - }); }); }); From c588794d657711f4b95294cc2ea46b287f89a697 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 15:21:22 -0400 Subject: [PATCH 05/38] Source code changes for timestamp precision flag --- handwritten/bigquery/src/bigquery.ts | 152 ++++++++++++++++++++++++--- handwritten/bigquery/src/job.ts | 37 +++++-- handwritten/bigquery/src/table.ts | 73 ++++++++++--- 3 files changed, 228 insertions(+), 34 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 116434f173d..010bea0e105 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -597,6 +597,9 @@ export class BigQuery extends Service { wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams; }, ) { // deep copy schema fields to avoid mutation @@ -1097,6 +1100,13 @@ export class BigQuery extends Service { }; }), }; + } else if ((providedType as string).toUpperCase() === 'TIMESTAMP(12)') { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + return { + type: 'TIMESTAMP', + timestampPrecision: '12', + }; + } } providedType = (providedType as string).toUpperCase(); @@ -2248,14 +2258,45 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - if (options.skipParsing) { - rows = res.rows; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listParams = { + 'formatOptions.timestampOutputFormat': + queryReq.formatOptions?.timestampOutputFormat, + 'formatOptions.useInt64Timestamp': + queryReq.formatOptions?.useInt64Timestamp, + }; + if (options.skipParsing) { + rows = res.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + listParams, + }); + delete res.rows; + } + } catch (e) { + (callback as SimpleQueryRowsCallback)(e as Error, null, job); + return; + } } else { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - }); - delete res.rows; + if (options.skipParsing) { + rows = res.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { + wrapIntegers: options.wrapIntegers || false, + parseJSON: options.parseJSON, + }); + delete res.rows; + } } } this.trace_('[runJobsQuery] job complete'); @@ -2337,6 +2378,25 @@ export class BigQuery extends Service { if (options.job) { return undefined; } + let formatOptions; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + timestampOutputFormat: 'ISO8601_STRING', + }; + formatOptions = extend(defaultOpts, { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }); + } else { + formatOptions = { + useInt64Timestamp: true, + }; + } const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, @@ -2345,9 +2405,7 @@ export class BigQuery extends Service { maximumBytesBilled: queryObj.maximumBytesBilled, timeoutMs: options.timeoutMs, location: queryObj.location || options.location, - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, maxResults: queryObj.maxResults || options.maxResults, query: queryObj.query, useLegacySql: false, @@ -2477,6 +2535,9 @@ function convertSchemaFieldValue( wrapIntegers: boolean | IntegerTypeCastOptions; selectedFields?: string[]; parseJSON?: boolean; + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams; }, ) { if (value === null) { @@ -2536,9 +2597,52 @@ function convertSchemaFieldValue( break; } case 'TIMESTAMP': { - const pd = new PreciseDate(); - pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); - value = BigQuery.timestamp(pd); + // High precision timestamp behaviour + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + /* + At this point, 'value' will equal the timestamp value returned from the + server. We need to parse this value differently depending on its format. + For example, value could be any of the following: + 1672574400123456 + 1672574400.123456 + 2023-01-01T12:00:00.123456789123Z + */ + const listParams = options.listParams; + const timestampOutputFormat = listParams + ? listParams['formatOptions.timestampOutputFormat'] + : undefined; + const useInt64Timestamp = listParams + ? listParams['formatOptions.useInt64Timestamp'] + : undefined; + if (timestampOutputFormat === 'ISO8601_STRING') { + // value is ISO string, create BigQueryTimestamp wrapping the string + value = BigQuery.timestamp(value); + } else if ( + useInt64Timestamp !== true && + timestampOutputFormat !== 'INT64' && + (useInt64Timestamp !== undefined || + timestampOutputFormat !== undefined) + ) { + // NOTE: The additional + // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) + // check is to ensure that calls to the /query endpoint remain + // unaffected as they will not be providing any listParams. + // + // If the program reaches this point in time then + // value is float seconds so convert to BigQueryTimestamp + value = BigQuery.timestamp(Number(value)); + } else { + // Expect int64 micros (default or explicit INT64) + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); + } + } else { + // Old behaviour + const pd = new PreciseDate(); + pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); + value = BigQuery.timestamp(pd); + } break; } case 'GEOGRAPHY': { @@ -2554,6 +2658,7 @@ function convertSchemaFieldValue( value = BigQueryRange.fromSchemaValue_( value, schemaField.rangeElementType!.type!, + options.listParams, // Required to convert TIMESTAMP values ); break; } @@ -2650,7 +2755,13 @@ export class BigQueryRange { return [start, end]; } - static fromSchemaValue_(value: string, elementType: string): BigQueryRange { + static fromSchemaValue_( + value: string, + elementType: string, + listParams?: + | bigquery.tabledata.IListParams + | bigquery.jobs.IGetQueryResultsParams, + ): BigQueryRange { const [start, end] = BigQueryRange.fromStringValue_(value); const convertRangeSchemaValue = (value: string) => { if (value === 'UNBOUNDED' || value === 'NULL') { @@ -2658,6 +2769,7 @@ export class BigQueryRange { } return convertSchemaFieldValue({type: elementType}, value, { wrapIntegers: false, + listParams, }); }; return BigQuery.range( @@ -2733,6 +2845,18 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { + /* + TODO: + When https://github.com/googleapis/nodejs-precise-date/pull/302 + is released and we have full support for picoseconds in PreciseData + then we can remove this if block. + */ + this.value = value; + return; + } + } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index dc0b068c2ad..a0a2b45c1ad 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -596,14 +596,37 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - if (options.skipParsing) { - rows = resp.rows; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + try { + /* + Without this try/catch block, calls to /query endpoint will hang + indefinitely if a call to mergeSchemaWithRows_ fails because the + error never makes it to the callback. Instead, pass the error to the + callback the user provides so that the user can see the error. + */ + if (options.skipParsing) { + rows = resp.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + delete resp.rows; + } + } catch (e) { + callback!(e as Error, null, null, resp); + return; + } } else { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); - delete resp.rows; + if (options.skipParsing) { + rows = resp.rows; + } else { + rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { + wrapIntegers, + parseJSON, + }); + delete resp.rows; + } } } diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index 62ed4521256..f425a21caef 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -1866,24 +1866,71 @@ class Table extends ServiceObject { callback!(err, null, null, resp); return; } - if (options.skipParsing) { - rows = rows || []; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + // High precision timestamp behaviour + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_( + this.metadata.schema, + rows || [], + { + wrapIntegers, + selectedFields, + parseJSON, + listParams: qs, + }, + ); + } + } catch (err) { + callback!(err as Error | null, null, null, resp); + return; + } } else { - rows = BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], { - wrapIntegers, - selectedFields, - parseJSON, - }); + // Old behaviour + if (options.skipParsing) { + rows = rows || []; + } else { + rows = BigQuery.mergeSchemaWithRows_( + this.metadata.schema, + rows || [], + { + wrapIntegers, + selectedFields, + parseJSON, + }, + ); + } } callback!(null, rows, nextQuery, resp); }; - const qs = extend( - { - 'formatOptions.useInt64Timestamp': true, - }, - options, - ); + let qs: any; + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + const defaultOpts = hasAnyFormatOpts + ? {} + : { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + }; + qs = extend(defaultOpts, options); + } else { + qs = extend( + { + 'formatOptions.useInt64Timestamp': true, + }, + options, + ); + } this.request( { From 73c4ed5f9d91b9a8c4c242738a5b2aa0edd52e63 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 15:36:50 -0400 Subject: [PATCH 06/38] Add tests back --- handwritten/bigquery/system-test/bigquery.ts | 381 ++++++++++++++++-- .../system-test/timestamp_output_format.ts | 0 2 files changed, 358 insertions(+), 23 deletions(-) create mode 100644 handwritten/bigquery/system-test/timestamp_output_format.ts diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index aced74c4b40..8a104d9d9db 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -1041,26 +1041,28 @@ describe('BigQuery', () => { } }); - it.skip('should create a table with timestampPrecision', async () => { - const table = dataset.table(generateName('timestamp-precision-table')); - const schema = { - fields: [ - { - name: 'ts_field', - type: 'TIMESTAMP', - timestampPrecision: 12, - }, - ], - }; - try { - await table.create({schema}); - const [metadata] = await table.getMetadata(); - assert.deepStrictEqual( - metadata.schema.fields[0].timestampPrecision, - '12', - ); - } catch (e) { - assert.ifError(e); + it('should create a table with timestampPrecision', async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + const table = dataset.table(generateName('timestamp-precision-table')); + const schema = { + fields: [ + { + name: 'ts_field', + type: 'TIMESTAMP', + timestampPrecision: 12, + }, + ], + }; + try { + await table.create({schema}); + const [metadata] = await table.getMetadata(); + assert.deepStrictEqual( + metadata.schema.fields[0].timestampPrecision, + '12', + ); + } catch (e) { + assert.ifError(e); + } } }); @@ -1495,9 +1497,14 @@ describe('BigQuery', () => { ], }, (err, rows) => { - assert.ifError(err); - assert.strictEqual(rows!.length, 1); - done(); + try { + // Without this try block the test runner silently fails + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); + } catch (e) { + done(e); + } }, ); }); @@ -1521,6 +1528,169 @@ describe('BigQuery', () => { }, ); }); + describe('High Precision Query System Tests', () => { + let bigquery: BigQuery; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = + '2023-01-01T12:00:00.123456789123Z'; + const expectedErrorMessage = + 'Cannot specify both timestamp_as_int and timestamp_output_format.'; + + before(() => { + bigquery = new BigQuery(); + }); + + const testCases = [ + { + name: 'TOF: FLOAT64, UI64: true (error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedTsValue: undefined, + expectedError: expectedErrorMessage, + }, + { + name: 'TOF: omitted, UI64: omitted (default INT64)', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'TOF: omitted, UI64: true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. Queries using TIMESTAMP_ADD are another example. + */ + const query = { + query: 'SELECT ? as ts', + params: [ + bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), + ], + types: ['TIMESTAMP(12)'], + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].ts.value !== undefined); + assert.strictEqual( + rows[0].ts.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + it(`should handle nested ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. + */ + const query = { + query: 'SELECT ? obj', + params: [ + { + nested: { + a: bigquery.timestamp( + '2023-01-01T12:00:00.123456789123Z', + ), + }, + }, + ], + types: [ + { + nested: { + a: 'TIMESTAMP(12)', + }, + }, + ], + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].obj.nested.a.value !== undefined); + assert.strictEqual( + rows[0].obj.nested.a.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + }); + }); }); describe('named', () => { @@ -1815,6 +1985,171 @@ describe('BigQuery', () => { }, ); }); + describe('High Precision Query System Tests', () => { + let bigquery: BigQuery; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = + '2023-01-01T12:00:00.123456789123Z'; + const expectedErrorMessage = + 'Cannot specify both timestamp_as_int and timestamp_output_format.'; + + before(() => { + bigquery = new BigQuery(); + }); + + const testCases = [ + { + name: 'TOF: FLOAT64, UI64: true (error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedTsValue: undefined, + expectedError: expectedErrorMessage, + }, + { + name: 'TOF: omitted, UI64: omitted (default INT64)', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'TOF: omitted, UI64: true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. Queries using TIMESTAMP_ADD are another example. + */ + const query = { + query: 'SELECT @r AS ts', + params: { + r: bigquery.timestamp('2023-01-01T12:00:00.123456789123Z'), + }, + types: { + r: 'TIMESTAMP(12)', + }, + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].ts.value !== undefined); + assert.strictEqual( + rows[0].ts.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + it(`should handle nested ${testCase.name}`, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + // These tests are only important when the high precision + // timestamp support is turned on. + return; + } + /* + The users use the new TIMESTAMP(12) type to indicate they want to + opt in to using timestampPrecision=12. The reason is that some queries + like `SELECT CAST(? as TIMESTAMP(12))` will fail if we set + timestampPrecision=12 and we don't want this code change to affect + existing users. + */ + const query = { + query: 'SELECT @r obj', + params: { + r: { + nested: { + a: bigquery.timestamp( + '2023-01-01T12:00:00.123456789123Z', + ), + }, + }, + }, + types: { + r: { + nested: { + a: 'TIMESTAMP(12)', + }, + }, + }, + }; + + const options: any = {}; + if (testCase.timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + testCase.timestampOutputFormat; + } + if (testCase.useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = + testCase.useInt64Timestamp; + } + + try { + const [rows] = await bigquery.query(query, options); + if (testCase.expectedError) { + assert.fail( + `Query should have failed for ${testCase.name}, but succeeded`, + ); + } + assert.ok(rows.length > 0); + assert.ok(rows[0].obj.nested.a.value !== undefined); + assert.strictEqual( + rows[0].obj.nested.a.value, + testCase.expectedTsValue, + ); + } catch (err: any) { + if (!testCase.expectedError) { + throw err; + } + + const message = err.message; + assert.strictEqual( + message, + testCase.expectedError, + `Expected ${testCase.expectedError} error for ${testCase.name}, got ${message} (${err.message})`, + ); + } + }); + }); + }); }); }); diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts new file mode 100644 index 00000000000..e69de29bb2d From ab16690d8cfdedecec278597cddea98cdaf98893 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 15:37:17 -0400 Subject: [PATCH 07/38] Add test file back in --- .../system-test/timestamp_output_format.ts | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index e69de29bb2d..0fe388e1e3b 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it, before, after} from 'mocha'; +import {BigQuery} from '../src/bigquery'; +import {randomUUID} from 'crypto'; +import {RequestResponse} from '@google-cloud/common/build/src/service-object'; + +const bigquery = new BigQuery(); + +interface TestCase { + name: string; + timestampOutputFormat?: string; + useInt64Timestamp?: boolean; + expectedError?: string; + expectedTsValue?: string; +} + +describe('Timestamp Output Format System Tests', () => { + const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; + const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; + const dataset = bigquery.dataset(datasetId); + const table = dataset.table(tableId); + const insertedTsValue = '2023-01-01T12:00:00.123456789123Z'; + const expectedTsValueNanoseconds = '2023-01-01T12:00:00.123456000Z'; + const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; + + before(async () => { + await dataset.create(); + await table.create({ + schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], + }); + // Insert a row to test retrieval + await table.insert([{ts: insertedTsValue}]); + }); + + after(async () => { + try { + await dataset.delete({force: true}); + } catch (e) { + console.error('Error deleting dataset:', e); + } + }); + + const testCases: TestCase[] = [ + { + name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=true', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp=false', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with FLOAT64 and useInt64Timestamp=true (expect error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: true, + expectedError: + 'Cannot specify both use_int64_timestamp and timestamp_output_format.', + }, + { + name: 'should call getRows with FLOAT64 and useInt64Timestamp=false', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with INT64 and useInt64Timestamp=true', + timestampOutputFormat: 'INT64', + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with INT64 and useInt64Timestamp=false', + timestampOutputFormat: 'INT64', + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=true (expect error)', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: true, + expectedError: + 'Cannot specify both use_int64_timestamp and timestamp_output_format.', + }, + { + name: 'should call getRows with ISO8601_STRING and useInt64Timestamp=false', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: false, + expectedTsValue: expectedTsValuePicoseconds, + }, + // Additional test cases for undefined combinations + { + name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp undefined', + timestampOutputFormat: undefined, + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + { + name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=true', + timestampOutputFormat: undefined, + useInt64Timestamp: true, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with timestampOutputFormat undefined and useInt64Timestamp=false', + timestampOutputFormat: undefined, + useInt64Timestamp: false, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED and useInt64Timestamp undefined', + timestampOutputFormat: 'TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with FLOAT64 and useInt64Timestamp undefined (expect error)', + timestampOutputFormat: 'FLOAT64', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with INT64 and useInt64Timestamp undefined', + timestampOutputFormat: 'INT64', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValueNanoseconds, + }, + { + name: 'should call getRows with ISO8601_STRING and useInt64Timestamp undefined (expect error)', + timestampOutputFormat: 'ISO8601_STRING', + useInt64Timestamp: undefined, + expectedTsValue: expectedTsValuePicoseconds, + }, + ]; + + testCases.forEach( + ({ + name, + timestampOutputFormat, + useInt64Timestamp, + expectedError, + expectedTsValue, + }) => { + it(name, async () => { + const options: {[key: string]: any} = {}; + if (timestampOutputFormat !== undefined) { + options['formatOptions.timestampOutputFormat'] = + timestampOutputFormat; + } + if (useInt64Timestamp !== undefined) { + options['formatOptions.useInt64Timestamp'] = useInt64Timestamp; + } + + if (expectedError) { + try { + await table.getRows(options); + assert.fail('The call should have thrown an error.'); + } catch (e) { + assert.strictEqual((e as Error).message, expectedError); + } + } else { + const [rows] = await table.getRows(options); + assert(rows.length > 0); + assert.strictEqual(rows[0].ts.value, expectedTsValue); + } + }); + }, + ); + + it('should make a request with ISO8601_STRING when no format options are being used', done => { + void (async () => { + const originalRequest = table.request; + const requestPromise: Promise = new Promise( + (resolve, reject) => { + const innerPromise = new Promise((innerResolve, innerReject) => { + innerResolve({}); + }); + resolve(innerPromise as Promise); + }, + ); + table.request = reqOpts => { + table.request = originalRequest; + if ( + reqOpts.qs['formatOptions.timestampOutputFormat'] === 'ISO8601_STRING' + ) { + done(); + } else { + done( + new Error( + 'The default timestampOutputFormat should be ISO8601_STRING', + ), + ); + } + return requestPromise; + }; + await table.getRows({}); + })(); + }); +}); From 2410d66976e17a8c2d9679e759c9142f7e5534c0 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:20:22 -0400 Subject: [PATCH 08/38] Modify tests to only run for high precision or not --- handwritten/bigquery/test/bigquery.ts | 92 +++++++++++++++++++++++++-- handwritten/bigquery/test/table.ts | 71 ++++++++++++++++----- 2 files changed, 142 insertions(+), 21 deletions(-) diff --git a/handwritten/bigquery/test/bigquery.ts b/handwritten/bigquery/test/bigquery.ts index bc006e70660..e5bbf0e222a 100644 --- a/handwritten/bigquery/test/bigquery.ts +++ b/handwritten/bigquery/test/bigquery.ts @@ -42,6 +42,7 @@ import { TableField, Query, QueryResultsOptions, + QueryOptions, } from '../src'; import {SinonStub} from 'sinon'; import {PreciseDate} from '@google-cloud/precise-date'; @@ -3438,6 +3439,14 @@ describe('BigQuery', () => { delete req[key]; } } + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + timestampOutputFormat: 'ISO8601_STRING', + } + : { + useInt64Timestamp: true, + }; const expectedReq = { query: QUERY_STRING, useLegacySql: false, @@ -3464,13 +3473,78 @@ describe('BigQuery', () => { key: 'value', }, jobCreationMode: 'JOB_CREATION_REQUIRED', - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, }; assert.deepStrictEqual(req, expectedReq); }); + describe('timestamp format options', () => { + const testCases: { + name: string; + opts: QueryOptions; + expected?: any; + bail?: boolean; + }[] = [ + { + name: 'TOF: omitted, UI64: omitted (default ISO8601_STRING)', + opts: {}, + expected: { + timestampOutputFormat: 'ISO8601_STRING', + }, + }, + { + name: 'TOF: omitted, UI64: true', + opts: { + ['formatOptions.useInt64Timestamp']: true, + }, + expected: { + useInt64Timestamp: true, + }, + }, + { + name: 'TOF: omitted, UI64: false (default ISO8601_STRING)', + opts: { + ['formatOptions.useInt64Timestamp']: false, + }, + expected: { + useInt64Timestamp: false, + }, + }, + ]; + + testCases.forEach(testCase => { + it(`should handle ${testCase.name}`, () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } + const req = bq.buildQueryRequest_(QUERY_STRING, testCase.opts); + + const expectedReq = { + query: QUERY_STRING, + useLegacySql: false, + requestId: req.requestId, + jobCreationMode: 'JOB_CREATION_OPTIONAL', + formatOptions: testCase.expected, + connectionProperties: undefined, + continuous: undefined, + createSession: undefined, + defaultDataset: undefined, + destinationEncryptionConfiguration: undefined, + labels: undefined, + location: undefined, + maxResults: undefined, + maximumBytesBilled: undefined, + preserveNulls: undefined, + reservation: undefined, + timeoutMs: undefined, + useQueryCache: undefined, + writeIncrementalResults: undefined, + }; + assert.deepStrictEqual(req, expectedReq); + }); + }); + }); + it('should create a QueryRequest from a SQL string', () => { const req = bq.buildQueryRequest_(QUERY_STRING, {}); for (const key in req) { @@ -3478,14 +3552,20 @@ describe('BigQuery', () => { delete req[key]; } } + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + timestampOutputFormat: 'ISO8601_STRING', + } + : { + useInt64Timestamp: true, + }; const expectedReq = { query: QUERY_STRING, useLegacySql: false, requestId: req.requestId, jobCreationMode: 'JOB_CREATION_OPTIONAL', - formatOptions: { - useInt64Timestamp: true, - }, + formatOptions, }; assert.deepStrictEqual(req, expectedReq); }); diff --git a/handwritten/bigquery/test/table.ts b/handwritten/bigquery/test/table.ts index bb3225459ea..f3e59d7f3d9 100644 --- a/handwritten/bigquery/test/table.ts +++ b/handwritten/bigquery/test/table.ts @@ -2041,12 +2041,20 @@ describe('BigQuery/Table', () => { it('should make correct API request', done => { const options = {a: 'b', c: 'd'}; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.strictEqual(reqOpts.uri, '/data'); assert.deepStrictEqual(reqOpts.qs, { ...options, - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, }); callback(null, {}); }; @@ -2201,13 +2209,16 @@ describe('BigQuery/Table', () => { sandbox.restore(); const mergeStub = sandbox.stub(BigQuery, 'mergeSchemaWithRows_'); - table.getRows({skipParsing: true}, (err: Error, rows_: {}[], nextQuery: {}, apiResponse: any) => { - assert.ifError(err); - assert.strictEqual(rows_, rows); - assert.strictEqual(mergeStub.called, false); - assert.deepStrictEqual(apiResponse.rows, rows); - done(); - }); + table.getRows( + {skipParsing: true}, + (err: Error, rows_: {}[], nextQuery: {}, apiResponse: any) => { + assert.ifError(err); + assert.strictEqual(rows_, rows); + assert.strictEqual(mergeStub.called, false); + assert.deepStrictEqual(apiResponse.rows, rows); + done(); + }, + ); }); it('should pass nextQuery if pageToken is returned', done => { @@ -2217,19 +2228,33 @@ describe('BigQuery/Table', () => { // Set a schema so it doesn't try to refresh the metadata. table.metadata = {schema: {}}; + const callbackResponse = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.useInt64Timestamp': true, + pageToken, + } + : { + pageToken, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { - callback(null, { - 'formatOptions.useInt64Timestamp': true, - pageToken, - }); + callback(null, callbackResponse); }; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.getRows(options, (err: Error, rows: {}, nextQuery: {}) => { assert.ifError(err); assert.deepStrictEqual(nextQuery, { a: 'b', c: 'd', - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, pageToken, }); // Original object isn't affected. @@ -2443,10 +2468,18 @@ describe('BigQuery/Table', () => { const wrapIntegers = {integerTypeCastFunction: sinon.stub()}; const options = {wrapIntegers}; const merged = [{name: 'stephen'}]; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, }); callback(null, {}); }; @@ -2467,10 +2500,18 @@ describe('BigQuery/Table', () => { parseJSON: true, }; const merged = [{name: 'stephen'}]; + const formatOptions = + process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true' + ? { + 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', + } + : { + 'formatOptions.useInt64Timestamp': true, + }; table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { assert.deepStrictEqual(reqOpts.qs, { - 'formatOptions.useInt64Timestamp': true, + ...formatOptions, }); callback(null, {}); }; From bdba4ddf8b4455c29d1d025e12902e7a56f9e9c9 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:28:20 -0400 Subject: [PATCH 09/38] listParams add --- handwritten/bigquery/src/bigquery.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 010bea0e105..187a8ea0762 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2294,6 +2294,7 @@ export class BigQuery extends Service { rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { wrapIntegers: options.wrapIntegers || false, parseJSON: options.parseJSON, + listParams, }); delete res.rows; } From ca0a51cd0edb2cfdb4421ad9e03dab5cb18b1fae Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:46:20 -0400 Subject: [PATCH 10/38] remove listParams --- handwritten/bigquery/src/bigquery.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 187a8ea0762..010bea0e105 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2294,7 +2294,6 @@ export class BigQuery extends Service { rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { wrapIntegers: options.wrapIntegers || false, parseJSON: options.parseJSON, - listParams, }); delete res.rows; } From a1eb4d20d3ef5c111d1b253be99a0a5a95a2de47 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:58:59 -0400 Subject: [PATCH 11/38] skip tests if picosecond support is not turned on --- .../bigquery/system-test/timestamp_output_format.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 0fe388e1e3b..c12cddb7d7d 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -28,7 +28,7 @@ interface TestCase { expectedTsValue?: string; } -describe('Timestamp Output Format System Tests', () => { +describe.only('Timestamp Output Format System Tests', () => { const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; const dataset = bigquery.dataset(datasetId); @@ -159,6 +159,9 @@ describe('Timestamp Output Format System Tests', () => { expectedTsValue, }) => { it(name, async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } const options: {[key: string]: any} = {}; if (timestampOutputFormat !== undefined) { options['formatOptions.timestampOutputFormat'] = @@ -185,6 +188,10 @@ describe('Timestamp Output Format System Tests', () => { ); it('should make a request with ISO8601_STRING when no format options are being used', done => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + done(); + return; + } void (async () => { const originalRequest = table.request; const requestPromise: Promise = new Promise( From a959b3ac379029012124e404c71423d48dc1253c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 1 Apr 2026 16:59:52 -0400 Subject: [PATCH 12/38] remove only --- handwritten/bigquery/system-test/timestamp_output_format.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index c12cddb7d7d..9c3325db632 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -28,7 +28,7 @@ interface TestCase { expectedTsValue?: string; } -describe.only('Timestamp Output Format System Tests', () => { +describe('Timestamp Output Format System Tests', () => { const datasetId = `timestamp_test_${randomUUID().replace(/-/g, '_')}`; const tableId = `timestamp_table_${randomUUID().replace(/-/g, '_')}`; const dataset = bigquery.dataset(datasetId); From f5b7e3199ba8cc9c0741249df6e4d4abfe44cc7c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 13:41:03 -0400 Subject: [PATCH 13/38] =?UTF-8?q?Don=E2=80=99t=20create=20separate=20code?= =?UTF-8?q?=20paths=20when=20not=20needed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handwritten/bigquery/src/bigquery.ts | 48 +++++++++++----------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 010bea0e105..b11b3afa7a3 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2258,45 +2258,33 @@ export class BigQuery extends Service { if (res && res.jobComplete) { let rows: any = []; if (res.schema && res.rows) { - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const listParams = { - 'formatOptions.timestampOutputFormat': - queryReq.formatOptions?.timestampOutputFormat, - 'formatOptions.useInt64Timestamp': - queryReq.formatOptions?.useInt64Timestamp, - }; - if (options.skipParsing) { - rows = res.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { - wrapIntegers: options.wrapIntegers || false, - parseJSON: options.parseJSON, - listParams, - }); - delete res.rows; - } - } catch (e) { - (callback as SimpleQueryRowsCallback)(e as Error, null, job); - return; - } - } else { + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listParams = { + 'formatOptions.timestampOutputFormat': + queryReq.formatOptions?.timestampOutputFormat, + 'formatOptions.useInt64Timestamp': + queryReq.formatOptions?.useInt64Timestamp, + }; if (options.skipParsing) { rows = res.rows; } else { rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { wrapIntegers: options.wrapIntegers || false, parseJSON: options.parseJSON, + listParams, }); delete res.rows; } + } catch (e) { + (callback as SimpleQueryRowsCallback)(e as Error, null, job); + return; } } this.trace_('[runJobsQuery] job complete'); From 61db403eff7cb8778659d5821458d3cde279a90b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 14:08:31 -0400 Subject: [PATCH 14/38] extend the default options --- handwritten/bigquery/src/bigquery.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index b11b3afa7a3..fbfcf2bd907 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2381,9 +2381,15 @@ export class BigQuery extends Service { useInt64Timestamp: options['formatOptions.useInt64Timestamp'], }); } else { - formatOptions = { - useInt64Timestamp: true, - }; + formatOptions = extend( + { + useInt64Timestamp: true, + }, + { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }, + ); } const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, From 01ab527f4c1d4b892cc9111aa0bf5918c689d46f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 14:09:39 -0400 Subject: [PATCH 15/38] Skip the before hook if picosecond support not on --- handwritten/bigquery/system-test/timestamp_output_format.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/handwritten/bigquery/system-test/timestamp_output_format.ts b/handwritten/bigquery/system-test/timestamp_output_format.ts index 9c3325db632..bd622b9dfec 100644 --- a/handwritten/bigquery/system-test/timestamp_output_format.ts +++ b/handwritten/bigquery/system-test/timestamp_output_format.ts @@ -38,6 +38,9 @@ describe('Timestamp Output Format System Tests', () => { const expectedTsValuePicoseconds = '2023-01-01T12:00:00.123456789123Z'; before(async () => { + if (process.env.BIGQUERY_PICOSECOND_SUPPORT !== 'true') { + return; + } await dataset.create(); await table.create({ schema: [{name: 'ts', type: 'TIMESTAMP', timestampPrecision: '12'}], From 67a48221d5433051d12bdf3e0d1664b7e32c7783 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 15:28:22 -0400 Subject: [PATCH 16/38] Add documentation back in for methods --- handwritten/bigquery/src/bigquery.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index fbfcf2bd907..f13a491ac4b 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2730,6 +2730,14 @@ export class BigQueryRange { }; } + /** + * This method returns start and end values for RANGE typed values returned from + * the server. It decodes the server RANGE value into start and end values so + * they can be used to construct a BigQueryRange. + * @private + * @param {string} value The range value. + * @returns {string[]} The start and end of the range. + */ private static fromStringValue_(value: string): [start: string, end: string] { let cleanedValue = value; if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { @@ -2749,6 +2757,17 @@ export class BigQueryRange { return [start, end]; } + /** + * This method is only used by convertSchemaFieldValue and only when range + * values are passed into convertSchemaFieldValue. It produces a value that is + * delivered to the user for read calls and it needs to pass along listParams + * to ensure TIMESTAMP types are converted properly. + * @private + * @param {string} value The range value. + * @param {string} elementType The element type. + * @param {bigquery.tabledata.IListParams | bigquery.jobs.IGetQueryResultsParams} [listParams] The list parameters. + * @returns {BigQueryRange} + */ static fromSchemaValue_( value: string, elementType: string, From 757dfdcf8ad78b2bea8272e3ef1513f9bf0f3053 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 15:39:48 -0400 Subject: [PATCH 17/38] Remove the if block since release is done --- handwritten/bigquery/src/bigquery.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index f13a491ac4b..23f1ec680bd 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2858,18 +2858,6 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { - /* - TODO: - When https://github.com/googleapis/nodejs-precise-date/pull/302 - is released and we have full support for picoseconds in PreciseData - then we can remove this if block. - */ - this.value = value; - return; - } - } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { From cbeeacc199289a9e20cc0edefd348bab4745c0d3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 15:46:17 -0400 Subject: [PATCH 18/38] Eliminate redundant feature flag check. --- handwritten/bigquery/src/job.ts | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/handwritten/bigquery/src/job.ts b/handwritten/bigquery/src/job.ts index a0a2b45c1ad..22554a1d8f3 100644 --- a/handwritten/bigquery/src/job.ts +++ b/handwritten/bigquery/src/job.ts @@ -596,28 +596,13 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - try { - /* - Without this try/catch block, calls to /query endpoint will hang - indefinitely if a call to mergeSchemaWithRows_ fails because the - error never makes it to the callback. Instead, pass the error to the - callback the user provides so that the user can see the error. - */ - if (options.skipParsing) { - rows = resp.rows; - } else { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, { - wrapIntegers, - parseJSON, - }); - delete resp.rows; - } - } catch (e) { - callback!(e as Error, null, null, resp); - return; - } - } else { + try { + /* + Without this try/catch block, calls to /query endpoint will hang + indefinitely if a call to mergeSchemaWithRows_ fails because the + error never makes it to the callback. Instead, pass the error to the + callback the user provides so that the user can see the error. + */ if (options.skipParsing) { rows = resp.rows; } else { @@ -627,6 +612,9 @@ class Job extends Operation { }); delete resp.rows; } + } catch (e) { + callback!(e as Error, null, null, resp); + return; } } From 83cfcd868c801e37d26e7ecc4a6d1bceebb44bfe Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 17:12:07 -0400 Subject: [PATCH 19/38] Add qs back in --- handwritten/bigquery/src/table.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index f425a21caef..343a0eadc0c 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -1905,6 +1905,7 @@ class Table extends ServiceObject { wrapIntegers, selectedFields, parseJSON, + listParams: qs, }, ); } From 403093a8f155aa80957c1959e1c1d60e22be7adb Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 2 Apr 2026 17:24:22 -0400 Subject: [PATCH 20/38] Add commands to test for picoseconds in system t --- handwritten/bigquery/package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/handwritten/bigquery/package.json b/handwritten/bigquery/package.json index e0e3039084d..e40a93135c1 100644 --- a/handwritten/bigquery/package.json +++ b/handwritten/bigquery/package.json @@ -37,7 +37,9 @@ "lint": "gts check", "samples-test": "cd samples/ && npm link ../ && npm test && cd ../", "test": "c8 mocha build/test", - "system-test": "mocha build/system-test --timeout 600000", + "system-test:pico": "BIGQUERY_PICOSECOND_SUPPORT=true mocha build/system-test --timeout 600000", + "system-test:standard": "mocha build/system-test --timeout 600000", + "system-test": "npm run system-test:pico && npm run system-test:standard", "presystem-test": "npm run compile", "clean": "gts clean", "compile": "tsc -p . && cp src/types.d.ts build/src/", From 83125de94dea7d833f173507be49c73e9daa5055 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 09:35:54 -0400 Subject: [PATCH 21/38] Revert "Remove the if block since release is done" This reverts commit 757dfdcf8ad78b2bea8272e3ef1513f9bf0f3053. --- handwritten/bigquery/src/bigquery.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 23f1ec680bd..f13a491ac4b 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2858,6 +2858,18 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); + if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { + if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { + /* + TODO: + When https://github.com/googleapis/nodejs-precise-date/pull/302 + is released and we have full support for picoseconds in PreciseData + then we can remove this if block. + */ + this.value = value; + return; + } + } } else { const floatValue = Number.parseFloat(value); if (!Number.isNaN(floatValue)) { From 4919e9f6034a05e3f2c64ae08c63f847d4620620 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 10:31:32 -0400 Subject: [PATCH 22/38] Eliminate environment variable for parsing check --- handwritten/bigquery/src/bigquery.ts | 73 ++++++++++++---------------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index f13a491ac4b..4065944f1a8 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2591,48 +2591,39 @@ function convertSchemaFieldValue( break; } case 'TIMESTAMP': { - // High precision timestamp behaviour - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - /* - At this point, 'value' will equal the timestamp value returned from the - server. We need to parse this value differently depending on its format. - For example, value could be any of the following: - 1672574400123456 - 1672574400.123456 - 2023-01-01T12:00:00.123456789123Z - */ - const listParams = options.listParams; - const timestampOutputFormat = listParams - ? listParams['formatOptions.timestampOutputFormat'] - : undefined; - const useInt64Timestamp = listParams - ? listParams['formatOptions.useInt64Timestamp'] - : undefined; - if (timestampOutputFormat === 'ISO8601_STRING') { - // value is ISO string, create BigQueryTimestamp wrapping the string - value = BigQuery.timestamp(value); - } else if ( - useInt64Timestamp !== true && - timestampOutputFormat !== 'INT64' && - (useInt64Timestamp !== undefined || - timestampOutputFormat !== undefined) - ) { - // NOTE: The additional - // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) - // check is to ensure that calls to the /query endpoint remain - // unaffected as they will not be providing any listParams. - // - // If the program reaches this point in time then - // value is float seconds so convert to BigQueryTimestamp - value = BigQuery.timestamp(Number(value)); - } else { - // Expect int64 micros (default or explicit INT64) - const pd = new PreciseDate(); - pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); - value = BigQuery.timestamp(pd); - } + /* + At this point, 'value' will equal the timestamp value returned from the + server. We need to parse this value differently depending on its format. + For example, value could be any of the following: + 1672574400123456 + 1672574400.123456 + 2023-01-01T12:00:00.123456789123Z + */ + const listParams = options.listParams; + const timestampOutputFormat = listParams + ? listParams['formatOptions.timestampOutputFormat'] + : undefined; + const useInt64Timestamp = listParams + ? listParams['formatOptions.useInt64Timestamp'] + : undefined; + if (timestampOutputFormat === 'ISO8601_STRING') { + // value is ISO string, create BigQueryTimestamp wrapping the string + value = BigQuery.timestamp(value); + } else if ( + useInt64Timestamp !== true && + timestampOutputFormat !== 'INT64' && + (useInt64Timestamp !== undefined || timestampOutputFormat !== undefined) + ) { + // NOTE: The additional + // (useInt64Timestamp !== undefined || timestampOutputFormat !== und...) + // check is to ensure that calls to the /query endpoint remain + // unaffected as they will not be providing any listParams. + // + // If the program reaches this point in time then + // value is float seconds so convert to BigQueryTimestamp + value = BigQuery.timestamp(Number(value)); } else { - // Old behaviour + // Expect int64 micros (default or explicit INT64) const pd = new PreciseDate(); pd.setFullTime(PreciseDate.parseFull(BigInt(value) * BigInt(1000))); value = BigQuery.timestamp(pd); From 901313bea253fef1caa9d12972a87404df7479c7 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 10:40:17 -0400 Subject: [PATCH 23/38] Always use try block --- handwritten/bigquery/src/table.ts | 39 ++++++++----------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index 343a0eadc0c..e93e2b42f94 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -1866,35 +1866,13 @@ class Table extends ServiceObject { callback!(err, null, null, resp); return; } - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - // High precision timestamp behaviour - try { - /* - Without this try/catch block, calls to getRows will hang indefinitely if - a call to mergeSchemaWithRows_ fails because the error never makes it to - the callback. Instead, pass the error to the callback the user provides - so that the user can see the error. - */ - if (options.skipParsing) { - rows = rows || []; - } else { - rows = BigQuery.mergeSchemaWithRows_( - this.metadata.schema, - rows || [], - { - wrapIntegers, - selectedFields, - parseJSON, - listParams: qs, - }, - ); - } - } catch (err) { - callback!(err as Error | null, null, null, resp); - return; - } - } else { - // Old behaviour + try { + /* + Without this try/catch block, calls to getRows will hang indefinitely if + a call to mergeSchemaWithRows_ fails because the error never makes it to + the callback. Instead, pass the error to the callback the user provides + so that the user can see the error. + */ if (options.skipParsing) { rows = rows || []; } else { @@ -1909,6 +1887,9 @@ class Table extends ServiceObject { }, ); } + } catch (err) { + callback!(err as Error | null, null, null, resp); + return; } callback!(null, rows, nextQuery, resp); }; From 4fb5ce288acc85c8539104a623aaeb48f504cb18 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 11:09:13 -0400 Subject: [PATCH 24/38] Change type to GetRowsOptions --- handwritten/bigquery/src/table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index e93e2b42f94..f50016e1ea3 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -1894,7 +1894,7 @@ class Table extends ServiceObject { callback!(null, rows, nextQuery, resp); }; - let qs: any; + let qs: GetRowsOptions; if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { const hasAnyFormatOpts = options['formatOptions.timestampOutputFormat'] !== undefined || From ceaf6c90a506961134ac5e2a3a2e346da43a4c16 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 11:27:45 -0400 Subject: [PATCH 25/38] Remove exception --- handwritten/bigquery/src/bigquery.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 4065944f1a8..c4b1556bb67 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2265,7 +2265,6 @@ export class BigQuery extends Service { the callback. Instead, pass the error to the callback the user provides so that the user can see the error. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any const listParams = { 'formatOptions.timestampOutputFormat': queryReq.formatOptions?.timestampOutputFormat, From c58058b18e8ac38875eb18a094a9d329db3f1e61 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 11:33:37 -0400 Subject: [PATCH 26/38] Some typescript simplifications --- handwritten/bigquery/src/bigquery.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index c4b1556bb67..3d8372a1d5e 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2599,12 +2599,9 @@ function convertSchemaFieldValue( 2023-01-01T12:00:00.123456789123Z */ const listParams = options.listParams; - const timestampOutputFormat = listParams - ? listParams['formatOptions.timestampOutputFormat'] - : undefined; - const useInt64Timestamp = listParams - ? listParams['formatOptions.useInt64Timestamp'] - : undefined; + const timestampOutputFormat = + listParams?.['formatOptions.timestampOutputFormat']; + const useInt64Timestamp = listParams?.['formatOptions.useInt64Timestamp']; if (timestampOutputFormat === 'ISO8601_STRING') { // value is ISO string, create BigQueryTimestamp wrapping the string value = BigQuery.timestamp(value); From 17593f5de7035e4a16963517898fe3ea58ba050e Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 16:06:13 -0400 Subject: [PATCH 27/38] Eliminate default options branching --- handwritten/bigquery/src/bigquery.ts | 34 ++++++++++++---------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 3d8372a1d5e..fb4ce371e89 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -54,7 +54,7 @@ import { GoogleErrorBody, RetryOptions, } from '@google-cloud/common/build/src/util'; -import bigquery from './types'; +import bigquery, {IDataFormatOptions} from './types'; import {logger, setLogFunction} from './logger'; // Third-Party Re-exports @@ -2365,31 +2365,25 @@ export class BigQuery extends Service { if (options.job) { return undefined; } - let formatOptions; + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + let defaultOpts: IDataFormatOptions = hasAnyFormatOpts + ? {} + : { + useInt64Timestamp: true, + }; if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts + defaultOpts = hasAnyFormatOpts ? {} : { timestampOutputFormat: 'ISO8601_STRING', }; - formatOptions = extend(defaultOpts, { - timestampOutputFormat: options['formatOptions.timestampOutputFormat'], - useInt64Timestamp: options['formatOptions.useInt64Timestamp'], - }); - } else { - formatOptions = extend( - { - useInt64Timestamp: true, - }, - { - timestampOutputFormat: options['formatOptions.timestampOutputFormat'], - useInt64Timestamp: options['formatOptions.useInt64Timestamp'], - }, - ); } + const formatOptions = extend(defaultOpts, { + timestampOutputFormat: options['formatOptions.timestampOutputFormat'], + useInt64Timestamp: options['formatOptions.useInt64Timestamp'], + }); const req: bigquery.IQueryRequest = { useQueryCache: queryObj.useQueryCache, labels: queryObj.labels, From 8d6420b6f3f614a55d23f5064034bea77923c91d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 16:14:50 -0400 Subject: [PATCH 28/38] Change defaults for current requests --- handwritten/bigquery/src/table.ts | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/handwritten/bigquery/src/table.ts b/handwritten/bigquery/src/table.ts index f50016e1ea3..6993ac58abf 100644 --- a/handwritten/bigquery/src/table.ts +++ b/handwritten/bigquery/src/table.ts @@ -1894,25 +1894,22 @@ class Table extends ServiceObject { callback!(null, rows, nextQuery, resp); }; - let qs: GetRowsOptions; + const hasAnyFormatOpts = + options['formatOptions.timestampOutputFormat'] !== undefined || + options['formatOptions.useInt64Timestamp'] !== undefined; + let defaultOpts: GetRowsOptions = hasAnyFormatOpts + ? {} + : { + 'formatOptions.useInt64Timestamp': true, + }; if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - const hasAnyFormatOpts = - options['formatOptions.timestampOutputFormat'] !== undefined || - options['formatOptions.useInt64Timestamp'] !== undefined; - const defaultOpts = hasAnyFormatOpts + defaultOpts = hasAnyFormatOpts ? {} : { 'formatOptions.timestampOutputFormat': 'ISO8601_STRING', }; - qs = extend(defaultOpts, options); - } else { - qs = extend( - { - 'formatOptions.useInt64Timestamp': true, - }, - options, - ); } + const qs: GetRowsOptions = extend(defaultOpts, options); this.request( { From 800a818ec39b9d9928b39393ee3c24ac1bf86146 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 16:17:47 -0400 Subject: [PATCH 29/38] =?UTF-8?q?Don=E2=80=99t=20need=20extra=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handwritten/bigquery/src/bigquery.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index fb4ce371e89..0220d1b2aef 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -54,7 +54,7 @@ import { GoogleErrorBody, RetryOptions, } from '@google-cloud/common/build/src/util'; -import bigquery, {IDataFormatOptions} from './types'; +import bigquery from './types'; import {logger, setLogFunction} from './logger'; // Third-Party Re-exports @@ -2368,7 +2368,7 @@ export class BigQuery extends Service { const hasAnyFormatOpts = options['formatOptions.timestampOutputFormat'] !== undefined || options['formatOptions.useInt64Timestamp'] !== undefined; - let defaultOpts: IDataFormatOptions = hasAnyFormatOpts + let defaultOpts: bigquery.IDataFormatOptions = hasAnyFormatOpts ? {} : { useInt64Timestamp: true, From dc3588fb836e501c6945af473cd10ba0b8b84036 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 7 Apr 2026 17:04:57 -0400 Subject: [PATCH 30/38] =?UTF-8?q?Don=E2=80=99t=20gate=20with=20picosecond?= =?UTF-8?q?=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handwritten/bigquery/src/bigquery.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/handwritten/bigquery/src/bigquery.ts b/handwritten/bigquery/src/bigquery.ts index 0220d1b2aef..e0e8f1411e2 100644 --- a/handwritten/bigquery/src/bigquery.ts +++ b/handwritten/bigquery/src/bigquery.ts @@ -2839,17 +2839,15 @@ export class BigQueryTimestamp { } else if (typeof value === 'string') { if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) { pd = new PreciseDate(value); - if (process.env.BIGQUERY_PICOSECOND_SUPPORT === 'true') { - if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { - /* - TODO: - When https://github.com/googleapis/nodejs-precise-date/pull/302 - is released and we have full support for picoseconds in PreciseData - then we can remove this if block. - */ - this.value = value; - return; - } + if (value.match(/\.\d{10,}/) && !Number.isNaN(pd.getTime())) { + /* + TODO: + When https://github.com/googleapis/nodejs-precise-date/pull/302 + is released and we have full support for picoseconds in PreciseData + then we can remove this if block. + */ + this.value = value; + return; } } else { const floatValue = Number.parseFloat(value); From bd434645a87597b63d25c136baa8bcbdc562e75d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 8 Apr 2026 11:28:51 -0400 Subject: [PATCH 31/38] Just add an extra test file --- handwritten/bigquery/package.json | 4 +--- handwritten/bigquery/system-test/picosecond.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 handwritten/bigquery/system-test/picosecond.ts diff --git a/handwritten/bigquery/package.json b/handwritten/bigquery/package.json index e40a93135c1..e0e3039084d 100644 --- a/handwritten/bigquery/package.json +++ b/handwritten/bigquery/package.json @@ -37,9 +37,7 @@ "lint": "gts check", "samples-test": "cd samples/ && npm link ../ && npm test && cd ../", "test": "c8 mocha build/test", - "system-test:pico": "BIGQUERY_PICOSECOND_SUPPORT=true mocha build/system-test --timeout 600000", - "system-test:standard": "mocha build/system-test --timeout 600000", - "system-test": "npm run system-test:pico && npm run system-test:standard", + "system-test": "mocha build/system-test --timeout 600000", "presystem-test": "npm run compile", "clean": "gts clean", "compile": "tsc -p . && cp src/types.d.ts build/src/", diff --git a/handwritten/bigquery/system-test/picosecond.ts b/handwritten/bigquery/system-test/picosecond.ts new file mode 100644 index 00000000000..17e30d8a5cf --- /dev/null +++ b/handwritten/bigquery/system-test/picosecond.ts @@ -0,0 +1,12 @@ +import { execSync } from 'child_process'; +import { describe, it } from 'mocha'; +import * as path from 'path'; + +describe('bigquery.ts with BIGQUERY_PICOSECOND_SUPPORT', () => { + it('should run bigquery.ts tests with picosecond support enabled', function () { + this.timeout(700000); + const testFile = path.resolve('system-test/bigquery.ts'); + const command = `BIGQUERY_PICOSECOND_SUPPORT=true npx mocha ${testFile} --timeout 600000`; + execSync(command, { stdio: 'inherit' }); + }); +}); From 781266f4e6251d568a1a434ad003e8b71d04adb1 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 7 Apr 2026 22:09:04 +0000 Subject: [PATCH 32/38] test(bigquery): parameterize system tests for picosecond support Wraps the BigQuery system tests in a parameterized function to run them both with and without the BIGQUERY_PICOSECOND_SUPPORT environment variable. Clients and resources are re-instantiated within a before hook to ensure the library picks up the environment changes for each run. The existing indentation is preserved to maintain a clean PR diff. Co-authored-by: danieljbruce <8935272+danieljbruce@users.noreply.github.com> --- handwritten/bigquery/system-test/bigquery.ts | 40 ++++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index 8a104d9d9db..c447bd39735 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -38,17 +38,39 @@ import { } from '../src'; import bq from '../src/types'; -const bigquery = new BigQuery(); -const storage = new Storage(); +let bigquery = new BigQuery(); +let storage = new Storage(); + +const runTests = (supportPicoseconds: boolean | undefined) => { + const label = supportPicoseconds ? 'with pico' : 'without pico'; + +describe(`BigQuery (${label})`, () => { + const originalValue = process.env.BIGQUERY_PICOSECOND_SUPPORT; + + let dataset = bigquery.dataset(generateName('dataset')); + let table = dataset.table(generateName('table')); + let bucket = storage.bucket(generateName('bucket')); + + before(() => { + if (supportPicoseconds === undefined) { + delete process.env.BIGQUERY_PICOSECOND_SUPPORT; + } else { + process.env.BIGQUERY_PICOSECOND_SUPPORT = 'true'; + } + bigquery = new BigQuery(); + storage = new Storage(); + dataset = bigquery.dataset(generateName('dataset')); + table = dataset.table(generateName('table')); + bucket = storage.bucket(generateName('bucket')); + }); + + after(() => { + process.env.BIGQUERY_PICOSECOND_SUPPORT = originalValue; + }); -describe('BigQuery', () => { const GCLOUD_TESTS_PREFIX = 'nodejs_bq_test'; const minCreationTime = Date.now().toString(); - const dataset = bigquery.dataset(generateName('dataset')); - const table = dataset.table(generateName('table')); - const bucket = storage.bucket(generateName('bucket')); - const query = 'SELECT url FROM `publicdata.samples.github_nested` LIMIT 100'; const SCHEMA = [ @@ -2390,3 +2412,7 @@ describe('BigQuery', () => { } } }); +}; + +runTests(true); +runTests(undefined); From ee5a084e0798e9ea4e035625926e15cbbf715fa7 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 15:26:59 +0000 Subject: [PATCH 33/38] test(bigquery): add high precision timestamp wrapper system test Creates a new system test file that programmatically re-runs the entire BigQuery system test suite with BIGQUERY_PICOSECOND_SUPPORT=true. This is achieved by clearing the Node.js require cache for the package and requiring the existing bigquery.ts test file within a new describe block. This approach leaves the original test file untouched, ensuring a clean PR diff and avoiding linting issues. Co-authored-by: danieljbruce <8935272+danieljbruce@users.noreply.github.com> --- handwritten/bigquery/system-test/bigquery.ts | 40 ++--------- .../system-test/high_precision_timestamp.ts | 72 +++++++++++++++++++ 2 files changed, 79 insertions(+), 33 deletions(-) create mode 100644 handwritten/bigquery/system-test/high_precision_timestamp.ts diff --git a/handwritten/bigquery/system-test/bigquery.ts b/handwritten/bigquery/system-test/bigquery.ts index c447bd39735..8a104d9d9db 100644 --- a/handwritten/bigquery/system-test/bigquery.ts +++ b/handwritten/bigquery/system-test/bigquery.ts @@ -38,39 +38,17 @@ import { } from '../src'; import bq from '../src/types'; -let bigquery = new BigQuery(); -let storage = new Storage(); - -const runTests = (supportPicoseconds: boolean | undefined) => { - const label = supportPicoseconds ? 'with pico' : 'without pico'; - -describe(`BigQuery (${label})`, () => { - const originalValue = process.env.BIGQUERY_PICOSECOND_SUPPORT; - - let dataset = bigquery.dataset(generateName('dataset')); - let table = dataset.table(generateName('table')); - let bucket = storage.bucket(generateName('bucket')); - - before(() => { - if (supportPicoseconds === undefined) { - delete process.env.BIGQUERY_PICOSECOND_SUPPORT; - } else { - process.env.BIGQUERY_PICOSECOND_SUPPORT = 'true'; - } - bigquery = new BigQuery(); - storage = new Storage(); - dataset = bigquery.dataset(generateName('dataset')); - table = dataset.table(generateName('table')); - bucket = storage.bucket(generateName('bucket')); - }); - - after(() => { - process.env.BIGQUERY_PICOSECOND_SUPPORT = originalValue; - }); +const bigquery = new BigQuery(); +const storage = new Storage(); +describe('BigQuery', () => { const GCLOUD_TESTS_PREFIX = 'nodejs_bq_test'; const minCreationTime = Date.now().toString(); + const dataset = bigquery.dataset(generateName('dataset')); + const table = dataset.table(generateName('table')); + const bucket = storage.bucket(generateName('bucket')); + const query = 'SELECT url FROM `publicdata.samples.github_nested` LIMIT 100'; const SCHEMA = [ @@ -2412,7 +2390,3 @@ describe(`BigQuery (${label})`, () => { } } }); -}; - -runTests(true); -runTests(undefined); diff --git a/handwritten/bigquery/system-test/high_precision_timestamp.ts b/handwritten/bigquery/system-test/high_precision_timestamp.ts new file mode 100644 index 00000000000..43c71f4eafd --- /dev/null +++ b/handwritten/bigquery/system-test/high_precision_timestamp.ts @@ -0,0 +1,72 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {describe, before, after} from 'mocha'; +import * as path from 'path'; + +// This file registers a second run of the BigQuery system tests with picosecond support enabled. +// It uses targeted cache clearing to ensure the library and tests are re-initialized with the +// 'BIGQUERY_PICOSECOND_SUPPORT' environment variable set. + +const originalValue = process.env.BIGQUERY_PICOSECOND_SUPPORT; + +/** + * Helper to clear the require cache for all files within the current package. + */ +function clearPackageCache() { + // Find the root of the current package by looking for package.json + const packageJsonPath = require.resolve('../package.json'); + const packageDir = path.dirname(packageJsonPath); + + Object.keys(require.cache).forEach((key) => { + if (key.startsWith(packageDir) && !key.includes('node_modules')) { + delete require.cache[key]; + } + }); +} + +// 1. REGISTRATION PHASE: +// Set the environment variable and clear the cache so that the subsequent 'require' +// of the system test file (and any library files it imports) sees the updated state. +process.env.BIGQUERY_PICOSECOND_SUPPORT = 'true'; +clearPackageCache(); + +describe('BigQuery System Tests (High Precision)', () => { + // 2. EXECUTION PHASE: + // Mocha runs 'before' hooks before the tests in the required file are executed. + before(() => { + process.env.BIGQUERY_PICOSECOND_SUPPORT = 'true'; + // We don't clear cache here as it's too late for Mocha's registration phase, + // but we ensure the env var is set for any execution-time checks. + }); + + after(() => { + if (originalValue === undefined) { + delete process.env.BIGQUERY_PICOSECOND_SUPPORT; + } else { + process.env.BIGQUERY_PICOSECOND_SUPPORT = originalValue; + } + }); + + // Programmatically require the tests. Mocha will discover and register them inside this 'describe' block. + require('./bigquery'); +}); + +// 3. CLEANUP: +// Restore the environment variable after the registration phase to minimize side effects on other test files. +if (originalValue === undefined) { + delete process.env.BIGQUERY_PICOSECOND_SUPPORT; +} else { + process.env.BIGQUERY_PICOSECOND_SUPPORT = originalValue; +} From 9e7b45250014bb69c2d1df0b1dbe53605031fb6f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 8 Apr 2026 11:37:18 -0400 Subject: [PATCH 34/38] linting change and copyright year --- handwritten/bigquery/system-test/high_precision_timestamp.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handwritten/bigquery/system-test/high_precision_timestamp.ts b/handwritten/bigquery/system-test/high_precision_timestamp.ts index 43c71f4eafd..0d538c31a1b 100644 --- a/handwritten/bigquery/system-test/high_precision_timestamp.ts +++ b/handwritten/bigquery/system-test/high_precision_timestamp.ts @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ function clearPackageCache() { const packageJsonPath = require.resolve('../package.json'); const packageDir = path.dirname(packageJsonPath); - Object.keys(require.cache).forEach((key) => { + Object.keys(require.cache).forEach(key => { if (key.startsWith(packageDir) && !key.includes('node_modules')) { delete require.cache[key]; } From 8707c1998041ac1bc0511f668ba25126098f6ec3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 8 Apr 2026 11:46:06 -0400 Subject: [PATCH 35/38] Correct package.json errors --- handwritten/bigquery/system-test/high_precision_timestamp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handwritten/bigquery/system-test/high_precision_timestamp.ts b/handwritten/bigquery/system-test/high_precision_timestamp.ts index 0d538c31a1b..daed1897284 100644 --- a/handwritten/bigquery/system-test/high_precision_timestamp.ts +++ b/handwritten/bigquery/system-test/high_precision_timestamp.ts @@ -26,7 +26,7 @@ const originalValue = process.env.BIGQUERY_PICOSECOND_SUPPORT; */ function clearPackageCache() { // Find the root of the current package by looking for package.json - const packageJsonPath = require.resolve('../package.json'); + const packageJsonPath = require.resolve('../../package.json'); const packageDir = path.dirname(packageJsonPath); Object.keys(require.cache).forEach(key => { From 33022c928cc0635e518348bb46be24dd346c6dca Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 8 Apr 2026 11:59:13 -0400 Subject: [PATCH 36/38] This file is not working well with test runner --- handwritten/bigquery/system-test/picosecond.ts | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 handwritten/bigquery/system-test/picosecond.ts diff --git a/handwritten/bigquery/system-test/picosecond.ts b/handwritten/bigquery/system-test/picosecond.ts deleted file mode 100644 index 17e30d8a5cf..00000000000 --- a/handwritten/bigquery/system-test/picosecond.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { execSync } from 'child_process'; -import { describe, it } from 'mocha'; -import * as path from 'path'; - -describe('bigquery.ts with BIGQUERY_PICOSECOND_SUPPORT', () => { - it('should run bigquery.ts tests with picosecond support enabled', function () { - this.timeout(700000); - const testFile = path.resolve('system-test/bigquery.ts'); - const command = `BIGQUERY_PICOSECOND_SUPPORT=true npx mocha ${testFile} --timeout 600000`; - execSync(command, { stdio: 'inherit' }); - }); -}); From e003fa23c69b7284f3d137842776ddad042cbad9 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 8 Apr 2026 13:21:01 -0400 Subject: [PATCH 37/38] Run the timestamp output format tests as well --- handwritten/bigquery/system-test/high_precision_timestamp.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/handwritten/bigquery/system-test/high_precision_timestamp.ts b/handwritten/bigquery/system-test/high_precision_timestamp.ts index daed1897284..beed685ac6f 100644 --- a/handwritten/bigquery/system-test/high_precision_timestamp.ts +++ b/handwritten/bigquery/system-test/high_precision_timestamp.ts @@ -61,6 +61,7 @@ describe('BigQuery System Tests (High Precision)', () => { // Programmatically require the tests. Mocha will discover and register them inside this 'describe' block. require('./bigquery'); + require('./timestamp_output_format'); }); // 3. CLEANUP: From 0d67c7e9ae5939bbe80e2fd7b8b5059d52cf6b3a Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 8 Apr 2026 13:35:02 -0400 Subject: [PATCH 38/38] Add a wrapper --- .../bigquery/system-test/high_precision_timestamp.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/handwritten/bigquery/system-test/high_precision_timestamp.ts b/handwritten/bigquery/system-test/high_precision_timestamp.ts index beed685ac6f..17c32320980 100644 --- a/handwritten/bigquery/system-test/high_precision_timestamp.ts +++ b/handwritten/bigquery/system-test/high_precision_timestamp.ts @@ -59,9 +59,14 @@ describe('BigQuery System Tests (High Precision)', () => { } }); - // Programmatically require the tests. Mocha will discover and register them inside this 'describe' block. - require('./bigquery'); - require('./timestamp_output_format'); + describe('Run all tests', () => { + // Wrap all the other tests in a describe block to ensure the entire file is + // executed when the environment variable is set to true. + + // Programmatically require the tests. Mocha will discover and register them inside this 'describe' block. + require('./bigquery'); + require('./timestamp_output_format'); + }); }); // 3. CLEANUP: