Skip to content

Commit cdd38a6

Browse files
stephenpluspluscallmehiphop
stephenplusplus
authored and
callmehiphop
committed
bigquery: convert nested objects to their native types (#1648)
1 parent dd5a482 commit cdd38a6

4 files changed

Lines changed: 280 additions & 53 deletions

File tree

‎packages/bigquery/src/table.js‎

Lines changed: 82 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,39 @@ Table.createSchemaFromString_ = function(str) {
177177
});
178178
};
179179

180+
/**
181+
* Convert a row entry from native types to their encoded types that the API
182+
* expects.
183+
*
184+
* @static
185+
* @private
186+
*
187+
* @param {*} value - The value to be converted.
188+
* @return {*} The converted value.
189+
*/
190+
Table.encodeValue_ = function(value) {
191+
if (value instanceof Buffer) {
192+
return value.toString('base64');
193+
}
194+
195+
if (is.date(value)) {
196+
return value.toJSON();
197+
}
198+
199+
if (is.array(value)) {
200+
return value.map(Table.encodeValue_);
201+
}
202+
203+
if (is.object(value)) {
204+
return Object.keys(value).reduce(function(acc, key) {
205+
acc[key] = Table.encodeValue_(value[key]);
206+
return acc;
207+
}, {});
208+
}
209+
210+
return value;
211+
};
212+
180213
/**
181214
* Merge a rowset returned from the API with a table schema.
182215
*
@@ -188,34 +221,19 @@ Table.createSchemaFromString_ = function(str) {
188221
* @return {array} Fields using their matching names from the table's schema.
189222
*/
190223
Table.mergeSchemaWithRows_ = function(schema, rows) {
191-
return rows.map(mergeSchema).map(flattenRows);
224+
return arrify(rows).map(mergeSchema).map(flattenRows);
192225

193226
function mergeSchema(row) {
194227
return row.f.map(function(field, index) {
195228
var schemaField = schema.fields[index];
196229
var value = field.v;
197230

198-
switch (schemaField.type) {
199-
case 'BOOLEAN': {
200-
value = value === 'true';
201-
break;
202-
}
203-
case 'FLOAT': {
204-
if (!is.nil(value)) {
205-
value = parseFloat(value);
206-
}
207-
break;
208-
}
209-
case 'INTEGER': {
210-
if (!is.nil(value)) {
211-
value = parseInt(value, 10);
212-
}
213-
break;
214-
}
215-
case 'TIMESTAMP': {
216-
value = new Date(value * 1000);
217-
break;
218-
}
231+
if (schemaField.mode === 'REPEATED') {
232+
value = value.map(function(val) {
233+
return convert(schemaField, val.v);
234+
});
235+
} else {
236+
value = convert(schemaField, value);
219237
}
220238

221239
var fieldObject = {};
@@ -224,6 +242,41 @@ Table.mergeSchemaWithRows_ = function(schema, rows) {
224242
});
225243
}
226244

245+
function convert(schemaField, value) {
246+
if (is.nil(value)) {
247+
return value;
248+
}
249+
250+
switch (schemaField.type) {
251+
case 'BOOLEAN': {
252+
value = value === 'true';
253+
break;
254+
}
255+
case 'BYTES': {
256+
value = new Buffer(value, 'base64');
257+
break;
258+
}
259+
case 'FLOAT': {
260+
value = parseFloat(value);
261+
break;
262+
}
263+
case 'INTEGER': {
264+
value = parseInt(value, 10);
265+
break;
266+
}
267+
case 'RECORD': {
268+
value = Table.mergeSchemaWithRows_(schemaField, value).pop();
269+
break;
270+
}
271+
case 'TIMESTAMP': {
272+
value = new Date(value * 1000);
273+
break;
274+
}
275+
}
276+
277+
return value;
278+
}
279+
227280
function flattenRows(rows) {
228281
return rows.reduce(function(acc, row) {
229282
var key = Object.keys(row)[0];
@@ -925,7 +978,7 @@ Table.prototype.insert = function(rows, options, callback) {
925978
if (!options.raw) {
926979
json.rows = arrify(rows).map(function(row) {
927980
return {
928-
json: row
981+
json: Table.encodeValue_(row)
929982
};
930983
});
931984
}
@@ -977,11 +1030,12 @@ Table.prototype.query = function(query, callback) {
9771030
* table.
9781031
* @param {string} metadata.name - A descriptive name for the table.
9791032
* @param {string|object} metadata.schema - A comma-separated list of name:type
980-
* pairs. Valid types are "string", "integer", "float", "boolean", and
981-
* "timestamp". If the type is omitted, it is assumed to be "string".
982-
* Example: "name:string, age:integer". Schemas can also be specified as a
983-
* JSON array of fields, which allows for nested and repeated fields. See
984-
* a [Table resource](http://goo.gl/sl8Dmg) for more detailed information.
1033+
* pairs. Valid types are "string", "integer", "float", "boolean", "bytes",
1034+
* "record", and "timestamp". If the type is omitted, it is assumed to be
1035+
* "string". Example: "name:string, age:integer". Schemas can also be
1036+
* specified as a JSON array of fields, which allows for nested and repeated
1037+
* fields. See a [Table resource](http://goo.gl/sl8Dmg) for more detailed
1038+
* information.
9851039
* @param {function} callback - The callback function.
9861040
* @param {?error} callback.err - An error returned while making this request.
9871041
* @param {object} callback.apiResponse - The full API response.

‎packages/bigquery/system-test/bigquery.js‎

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,55 @@ describe('BigQuery', function() {
3838

3939
var query = 'SELECT url FROM [publicdata:samples.github_nested] LIMIT 100';
4040

41+
var SCHEMA = [
42+
{
43+
name: 'id',
44+
type: 'INTEGER'
45+
},
46+
{
47+
name: 'breed',
48+
type: 'STRING'
49+
},
50+
{
51+
name: 'name',
52+
type: 'STRING'
53+
},
54+
{
55+
name: 'dob',
56+
type: 'TIMESTAMP'
57+
},
58+
{
59+
name: 'around',
60+
type: 'BOOLEAN'
61+
},
62+
{
63+
name: 'buffer',
64+
type: 'BYTES'
65+
},
66+
{
67+
name: 'arrayOfInts',
68+
type: 'INTEGER',
69+
mode: 'REPEATED'
70+
},
71+
{
72+
name: 'recordOfRecords',
73+
type: 'RECORD',
74+
fields: [
75+
{
76+
name: 'records',
77+
type: 'RECORD',
78+
mode: 'REPEATED',
79+
fields: [
80+
{
81+
name: 'record',
82+
type: 'BOOLEAN'
83+
}
84+
]
85+
}
86+
]
87+
}
88+
];
89+
4190
before(function(done) {
4291
async.series([
4392
// Remove buckets created for the tests.
@@ -51,7 +100,7 @@ describe('BigQuery', function() {
51100

52101
// Create the test table.
53102
table.create.bind(table, {
54-
schema: 'id:integer,breed,name,dob:timestamp,around:boolean'
103+
schema: SCHEMA
55104
}),
56105

57106
// Create a Bucket.
@@ -312,15 +361,7 @@ describe('BigQuery', function() {
312361
var TEST_DATA_JSON_PATH = require.resolve('./data/kitten-test-data.json');
313362

314363
it('should have created the correct schema', function() {
315-
assert.deepEqual(table.metadata.schema, {
316-
fields: [
317-
{ name: 'id', type: 'INTEGER' },
318-
{ name: 'breed', type: 'STRING' },
319-
{ name: 'name', type: 'STRING' },
320-
{ name: 'dob', type: 'TIMESTAMP' },
321-
{ name: 'around', type: 'BOOLEAN' }
322-
]
323-
});
364+
assert.deepEqual(table.metadata.schema.fields, SCHEMA);
324365
});
325366

326367
it('should get the rows in a table', function(done) {
@@ -388,14 +429,21 @@ describe('BigQuery', function() {
388429
});
389430

390431
it('should convert values to their schema types', function(done) {
391-
var now = new Date();
392-
393432
var data = {
394433
name: 'dave',
395434
breed: 'british shorthair',
396435
id: 99,
397-
dob: now.toJSON(),
398-
around: true
436+
dob: new Date(),
437+
around: true,
438+
buffer: new Buffer('test'),
439+
arrayOfInts: [1, 3, 5],
440+
recordOfRecords: {
441+
records: [
442+
{
443+
record: true
444+
}
445+
]
446+
}
399447
};
400448

401449
table.insert(data, function(err, insertErrors) {
@@ -409,7 +457,11 @@ describe('BigQuery', function() {
409457
function query(callback) {
410458
var row;
411459

412-
table.query('SELECT * FROM ' + table.id + ' WHERE id = ' + data.id)
460+
table
461+
.query({
462+
query: 'SELECT * FROM ' + table.id + ' WHERE id = ' + data.id,
463+
useLegacySql: false
464+
})
413465
.on('error', callback)
414466
.once('data', function(row_) { row = row_; })
415467
.on('end', function() {
@@ -418,10 +470,7 @@ describe('BigQuery', function() {
418470
return;
419471
}
420472

421-
assert.strictEqual(row.name, data.name);
422-
assert.strictEqual(row.breed, data.breed);
423-
assert.strictEqual(row.id, data.id);
424-
assert.deepEqual(row.dob, now);
473+
assert.deepEqual(row, data);
425474
callback();
426475
});
427476
}

‎packages/bigquery/system-test/data/kitten-test-data.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
{ "name": "silvano", "breed": "the cat kind", "id": 1, "dob": 1414634759011, "around": false }
2-
{ "name": "ryan", "breed": "golden retriever?", "id": 2, "dob": 1414634759012, "around": false }
3-
{ "name": "stephen", "breed": "idkanycatbreeds", "id": 3, "dob": 1414634759013, "around": true }
1+
{ "name": "silvano", "breed": "the cat kind", "id": 1, "dob": 1414634759011, "around": false, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
2+
{ "name": "ryan", "breed": "golden retriever?", "id": 2, "dob": 1414634759012, "around": false, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
3+
{ "name": "stephen", "breed": "idkanycatbreeds", "id": 3, "dob": 1414634759013, "around": true, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }

0 commit comments

Comments
 (0)