Skip to content

Commit b6270e5

Browse files
stephenpluspluscallmehiphop
stephenplusplus
authored and
callmehiphop
committed
treat partial errors as callback(err)ors (#1760)
1 parent 525fb0f commit b6270e5

12 files changed

Lines changed: 364 additions & 231 deletions

File tree

‎packages/bigquery/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"bigquery"
5151
],
5252
"dependencies": {
53-
"@google-cloud/common": "^0.7.0",
53+
"@google-cloud/common": "^0.8.0",
5454
"arrify": "^1.0.0",
5555
"duplexify": "^3.2.0",
5656
"extend": "^3.0.0",

‎packages/bigquery/src/table.js‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -945,6 +945,7 @@ Table.prototype.import = function(source, metadata, callback) {
945945
* {module:bigquery/table#import} instead.
946946
*
947947
* @resource [Tabledata: insertAll API Documentation]{@link https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll}
948+
* @resource [Troubleshooting Errors]{@link https://developers.google.com/bigquery/troubleshooting-errors}
948949
*
949950
* @param {object|object[]} rows - The rows to insert into the table.
950951
* @param {object=} options - Configuration object.
@@ -964,7 +965,9 @@ Table.prototype.import = function(source, metadata, callback) {
964965
* for considerations when working with templates tables.
965966
* @param {function} callback - The callback function.
966967
* @param {?error} callback.err - An error returned while making this request.
967-
* @param {array} callback.insertErrors - A list of errors for insert failures.
968+
* @param {object[]} callback.err.errors - If present, these represent partial
969+
* failures. It's possible for part of your request to be completed
970+
* successfully, while the other part was not.
968971
* @param {object} callback.apiResponse - The full API response.
969972
*
970973
* @example
@@ -1011,22 +1014,22 @@ Table.prototype.import = function(source, metadata, callback) {
10111014
* table.insert(row, options, insertHandler);
10121015
*
10131016
* //-
1014-
* // Handling the response.
1017+
* // Handling the response. See <a href="https://developers.google.com/bigquery/troubleshooting-errors">
1018+
* // Troubleshooting Errors</a> for best practices on how to handle errors.
10151019
* //-
1016-
* function insertHandler(err, insertErrors, apiResponse) {
1017-
* // err (object):
1018-
* // An API error occurred.
1019-
*
1020-
* // insertErrors (object[]):
1021-
* // If populated, some rows failed to insert, while others may have
1022-
* // succeeded.
1023-
* //
1024-
* // insertErrors[].row (original individual row object passed to `insert`)
1025-
* // insertErrors[].errors[].reason
1026-
* // insertErrors[].errors[].message
1027-
*
1028-
* // See https://developers.google.com/bigquery/troubleshooting-errors for
1029-
* // recommendations on handling errors.
1020+
* function insertHandler(err, apiResponse) {
1021+
* if (err) {
1022+
* // An API error or partial failure occurred.
1023+
*
1024+
* if (err.name === 'PartialFailureError') {
1025+
* // Some rows failed to insert, while others may have succeeded.
1026+
*
1027+
* // err.errors (object[]):
1028+
* // err.errors[].row (original row object passed to `insert`)
1029+
* // err.errors[].errors[].reason
1030+
* // err.errors[].errors[].message
1031+
* }
1032+
* }
10301033
* }
10311034
*
10321035
* //-
@@ -1063,23 +1066,30 @@ Table.prototype.insert = function(rows, options, callback) {
10631066
json: json
10641067
}, function(err, resp) {
10651068
if (err) {
1066-
callback(err, null, resp);
1069+
callback(err, resp);
10671070
return;
10681071
}
10691072

1070-
var failedToInsert = (resp.insertErrors || []).map(function(insertError) {
1073+
var partialFailures = (resp.insertErrors || []).map(function(insertError) {
10711074
return {
10721075
errors: insertError.errors.map(function(error) {
10731076
return {
10741077
message: error.message,
10751078
reason: error.reason
10761079
};
10771080
}),
1078-
row: json.rows[insertError.index].json
1081+
row: rows[insertError.index]
10791082
};
10801083
});
10811084

1082-
callback(null, failedToInsert, resp);
1085+
if (partialFailures.length > 0) {
1086+
err = new common.util.PartialFailureError({
1087+
errors: partialFailures,
1088+
response: resp
1089+
});
1090+
}
1091+
1092+
callback(err, resp);
10831093
});
10841094
};
10851095

‎packages/bigquery/system-test/bigquery.js‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@ describe('BigQuery', function() {
485485
it('should convert values to their schema types', function(done) {
486486
var data = {
487487
name: 'dave',
488-
breed: 'british shorthair',
488+
breed: 'husky',
489489
id: 99,
490490
dob: new Date(),
491491
around: true,
@@ -500,14 +500,9 @@ describe('BigQuery', function() {
500500
}
501501
};
502502

503-
table.insert(data, function(err, insertErrors) {
503+
table.insert(data, function(err) {
504504
assert.ifError(err);
505505

506-
if (insertErrors.length > 0) {
507-
done(insertErrors[0].errors[0]);
508-
return;
509-
}
510-
511506
function query(callback) {
512507
var query = {
513508
query: 'SELECT * FROM ' + table.id + ' WHERE id = ' + data.id,
@@ -533,6 +528,55 @@ describe('BigQuery', function() {
533528
});
534529
});
535530

531+
it('should return partial errors', function(done) {
532+
var data = {
533+
name: 'dave',
534+
breed: 'british husky',
535+
id: 99,
536+
dob: new Date(),
537+
around: true,
538+
buffer: new Buffer('test'),
539+
arrayOfInts: [1, 3, 5],
540+
recordOfRecords: {
541+
records: [
542+
{
543+
record: true
544+
}
545+
]
546+
}
547+
};
548+
549+
var improperData = {
550+
name: 11
551+
};
552+
553+
table.insert([data, improperData], function(err) {
554+
assert.strictEqual(err.name, 'PartialFailureError');
555+
556+
assert.deepEqual(err.errors[0], {
557+
errors: [
558+
{
559+
message: 'Conversion from int64 to string is unsupported.',
560+
reason: 'invalid'
561+
}
562+
],
563+
row: improperData
564+
});
565+
566+
assert.deepEqual(err.errors[1], {
567+
errors: [
568+
{
569+
message: undefined,
570+
reason: 'stopped'
571+
}
572+
],
573+
row: data
574+
});
575+
576+
done();
577+
});
578+
});
579+
536580
it('should export data to a file in your bucket', function(done) {
537581
var file = bucket.file('kitten-test-data-backup.json');
538582

‎packages/bigquery/test/table.js‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1221,9 +1221,8 @@ describe('BigQuery/Table', function() {
12211221
callback(null, apiResponse);
12221222
};
12231223

1224-
table.insert(data, function(err, insertErrors, apiResponse_) {
1224+
table.insert(data, function(err, apiResponse_) {
12251225
assert.ifError(err);
1226-
assert.deepEqual(insertErrors, []);
12271226
assert.strictEqual(apiResponse_, apiResponse);
12281227
done();
12291228
});
@@ -1237,15 +1236,14 @@ describe('BigQuery/Table', function() {
12371236
callback(error, apiResponse);
12381237
};
12391238

1240-
table.insert(data, function(err, insertErrors, apiResponse_) {
1239+
table.insert(data, function(err, apiResponse_) {
12411240
assert.strictEqual(err, error);
1242-
assert.strictEqual(insertErrors, null);
12431241
assert.strictEqual(apiResponse_, apiResponse);
12441242
done();
12451243
});
12461244
});
12471245

1248-
it('should return insert failures to the callback', function(done) {
1246+
it('should return partial failures', function(done) {
12491247
var row0Error = { message: 'Error.', reason: 'notFound' };
12501248
var row1Error = { message: 'Error.', reason: 'notFound' };
12511249

@@ -1258,12 +1256,18 @@ describe('BigQuery/Table', function() {
12581256
});
12591257
};
12601258

1261-
table.insert(data, function(err, insertErrors) {
1262-
assert.ifError(err);
1259+
table.insert(data, function(err) {
1260+
assert.strictEqual(err.name, 'PartialFailureError');
12631261

1264-
assert.deepEqual(insertErrors, [
1265-
{ row: dataApiFormat.rows[0].json, errors: [row0Error] },
1266-
{ row: dataApiFormat.rows[1].json, errors: [row1Error] }
1262+
assert.deepEqual(err.errors, [
1263+
{
1264+
row: dataApiFormat.rows[0].json,
1265+
errors: [row0Error]
1266+
},
1267+
{
1268+
row: dataApiFormat.rows[1].json,
1269+
errors: [row1Error]
1270+
}
12671271
]);
12681272

12691273
done();

‎packages/bigtable/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"bigtable"
5151
],
5252
"dependencies": {
53-
"@google-cloud/common": "^0.7.0",
53+
"@google-cloud/common": "^0.8.0",
5454
"arrify": "^1.0.0",
5555
"concat-stream": "^1.5.0",
5656
"create-error-class": "^3.0.2",

‎packages/bigtable/src/table.js‎

Lines changed: 31 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -683,30 +683,21 @@ Table.prototype.getRows = function(options, callback) {
683683
* See {module:bigtable/table#mutate}.
684684
* @param {function} callback - The callback function.
685685
* @param {?error} callback.err - An error returned while making this request.
686-
* @param {object[]} callback.insertErrors - A status object for each failed
687-
* insert.
686+
* @param {object[]} callback.err.errors - If present, these represent partial
687+
* failures. It's possible for part of your request to be completed
688+
* successfully, while the other part was not.
688689
*
689690
* @example
690-
* var callback = function(err, insertErrors) {
691+
* var callback = function(err) {
691692
* if (err) {
692-
* // Error handling omitted.
693-
* }
693+
* // An API error or partial failure occurred.
694694
*
695-
* // insertErrors = [
696-
* // {
697-
* // code: 500,
698-
* // message: 'Internal Server Error',
699-
* // entry: {
700-
* // key: 'gwashington',
701-
* // data: {
702-
* // follows: {
703-
* // jadams: 1
704-
* // }
705-
* // }
706-
* // }
707-
* // },
708-
* // ...
709-
* // ]
695+
* if (err.name === 'PartialFailureError') {
696+
* // err.errors[].code = 'Response code'
697+
* // err.errors[].message = 'Error message'
698+
* // err.errors[].entry = The original entry
699+
* }
700+
* }
710701
* };
711702
*
712703
* var entries = [
@@ -769,34 +760,24 @@ Table.prototype.insert = function(entries, callback) {
769760
* deleted.
770761
* @param {function} callback - The callback function.
771762
* @param {?error} callback.err - An error returned while making this request.
772-
* @param {object[]} callback.mutationErrors - A status object for each failed
773-
* mutation.
763+
* @param {object[]} callback.err.errors - If present, these represent partial
764+
* failures. It's possible for part of your request to be completed
765+
* successfully, while the other part was not.
774766
*
775767
* @example
776768
* //-
777769
* // Insert entities. See {module:bigtable/table#insert}
778770
* //-
779-
* var callback = function(err, mutationErrors) {
771+
* var callback = function(err) {
780772
* if (err) {
781-
* // Error handling omitted.
782-
* }
773+
* // An API error or partial failure occurred.
783774
*
784-
* // mutationErrors = [
785-
* // {
786-
* // code: 500,
787-
* // message: 'Internal Server Error',
788-
* // entry: {
789-
* // method: 'insert',
790-
* // key: 'gwashington',
791-
* // data: {
792-
* // follows: {
793-
* // jadams: 1
794-
* // }
795-
* // }
796-
* // }
797-
* // },
798-
* // ...
799-
* // ]
775+
* if (err.name === 'PartialFailureError') {
776+
* // err.errors[].code = 'Response code'
777+
* // err.errors[].message = 'Error message'
778+
* // err.errors[].entry = The original entry
779+
* }
780+
* }
800781
* };
801782
*
802783
* var entries = [
@@ -910,7 +891,6 @@ Table.prototype.mutate = function(entries, callback) {
910891
var status = common.GrpcService.decorateStatus_(entry.status);
911892
status.entry = entries[entry.index];
912893

913-
914894
if (!isCallbackMode) {
915895
emitter.emit('error', status);
916896
return;
@@ -932,7 +912,15 @@ Table.prototype.mutate = function(entries, callback) {
932912
stream
933913
.on('error', callback)
934914
.pipe(concat(function(mutationErrors) {
935-
callback(null, mutationErrors);
915+
var err = null;
916+
917+
if (mutationErrors && mutationErrors.length > 0) {
918+
err = new common.util.PartialFailureError({
919+
errors: mutationErrors
920+
});
921+
}
922+
923+
callback(err);
936924
}));
937925
};
938926

‎packages/bigtable/system-test/bigtable.js‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -405,11 +405,7 @@ describe('Bigtable', function() {
405405
}
406406
}];
407407

408-
TABLE.insert(rows, function(err, insertErrors) {
409-
assert.ifError(err);
410-
assert.strictEqual(insertErrors.length, 0);
411-
done();
412-
});
408+
TABLE.insert(rows, done);
413409
});
414410

415411
it('should create an individual row', function(done) {

0 commit comments

Comments
 (0)