rowIndexToErrorMessage, boolean hasDataUnknownError) {
+ this.rowIndexToErrorMessage = rowIndexToErrorMessage;
+ this.hasDataUnknownError = hasDataUnknownError;
+ }
+
+ // This message should not be exposed to the user directly.
+ // Please examine individual row's error through `rowIndexToErrorMessage`.
+ public String getMessage() {
+ return "The map of row index to error message is " + rowIndexToErrorMessage.toString();
+ }
+
+ public boolean hasDataUnknownError() {
+ return hasDataUnknownError;
+ }
+ }
+
/** This exception is used internally to handle field level parsing errors. */
public static class FieldParseError extends IllegalArgumentException {
private final String fieldName;
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
index 3d1e1e0b5d..5ac0a34ae8 100644
--- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
+++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
@@ -16,6 +16,7 @@
package com.google.cloud.bigquery.storage.v1;
import com.google.api.pathtemplate.ValidationException;
+import com.google.cloud.bigquery.storage.v1.Exceptions.RowIndexToErrorException;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.Doubles;
@@ -29,7 +30,10 @@
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -121,7 +125,10 @@ public static DynamicMessage convertJsonToProtoMessage(
}
/**
- * Converts input message to Protobuf
+ * Converts input message to Protobuf.
+ *
+ * WARNING: it's much more efficient to call the other APIs accepting json array if the jsons
+ * share the same table schema.
*
* @param protoSchema the schema of the output Protobuf schems.
* @param tableSchema tha underlying table schema for which Protobuf is being built.
@@ -130,15 +137,37 @@ public static DynamicMessage convertJsonToProtoMessage(
* schema should be accepted.
* @return Converted message in Protobuf format.
*/
- @Override
public DynamicMessage convertToProtoMessage(
Descriptor protoSchema, TableSchema tableSchema, Object json, boolean ignoreUnknownFields) {
return convertToProtoMessage(protoSchema, tableSchema, (JSONObject) json, ignoreUnknownFields);
}
+ /**
+ * Converts Json array to list of Protobuf
+ *
+ * @param protoSchema the schema of the output Protobuf schems.
+ * @param tableSchema tha underlying table schema for which Protobuf is being built.
+ * @param jsonArray the input JSON array converted to Protobuf.
+ * @param ignoreUnknownFields flag indicating that the additional fields not present in the output
+ * schema should be accepted.
+ * @return Converted message in Protobuf format.
+ */
+ @Override
+ public List convertToProtoMessage(
+ Descriptor protoSchema,
+ TableSchema tableSchema,
+ Iterable jsonArray,
+ boolean ignoreUnknownFields) {
+ return convertToProtoMessage(
+ protoSchema, tableSchema, (JSONArray) jsonArray, ignoreUnknownFields);
+ }
+
/**
* Converts Json data to protocol buffer messages given the protocol buffer descriptor.
*
+ * WARNING: it's much more efficient to call the other APIs accepting json array if the jsons
+ * share the same table schema.
+ *
* @param protoSchema
* @param json
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
@@ -155,6 +184,9 @@ public DynamicMessage convertToProtoMessage(Descriptor protoSchema, JSONObject j
/**
* Converts Json data to protocol buffer messages given the protocol buffer descriptor.
*
+ *
WARNING: it's much more efficient to call the other APIs accepting json array if the jsons
+ * share the same table schema.
+ *
* @param protoSchema
* @param tableSchema bigquery table schema is needed for type conversion of DATETIME, TIME,
* NUMERIC, BIGNUMERIC
@@ -175,6 +207,9 @@ public DynamicMessage convertToProtoMessage(
/**
* Converts Json data to protocol buffer messages given the protocol buffer descriptor.
*
+ *
WARNING: it's much more efficient to call the other APIs accepting json array if the jsons
+ * share the same table schema.
+ *
* @param protoSchema
* @param tableSchema bigquery table schema is needed for type conversion of DATETIME, TIME,
* NUMERIC, BIGNUMERIC
@@ -189,11 +224,48 @@ public DynamicMessage convertToProtoMessage(
Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null.");
Preconditions.checkNotNull(tableSchema, "TableSchema is null.");
Preconditions.checkState(json.length() != 0, "JSONObject is empty.");
-
return convertToProtoMessage(
protoSchema, tableSchema.getFieldsList(), json, "root", ignoreUnknownFields);
}
+ /**
+ * Converts Json array to list of protocol buffer messages given the protocol buffer descriptor.
+ *
+ * @param protoSchema
+ * @param tableSchema bigquery table schema is needed for type conversion of DATETIME, TIME,
+ * NUMERIC, BIGNUMERIC
+ * @param jsonArray
+ * @param ignoreUnknownFields allows unknown fields in JSON input to be ignored.
+ * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
+ */
+ public List convertToProtoMessage(
+ Descriptor protoSchema,
+ TableSchema tableSchema,
+ JSONArray jsonArray,
+ boolean ignoreUnknownFields)
+ throws IllegalArgumentException {
+ Preconditions.checkNotNull(jsonArray, "jsonArray is null.");
+ Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null.");
+ Preconditions.checkNotNull(tableSchema, "tableSchema is null.");
+ Preconditions.checkState(jsonArray.length() != 0, "jsonArray is empty.");
+
+ return convertToProtoMessage(
+ protoSchema, tableSchema.getFieldsList(), jsonArray, "root", ignoreUnknownFields);
+ }
+
+ private DynamicMessage convertToProtoMessage(
+ Descriptor protoSchema,
+ List tableSchema,
+ JSONObject jsonObject,
+ String jsonScope,
+ boolean ignoreUnknownFields) {
+ JSONArray jsonArray = new JSONArray();
+ jsonArray.put(jsonObject);
+ return convertToProtoMessage(
+ protoSchema, tableSchema, jsonArray, jsonScope, ignoreUnknownFields)
+ .get(0);
+ }
+
/**
* Converts Json data to protocol buffer messages given the protocol buffer descriptor.
*
@@ -202,84 +274,162 @@ public DynamicMessage convertToProtoMessage(
* @param jsonScope Debugging purposes
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
*/
- private DynamicMessage convertToProtoMessage(
+ private List convertToProtoMessage(
Descriptor protoSchema,
List tableSchema,
- JSONObject json,
+ JSONArray jsonArray,
String jsonScope,
boolean ignoreUnknownFields)
- throws IllegalArgumentException {
-
- DynamicMessage.Builder protoMsg = DynamicMessage.newBuilder(protoSchema);
- String[] jsonNames = JSONObject.getNames(json);
- if (jsonNames == null) {
- return protoMsg.build();
- }
- for (String jsonName : jsonNames) {
- // We want lowercase here to support case-insensitive data writes.
- // The protobuf descriptor that is used is assumed to have all lowercased fields
- String jsonFieldLocator = jsonName.toLowerCase();
+ throws RowIndexToErrorException {
+ List messageList = new ArrayList<>();
+ Map jsonNameToMetadata = new HashMap<>();
+ Map rowIndexToErrorMessage = new HashMap<>();
- // If jsonName is not compatible with proto naming convention, we should look by its
- // placeholder name.
- if (!BigQuerySchemaUtil.isProtoCompatible(jsonFieldLocator)) {
- jsonFieldLocator = BigQuerySchemaUtil.generatePlaceholderFieldName(jsonFieldLocator);
- }
- String currentScope = jsonScope + "." + jsonName;
- FieldDescriptor field = protoSchema.findFieldByName(jsonFieldLocator);
- if (field == null && !ignoreUnknownFields) {
- throw new Exceptions.DataHasUnknownFieldException(currentScope);
- } else if (field == null) {
- continue;
- }
- TableFieldSchema fieldSchema = null;
- if (tableSchema != null) {
- // protoSchema is generated from tableSchema so their field ordering should match.
- fieldSchema = tableSchema.get(field.getIndex());
- if (!fieldSchema.getName().toLowerCase().equals(BigQuerySchemaUtil.getFieldName(field))) {
- throw new ValidationException(
- "Field at index "
- + field.getIndex()
- + " has mismatch names ("
- + fieldSchema.getName()
- + ") ("
- + field.getName()
- + ")");
- }
- }
+ boolean hasDataUnknownError = false;
+ for (int i = 0; i < jsonArray.length(); i++) {
try {
- if (!field.isRepeated()) {
- fillField(
- protoMsg, field, fieldSchema, json, jsonName, currentScope, ignoreUnknownFields);
+ DynamicMessage.Builder protoMsg = DynamicMessage.newBuilder(protoSchema);
+ JSONObject jsonObject = jsonArray.getJSONObject(i);
+ String[] jsonNames = JSONObject.getNames(jsonObject);
+ if (jsonNames == null) {
+ messageList.add(protoMsg.build());
+ continue;
+ }
+ for (String jsonName : jsonNames) {
+ String currentScope = jsonScope + "." + jsonName;
+ FieldDescriptorAndFieldTableSchema fieldDescriptorAndFieldTableSchema =
+ jsonNameToMetadata.computeIfAbsent(
+ currentScope,
+ k -> {
+ return computeDescriptorAndSchema(
+ currentScope, ignoreUnknownFields, jsonName, protoSchema, tableSchema);
+ });
+ if (fieldDescriptorAndFieldTableSchema == null) {
+ continue;
+ }
+ FieldDescriptor field = fieldDescriptorAndFieldTableSchema.fieldDescriptor;
+ TableFieldSchema tableFieldSchema = fieldDescriptorAndFieldTableSchema.tableFieldSchema;
+ try {
+ if (!field.isRepeated()) {
+ fillField(
+ protoMsg,
+ field,
+ tableFieldSchema,
+ jsonObject,
+ jsonName,
+ currentScope,
+ ignoreUnknownFields);
+ } else {
+ fillRepeatedField(
+ protoMsg,
+ field,
+ tableFieldSchema,
+ jsonObject,
+ jsonName,
+ currentScope,
+ ignoreUnknownFields);
+ }
+ } catch (Exceptions.FieldParseError ex) {
+ throw ex;
+ } catch (Exception ex) {
+ // This function is recursively called, so this throw will be caught and throw directly
+ // out by the catch above.
+ throw new Exceptions.FieldParseError(
+ currentScope,
+ tableFieldSchema != null
+ ? tableFieldSchema.getType().name()
+ : field.getType().name(),
+ ex);
+ }
+ }
+ DynamicMessage msg;
+ try {
+ msg = protoMsg.build();
+ } catch (UninitializedMessageException e) {
+ String errorMsg = e.getMessage();
+ int idxOfColon = errorMsg.indexOf(":");
+ String missingFieldName = errorMsg.substring(idxOfColon + 2);
+ throw new IllegalArgumentException(
+ String.format(
+ "JSONObject does not have the required field %s.%s.",
+ jsonScope, missingFieldName));
+ }
+ messageList.add(msg);
+ } catch (IllegalArgumentException exception) {
+ if (exception instanceof Exceptions.DataHasUnknownFieldException) {
+ hasDataUnknownError = true;
+ }
+ if (exception instanceof Exceptions.FieldParseError) {
+ Exceptions.FieldParseError ex = (Exceptions.FieldParseError) exception;
+ rowIndexToErrorMessage.put(
+ i,
+ "Field "
+ + ex.getFieldName()
+ + " failed to convert to "
+ + ex.getBqType()
+ + ". Error: "
+ + ex.getCause().getMessage());
} else {
- fillRepeatedField(
- protoMsg, field, fieldSchema, json, jsonName, currentScope, ignoreUnknownFields);
+ rowIndexToErrorMessage.put(i, exception.getMessage());
}
- } catch (Exceptions.FieldParseError ex) {
- throw ex;
- } catch (Exception ex) {
- // This function is recursively called, so this throw will be caught and throw directly out
- // by the catch
- // above.
- throw new Exceptions.FieldParseError(
- currentScope,
- fieldSchema != null ? fieldSchema.getType().name() : field.getType().name(),
- ex);
}
}
+ if (!rowIndexToErrorMessage.isEmpty()) {
+ throw new RowIndexToErrorException(rowIndexToErrorMessage, hasDataUnknownError);
+ }
+ return messageList;
+ }
- DynamicMessage msg;
- try {
- msg = protoMsg.build();
- } catch (UninitializedMessageException e) {
- String errorMsg = e.getMessage();
- int idxOfColon = errorMsg.indexOf(":");
- String missingFieldName = errorMsg.substring(idxOfColon + 2);
- throw new IllegalArgumentException(
- String.format(
- "JSONObject does not have the required field %s.%s.", jsonScope, missingFieldName));
+ private static final class FieldDescriptorAndFieldTableSchema {
+ TableFieldSchema tableFieldSchema;
+
+ // Field descriptor
+ FieldDescriptor fieldDescriptor;
+ }
+
+ private FieldDescriptorAndFieldTableSchema computeDescriptorAndSchema(
+ String currentScope,
+ boolean ignoreUnknownFields,
+ String jsonName,
+ Descriptor protoSchema,
+ List tableFieldSchemaList) {
+
+ // We want lowercase here to support case-insensitive data writes.
+ // The protobuf descriptor that is used is assumed to have all lowercased fields
+ String jsonFieldLocator = jsonName.toLowerCase();
+
+ // If jsonName is not compatible with proto naming convention, we should look by its
+ // placeholder name.
+ if (!BigQuerySchemaUtil.isProtoCompatible(jsonFieldLocator)) {
+ jsonFieldLocator = BigQuerySchemaUtil.generatePlaceholderFieldName(jsonFieldLocator);
}
- return msg;
+
+ FieldDescriptor field = protoSchema.findFieldByName(jsonFieldLocator);
+ if (field == null && !ignoreUnknownFields) {
+ throw new Exceptions.DataHasUnknownFieldException(currentScope);
+ } else if (field == null) {
+ return null;
+ }
+ TableFieldSchema fieldSchema = null;
+ if (tableFieldSchemaList != null) {
+ // protoSchema is generated from tableSchema so their field ordering should match.
+ fieldSchema = tableFieldSchemaList.get(field.getIndex());
+ if (!fieldSchema.getName().toLowerCase().equals(BigQuerySchemaUtil.getFieldName(field))) {
+ throw new ValidationException(
+ "Field at index "
+ + field.getIndex()
+ + " has mismatch names ("
+ + fieldSchema.getName()
+ + ") ("
+ + field.getName()
+ + ")");
+ }
+ }
+ FieldDescriptorAndFieldTableSchema fieldDescriptorAndFieldTableSchema =
+ new FieldDescriptorAndFieldTableSchema();
+ fieldDescriptorAndFieldTableSchema.fieldDescriptor = field;
+ fieldDescriptorAndFieldTableSchema.tableFieldSchema = fieldSchema;
+ return fieldDescriptorAndFieldTableSchema;
}
/**
@@ -302,7 +452,6 @@ private void fillField(
String currentScope,
boolean ignoreUnknownFields)
throws IllegalArgumentException {
-
java.lang.Object val = json.get(exactJsonKeyName);
if (val == JSONObject.NULL) {
return;
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SchemaAwareStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SchemaAwareStreamWriter.java
index 10fceeee68..2c5a79af64 100644
--- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SchemaAwareStreamWriter.java
+++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SchemaAwareStreamWriter.java
@@ -21,13 +21,15 @@
import com.google.api.gax.core.ExecutorProvider;
import com.google.api.gax.rpc.TransportChannelProvider;
import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializationError;
+import com.google.cloud.bigquery.storage.v1.Exceptions.RowIndexToErrorException;
import com.google.common.base.Preconditions;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.DescriptorValidationException;
-import com.google.protobuf.Message;
+import com.google.protobuf.DynamicMessage;
import com.google.rpc.Code;
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.regex.Matcher;
@@ -124,19 +126,23 @@ private void refreshWriter(TableSchema updatedSchema)
this.streamWriter = streamWriterBuilder.setWriterSchema(this.protoSchema).build();
}
- private Message buildMessage(T item)
+ private List buildMessage(Iterable items)
throws InterruptedException, DescriptorValidationException, IOException {
try {
return this.toProtoConverter.convertToProtoMessage(
- this.descriptor, this.tableSchema, item, ignoreUnknownFields);
- } catch (Exceptions.DataHasUnknownFieldException ex) {
+ this.descriptor, this.tableSchema, items, ignoreUnknownFields);
+ } catch (RowIndexToErrorException ex) {
+ // We only retry for data unknown error.
+ if (!ex.hasDataUnknownError) {
+ throw ex;
+ }
// Directly return error when stream writer refresh is disabled.
if (this.skipRefreshStreamWriter) {
throw ex;
}
LOG.warning(
- "Saw unknown field "
- + ex.getFieldName()
+ "Saw unknown field error during proto message conversin within error messages"
+ + ex.rowIndexToErrorMessage
+ ", try to refresh the writer with updated schema, stream: "
+ streamName);
GetWriteStreamRequest writeStreamRequest =
@@ -147,7 +153,7 @@ private Message buildMessage(T item)
WriteStream writeStream = client.getWriteStream(writeStreamRequest);
refreshWriter(writeStream.getTableSchema());
return this.toProtoConverter.convertToProtoMessage(
- this.descriptor, this.tableSchema, item, ignoreUnknownFields);
+ this.descriptor, this.tableSchema, items, ignoreUnknownFields);
}
}
/**
@@ -169,7 +175,6 @@ public ApiFuture append(Iterable items, long offset)
if (!this.skipRefreshStreamWriter && this.streamWriter.getUpdatedSchema() != null) {
refreshWriter(this.streamWriter.getUpdatedSchema());
}
-
ProtoRows.Builder rowsBuilder = ProtoRows.newBuilder();
// Any error in convertToProtoMessage will throw an
// IllegalArgumentException/IllegalStateException/NullPointerException.
@@ -177,29 +182,15 @@ public ApiFuture append(Iterable items, long offset)
// After the conversion is finished an AppendSerializtionError exception that contains all the
// conversion errors will be thrown.
Map rowIndexToErrorMessage = new HashMap<>();
- int i = -1;
- for (T item : items) {
- i += 1;
- try {
- Message protoMessage = buildMessage(item);
- rowsBuilder.addSerializedRows(protoMessage.toByteString());
- } catch (IllegalArgumentException exception) {
- if (exception instanceof Exceptions.FieldParseError) {
- Exceptions.FieldParseError ex = (Exceptions.FieldParseError) exception;
- rowIndexToErrorMessage.put(
- i,
- "Field "
- + ex.getFieldName()
- + " failed to convert to "
- + ex.getBqType()
- + ". Error: "
- + ex.getCause().getMessage());
- } else {
- rowIndexToErrorMessage.put(i, exception.getMessage());
- }
- } catch (InterruptedException ex) {
- throw new RuntimeException(ex);
+ try {
+ List protoMessages = buildMessage(items);
+ for (DynamicMessage dynamicMessage : protoMessages) {
+ rowsBuilder.addSerializedRows(dynamicMessage.toByteString());
}
+ } catch (RowIndexToErrorException exception) {
+ rowIndexToErrorMessage = exception.rowIndexToErrorMessage;
+ } catch (InterruptedException ex) {
+ throw new RuntimeException(ex);
}
if (!rowIndexToErrorMessage.isEmpty()) {
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ToProtoConverter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ToProtoConverter.java
index ca17ed11e7..76ef223e24 100644
--- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ToProtoConverter.java
+++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ToProtoConverter.java
@@ -17,11 +17,12 @@
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
+import java.util.List;
public interface ToProtoConverter {
- DynamicMessage convertToProtoMessage(
+ List convertToProtoMessage(
Descriptors.Descriptor protoSchema,
TableSchema tableSchema,
- T inputObject,
+ Iterable inputObject,
boolean ignoreUnknownFields);
}
diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
index 5c44d014d4..c2fab22c6c 100644
--- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
+++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
@@ -20,6 +20,7 @@
import com.google.cloud.bigquery.storage.test.JsonTest.*;
import com.google.cloud.bigquery.storage.test.SchemaTest.*;
+import com.google.cloud.bigquery.storage.v1.Exceptions.RowIndexToErrorException;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.ByteString;
@@ -29,6 +30,7 @@
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import org.json.JSONArray;
@@ -598,7 +600,7 @@ public void testInt32NotMatchInt64() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(TestInt32.getDescriptor(), json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals("JSONObject does not have a int32 field at root.int.", e.getMessage());
+ assertTrue(e.getMessage().contains("JSONObject does not have a int32 field at root.int."));
}
}
@@ -619,7 +621,8 @@ public void testDateTimeMismatch() throws Exception {
TestDatetime.getDescriptor(), tableSchema, json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals("JSONObject does not have a int64 field at root.datetime.", e.getMessage());
+ assertTrue(
+ e.getMessage().contains("JSONObject does not have a int64 field at root.datetime."));
}
}
@@ -640,7 +643,8 @@ public void testTimeMismatch() throws Exception {
TestTime.getDescriptor(), tableSchema, json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals("JSONObject does not have a int64 field at root.time[0].", e.getMessage());
+ assertTrue(
+ e.getMessage().contains("JSONObject does not have a int64 field at root.time[0]."));
}
}
@@ -811,9 +815,12 @@ public void testAllTypes() throws Exception {
assertEquals(protoMsg, AllTypesToCorrectProto.get(entry.getKey())[success]);
success += 1;
} catch (IllegalArgumentException e) {
- assertEquals(
- "JSONObject does not have a " + entry.getValue() + " field at root.test_field_type.",
- e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "JSONObject does not have a "
+ + entry.getValue()
+ + " field at root.test_field_type."));
}
}
if (entry.getKey() == DoubleType.getDescriptor()) {
@@ -846,12 +853,12 @@ public void testAllRepeatedTypesWithLimits() throws Exception {
LOG.info(e.getMessage());
assertTrue(
e.getMessage()
- .equals(
+ .contains(
"JSONObject does not have a "
+ entry.getValue()
+ " field at root.test_repeated[0].")
|| e.getMessage()
- .equals("Error: root.test_repeated[0] could not be converted to byte[]."));
+ .contains("Error: root.test_repeated[0] could not be converted to byte[]."));
}
}
if (entry.getKey() == RepeatedDouble.getDescriptor()) {
@@ -897,8 +904,9 @@ public void testRequired() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(TestRequired.getDescriptor(), json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals(
- "JSONObject does not have the required field root.required_double.", e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains("JSONObject does not have the required field root.required_double."));
}
}
@@ -929,9 +937,10 @@ public void testStructSimpleFail() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(MessageType.getDescriptor(), json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals(
- "JSONObject does not have a string field at root.test_field_type.test_field_type.",
- e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "JSONObject does not have a string field at root.test_field_type.test_field_type."));
}
}
@@ -1089,8 +1098,9 @@ public void testStructComplexFail() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(ComplexRoot.getDescriptor(), json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals(
- "JSONObject does not have a int64 field at root.complex_lvl1.test_int.", e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains("JSONObject does not have a int64 field at root.complex_lvl1.test_int."));
}
}
@@ -1103,8 +1113,9 @@ public void testRepeatedWithMixedTypes() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(RepeatedDouble.getDescriptor(), json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals(
- "JSONObject does not have a double field at root.test_repeated[2].", e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains("JSONObject does not have a double field at root.test_repeated[2]."));
}
}
@@ -1165,9 +1176,10 @@ public void testNestedRepeatedComplexFail() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(NestedRepeated.getDescriptor(), json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
- assertEquals(
- "JSONObject does not have a string field at root.repeated_string.test_repeated[0].",
- e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "JSONObject does not have a string field at root.repeated_string.test_repeated[0]."));
}
}
@@ -1198,10 +1210,10 @@ public void testAllowUnknownFieldsError() throws Exception {
DynamicMessage protoMsg =
JsonToProtoMessage.INSTANCE.convertToProtoMessage(RepeatedInt64.getDescriptor(), json);
Assert.fail("Should fail");
- } catch (Exceptions.DataHasUnknownFieldException e) {
- assertEquals(
- "The source object has fields unknown to BigQuery: root.string.", e.getMessage());
- assertEquals("root.string", e.getFieldName());
+ } catch (IllegalArgumentException e) {
+ assertTrue(
+ e.getMessage()
+ .contains("The source object has fields unknown to BigQuery: " + "root.string."));
}
}
@@ -1262,9 +1274,10 @@ public void testAllowUnknownFieldsSecondLevel() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(ComplexLvl1.getDescriptor(), json);
Assert.fail("Should fail");
} catch (IllegalArgumentException e) {
- assertEquals(
- "The source object has fields unknown to BigQuery: root.complex_lvl2.no_match.",
- e.getMessage());
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "The source object has fields unknown to BigQuery: root.complex_lvl2.no_match."));
}
}
@@ -1327,9 +1340,9 @@ public void testBadJsonFieldRepeated() throws Exception {
JsonToProtoMessage.INSTANCE.convertToProtoMessage(
RepeatedBytes.getDescriptor(), ts, json);
Assert.fail("Should fail");
- } catch (Exceptions.FieldParseError ex) {
- assertEquals(ex.getBqType(), "NUMERIC");
- assertEquals(ex.getFieldName(), "root.test_repeated");
+ } catch (RowIndexToErrorException ex) {
+ assertTrue(ex.rowIndexToErrorMessage.size() == 1);
+ assertTrue(ex.getMessage().contains("root.test_repeated failed to convert to NUMERIC."));
}
}
@@ -1354,7 +1367,7 @@ public void testBadJsonFieldIntRepeated() throws Exception {
RepeatedInt32.getDescriptor(), ts, json);
Assert.fail("Should fail");
} catch (IllegalArgumentException ex) {
- assertEquals(ex.getMessage(), "Text 'blah' could not be parsed at index 0");
+ assertTrue(ex.getMessage().contains("Text 'blah' could not be parsed at index 0"));
}
}
@@ -1421,6 +1434,51 @@ public void testDoubleAndFloatToNumericConversion() {
assertEquals(expectedProto, protoMsg);
}
+ @Test
+ public void testDoubleAndFloatToNumericConversionWithJsonArray() {
+ TableSchema ts =
+ TableSchema.newBuilder()
+ .addFields(
+ 0,
+ TableFieldSchema.newBuilder()
+ .setName("numeric")
+ .setType(TableFieldSchema.Type.NUMERIC)
+ .build())
+ .build();
+ List protoList = new ArrayList<>();
+ int protoNum = 10;
+ for (int i = 0; i < protoNum; i++) {
+ protoList.add(
+ TestNumeric.newBuilder()
+ .setNumeric(
+ BigDecimalByteStringEncoder.encodeToNumericByteString(
+ new BigDecimal("24.678" + i)))
+ .build());
+ }
+
+ JSONArray doubleJsonArray = new JSONArray();
+ JSONArray floatJsonArray = new JSONArray();
+ for (int i = 0; i < protoNum; i++) {
+ JSONObject doubleJson = new JSONObject();
+ doubleJson.put("numeric", new Double(24.678 + (i * 0.0001)));
+ doubleJsonArray.put(doubleJson);
+
+ JSONObject floatJson = new JSONObject();
+ floatJson.put("numeric", new Float(24.678 + (i * 0.0001)));
+ floatJsonArray.put(floatJson);
+ }
+
+ List protoMsgList =
+ JsonToProtoMessage.INSTANCE.convertToProtoMessage(
+ TestNumeric.getDescriptor(), ts, doubleJsonArray, false);
+ assertEquals(protoList, protoMsgList);
+
+ protoMsgList =
+ JsonToProtoMessage.INSTANCE.convertToProtoMessage(
+ TestNumeric.getDescriptor(), ts, floatJsonArray, false);
+ assertEquals(protoList, protoMsgList);
+ }
+
@Test
public void testBigDecimalToBigNumericConversion() {
TableSchema ts =