Skip to content

Commit d04c917

Browse files
maximevwmaximevw
committed
Add some tests for COPY FROM command
1 parent a50218d commit d04c917

12 files changed

+178
-32
lines changed

Diff for: ‎src/main/java/com/ing/data/cassandra/jdbc/commands/AbstractCopyCommandExecutor.java

+5
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,11 @@ void configureFormatters() {
109109
this.decimalFormat.setDecimalFormatSymbols(decimalSymbols);
110110
}
111111

112+
String getOptionValueAsString(final String optionName, final String defaultValue) {
113+
final String optionValue = this.options.getProperty(optionName);
114+
return StringUtils.defaultIfBlank(optionValue, defaultValue);
115+
}
116+
112117
char getOptionValueAsChar(final String optionName, final char defaultValue) {
113118
final String optionValue = this.options.getProperty(optionName);
114119
if (StringUtils.isNotEmpty(optionValue)) {

Diff for: ‎src/main/java/com/ing/data/cassandra/jdbc/commands/CopyFromCommandExecutor.java

+12-6
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
* <li>{@code ESCAPE}: the character that is used to escape the literal uses of the {@code QUOTE} character.
7676
* Defaults to {@value #DEFAULT_ESCAPE_CHAR}.</li>
7777
* <li>{@code HEADER}: whether the first line in the CSV input file will contain the column names.
78-
* If {@code false}, the option {@code SKIPCOLS} is ignored. Defaults to {@code false}.</li>
78+
* Defaults to {@code false}.</li>
7979
* <li>{@code MAXROWS}: the maximum number of rows to import, negative meaning unlimited.
8080
* Defaults to {@value #DEFAULT_MAX_ROWS}.</li>
8181
* <li>{@code NULLVAL}: the string placeholder for null values. Defaults to {@value #DEFAULT_NULL_FORMAT}.</li>
@@ -223,7 +223,7 @@ public ResultSet execute(final CassandraStatement statement, final String cql) t
223223
for (int i = startRow; i < maxRows; i++) {
224224
handleValuesToInsert(insertStatement, allRows.get(i), csvColumns, getColumnsToImport(skippedColumns));
225225
final int rowsInBatch = i - startRow + 1;
226-
if (rowsInBatch % batchSize == 0 || rowsInBatch == maxRows) {
226+
if (rowsInBatch % batchSize == 0 || rowsInBatch == maxRows - startRow) {
227227
insertStatement.executeBatch();
228228
}
229229
}
@@ -287,7 +287,7 @@ private List<String> getColumnsToImport(final Set<String> skippedColumns) {
287287
}
288288

289289
private Map<Integer, String> mapColumnsFromArray(final String[] columnsNames) {
290-
final Map<Integer, String> csvColumns = new HashMap<>();
290+
final Map<Integer, String> csvColumns = new LinkedHashMap<>();
291291
for (int i = 0; i < columnsNames.length; i++) {
292292
csvColumns.put(i, columnsNames[i]);
293293
}
@@ -307,7 +307,10 @@ private void handleValuesToInsert(final Statement statement,
307307
values.add(o);
308308
}
309309
}
310-
final String cql = String.format("INSERT INTO %s(%s) VALUES (%s)", this.tableName, this.columns,
310+
final String cql = String.format("INSERT INTO %s(%s) VALUES (%s)", this.tableName,
311+
csvColumns.values().stream()
312+
.filter(columnsToImport::contains)
313+
.collect(Collectors.joining(COMMA)),
311314
String.join(COMMA, values));
312315
try {
313316
statement.addBatch(cql);
@@ -317,8 +320,11 @@ private void handleValuesToInsert(final Statement statement,
317320
}
318321

319322
private String handleNumberValue(final String strValue) {
323+
if (strValue == null) {
324+
return null;
325+
}
320326
try {
321-
final Number number = this.decimalFormat.parse(strValue);
327+
final Number number = this.decimalFormat.parse(strValue.trim());
322328
return number.toString();
323329
} catch (final ParseException e) {
324330
LOG.warn("Failed to parse and convert value: {}, the value will be ignored.", strValue);
@@ -327,7 +333,7 @@ private String handleNumberValue(final String strValue) {
327333
}
328334

329335
private String parseValue(final String value, final Integer colType) {
330-
if (DEFAULT_NULL_FORMAT.equals(value)) {
336+
if (getOptionValueAsString(OPTION_NULLVAL, DEFAULT_NULL_FORMAT).equals(value)) {
331337
return null;
332338
}
333339
switch (colType) {

Diff for: ‎src/main/java/com/ing/data/cassandra/jdbc/commands/CopyToCommandExecutor.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -189,11 +189,12 @@ public ResultSet execute(final CassandraStatement statement, final String cql) t
189189
}
190190

191191
private ResultSetHelperService configureResultSetHelperService() {
192-
final ResultSetHelperService rsHelperService = new EnhancedResultSetHelperService();
192+
final EnhancedResultSetHelperService rsHelperService = new EnhancedResultSetHelperService();
193193
rsHelperService.setDateFormat(this.dateFormat);
194194
rsHelperService.setDateTimeFormat(this.dateTimeFormat);
195195
rsHelperService.setFloatingPointFormat(this.decimalFormat);
196196
rsHelperService.setIntegerFormat(this.decimalFormat);
197+
rsHelperService.setNullFormat(getOptionValueAsString(OPTION_NULLVAL, DEFAULT_NULL_FORMAT));
197198
return rsHelperService;
198199
}
199200

Diff for: ‎src/test/java/com/ing/data/cassandra/jdbc/UsingCassandraContainerTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ static void initConnectionUsingIpV6(final String keyspace, final String... param
6666
sqlConnection = newConnection(keyspace, true, parameters);
6767
}
6868

69-
static CassandraConnection newConnection(final String keyspace, final String... parameters) throws Exception {
69+
public static CassandraConnection newConnection(final String keyspace,
70+
final String... parameters) throws Exception {
7071
return newConnection(keyspace, false, parameters);
7172
}
7273

Diff for: ‎src/test/java/com/ing/data/cassandra/jdbc/commands/CopyFromCommandTest.java

+75-22
Original file line numberDiff line numberDiff line change
@@ -18,52 +18,105 @@
1818
import com.ing.data.cassandra.jdbc.UsingCassandraContainerTest;
1919
import org.junit.jupiter.api.BeforeAll;
2020
import org.junit.jupiter.api.Test;
21+
import org.junit.jupiter.params.ParameterizedTest;
22+
import org.junit.jupiter.params.provider.Arguments;
23+
import org.junit.jupiter.params.provider.MethodSource;
2124
import org.slf4j.Logger;
2225
import org.slf4j.LoggerFactory;
2326

27+
import java.math.BigDecimal;
2428
import java.net.URL;
25-
import java.sql.PreparedStatement;
26-
import java.sql.ResultSet;
2729
import java.sql.SQLException;
30+
import java.sql.Statement;
31+
import java.util.stream.Stream;
2832

29-
import static org.junit.jupiter.api.Assertions.assertEquals;
33+
import static com.ing.data.cassandra.jdbc.utils.CopyCommandsTestUtils.COPY_CMD_TEST_KEYSPACE;
34+
import static com.ing.data.cassandra.jdbc.utils.CopyCommandsTestUtils.COPY_CMD_TEST_PARTIAL_TABLE;
35+
import static com.ing.data.cassandra.jdbc.utils.CopyCommandsTestUtils.COPY_CMD_TEST_PARTIAL_TABLE_NAME;
36+
import static com.ing.data.cassandra.jdbc.utils.CopyCommandsTestUtils.COPY_CMD_TEST_TABLE;
37+
import static com.ing.data.cassandra.jdbc.utils.CopyCommandsTestUtils.COPY_CMD_TEST_TABLE_NAME;
38+
import static com.ing.data.cassandra.jdbc.utils.CopyCommandsTestUtils.assertRowValues;
39+
import static org.apache.commons.lang3.StringUtils.EMPTY;
3040
import static org.junit.jupiter.api.Assertions.assertNotNull;
31-
import static org.junit.jupiter.api.Assertions.assertTrue;
3241
import static org.junit.jupiter.api.Assertions.fail;
3342

3443
public class CopyFromCommandTest extends UsingCassandraContainerTest {
3544

3645
private static final Logger LOG = LoggerFactory.getLogger(CopyFromCommandTest.class);
37-
private static final String KEYSPACE = "test_keyspace";
3846

3947
@BeforeAll
4048
static void finalizeSetUpTests() throws Exception {
41-
initConnection(KEYSPACE, "localdatacenter=datacenter1");
49+
initConnection(COPY_CMD_TEST_KEYSPACE, "localdatacenter=datacenter1");
4250
}
4351

4452
static String getTestOriginPath(final String csvFile) {
45-
final URL cqlScriptResourceUrl = CopyFromCommandTest.class.getClassLoader().getResource(csvFile);
53+
final URL cqlScriptResourceUrl =
54+
CopyFromCommandTest.class.getClassLoader().getResource("copyFromTests/" + csvFile);
4655
if (cqlScriptResourceUrl == null) {
47-
fail("Could not find the CSV script to import: " + csvFile);
56+
fail("Could not find the CSV script to import in 'copyFromTests' directory: " + csvFile);
4857
}
4958
return cqlScriptResourceUrl.getPath();
5059
}
5160

52-
@Test
53-
void givenTableAndOriginFile_whenExecuteCopyFromCommand_executeExpectedStatements() throws SQLException {
61+
private void executeCopyFromCommand(final String targetTable, final String csvSourceFile,
62+
final String options) throws SQLException {
5463
assertNotNull(sqlConnection);
55-
sqlConnection.createStatement()
56-
.execute(String.format("COPY cf_test1 FROM '%s'", getTestOriginPath("test_copy_from_cmd.csv")));
57-
58-
// Verify the rows to insert as specified in the executed script are effectively present.
59-
final PreparedStatement verifyStmt = sqlConnection.prepareStatement(
60-
"SELECT keyname, t1bValue, t1iValue FROM cf_test1 WHERE keyname = ?");
61-
verifyStmt.setString(1, "key200");
62-
ResultSet verifyRs = verifyStmt.executeQuery();
63-
assertTrue(verifyRs.next());
64-
assertEquals("key200", verifyRs.getString("keyname"));
65-
assertTrue(verifyRs.getBoolean("t1bValue"));
66-
assertEquals(654, verifyRs.getInt("t1iValue"));
64+
final Statement truncateTableStmt = sqlConnection.createStatement();
65+
truncateTableStmt.execute(String.format("TRUNCATE TABLE %s", targetTable));
66+
truncateTableStmt.close();
67+
sqlConnection.createStatement().execute(String.format("COPY %s FROM '%s' %s", targetTable,
68+
getTestOriginPath(csvSourceFile), options));
69+
}
70+
71+
static Stream<Arguments> buildCopyFromCommandVariableParameters() {
72+
return Stream.of(
73+
Arguments.of("test_simple.csv", EMPTY),
74+
Arguments.of("test_with_headers.csv", "WITH HEADER=true"),
75+
Arguments.of("test_with_special_format.csv",
76+
"WITH DELIMITER=| AND QUOTE=` AND DECIMALSEP=, AND THOUSANDSSEP=.")
77+
);
78+
}
79+
80+
@ParameterizedTest
81+
@MethodSource("buildCopyFromCommandVariableParameters")
82+
void givenTableAndOriginFile_whenExecuteCopyFromCommand_executeExpectedStatements(final String csvFile,
83+
final String options)
84+
throws SQLException {
85+
executeCopyFromCommand(COPY_CMD_TEST_TABLE_NAME, csvFile, options);
86+
assertRowValues(sqlConnection, COPY_CMD_TEST_TABLE, "key1", true, new BigDecimal("654000.7"));
87+
assertRowValues(sqlConnection, COPY_CMD_TEST_TABLE, "key2", false, new BigDecimal("321000.8"));
88+
}
89+
90+
@Test
91+
void givenTableAndOriginFile_whenExecuteCopyFromCommandWithSkipRows_executeExpectedStatements()
92+
throws SQLException {
93+
executeCopyFromCommand(COPY_CMD_TEST_PARTIAL_TABLE_NAME, "test_partial_import.csv", "WITH SKIPROWS=2");
94+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key3", 3, "N/A");
95+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key4", 4, "test4");
96+
}
97+
98+
@Test
99+
void givenTableAndOriginFile_whenExecuteCopyFromCommandWithMaxRows_executeExpectedStatements()
100+
throws SQLException {
101+
executeCopyFromCommand(COPY_CMD_TEST_PARTIAL_TABLE_NAME, "test_partial_import.csv", "WITH MAXROWS=1");
102+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key1", 1, "test1");
103+
}
104+
105+
@Test
106+
void givenTableAndOriginFile_whenExecuteCopyFromCommandWithPartialImportOptions_executeExpectedStatements()
107+
throws SQLException {
108+
executeCopyFromCommand(COPY_CMD_TEST_PARTIAL_TABLE_NAME, "test_partial_import.csv",
109+
"WITH MAXROWS=2 AND SKIPROWS=1 AND SKIPCOLS=str_val");
110+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key2", 2, null);
111+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key3", 3, null);
112+
}
113+
114+
@Test
115+
void givenTableAndOriginFile_whenExecuteCopyFromCommandWithNullValueOption_executeExpectedStatements()
116+
throws SQLException {
117+
executeCopyFromCommand(COPY_CMD_TEST_PARTIAL_TABLE_NAME, "test_partial_import.csv", "WITH NULLVAL=N/A");
118+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key1", 1, "test1");
119+
assertRowValues(sqlConnection, COPY_CMD_TEST_PARTIAL_TABLE, "key3", 3, null);
67120
}
68121

69122
}

Diff for: ‎src/test/java/com/ing/data/cassandra/jdbc/utils/CopyCommandsTestUtils.java

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.ing.data.cassandra.jdbc.utils;
15+
16+
import org.apache.commons.lang3.tuple.Pair;
17+
18+
import java.sql.Connection;
19+
import java.sql.PreparedStatement;
20+
import java.sql.ResultSet;
21+
import java.sql.SQLException;
22+
23+
import static org.junit.jupiter.api.Assertions.assertEquals;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
public class CopyCommandsTestUtils {
27+
28+
public static final String COPY_CMD_TEST_KEYSPACE = "copy_cmd_keyspace";
29+
30+
public static final String COPY_CMD_TEST_TABLE_NAME = "copy_cmd_table";
31+
public static final Pair<String, String> COPY_CMD_TEST_TABLE =
32+
Pair.of(COPY_CMD_TEST_TABLE_NAME, "table_key, bool_val, decimal_val");
33+
34+
public static final String COPY_CMD_TEST_PARTIAL_TABLE_NAME = "copy_cmd_skip_rows_table";
35+
public static final Pair<String, String> COPY_CMD_TEST_PARTIAL_TABLE =
36+
Pair.of(COPY_CMD_TEST_PARTIAL_TABLE_NAME, "table_key, int_val, str_val");
37+
38+
public static final String COPY_CMD_TEST_ALL_TYPES_TABLE = "copy_cmd_all_types_table";
39+
40+
public static void assertRowValues(final Connection connection,
41+
final Pair<String, String> tableDesc,
42+
final Object... expectedValues) throws SQLException {
43+
assert expectedValues != null && expectedValues.length > 0 : "Specify at least one expected value";
44+
final PreparedStatement verifyStmt = connection.prepareStatement(
45+
String.format("SELECT %s FROM %s WHERE table_key = ?", tableDesc.getValue(), tableDesc.getKey()));
46+
verifyStmt.setObject(1, expectedValues[0]);
47+
final ResultSet verifyRs = verifyStmt.executeQuery();
48+
assertTrue(verifyRs.next());
49+
for (int i = 1; i <= expectedValues.length; i++) {
50+
assertEquals(expectedValues[i - 1], verifyRs.getObject(i));
51+
}
52+
verifyRs.close();
53+
verifyStmt.close();
54+
}
55+
56+
}

Diff for: ‎src/test/resources/copyFromTests/test_partial_import.csv

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
"key1", 1, "test1"
2+
"key2", 2, "test2"
3+
"key3", 3, "N/A"
4+
"key4", 4, "test4"

Diff for: ‎src/test/resources/copyFromTests/test_simple.csv

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
"key1", "true", "654000.7"
2+
"key2", "false", "321000.8"

Diff for: ‎src/test/resources/copyFromTests/test_with_headers.csv

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"table_key", "decimal_val", "bool_val"
2+
"key1", 654000.7, "true"
3+
"key2", 321000.8, "false"

Diff for: ‎src/test/resources/copyFromTests/test_with_special_format.csv

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
`key1`|`true`|`654.000,7`
2+
`key2`|`false`|`321.000,8`

Diff for: ‎src/test/resources/initEmbeddedCassandra.cql

+15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
/* Init keyspaces and tables for Copy<From|To>CommandTest */
2+
DROP KEYSPACE IF EXISTS copy_cmd_keyspace;
3+
CREATE KEYSPACE "copy_cmd_keyspace" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
4+
5+
USE copy_cmd_keyspace;
6+
CREATE TABLE copy_cmd_table (
7+
table_key text PRIMARY KEY,
8+
bool_val boolean,
9+
decimal_val decimal);
10+
11+
CREATE TABLE copy_cmd_skip_rows_table (
12+
table_key text PRIMARY KEY,
13+
int_val int,
14+
str_val text);
15+
116
/* Init keyspaces and tables for MetadataResultSetsUnitTest */
217
DROP KEYSPACE IF EXISTS test_keyspace;
318
CREATE KEYSPACE "test_keyspace" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};

Diff for: ‎src/test/resources/test_copy_from_cmd.csv

-2
This file was deleted.

0 commit comments

Comments
 (0)