NIFI-14491 Support other types in CreateBoxFileMetadataInstance

awelless · pvillard31 · commit 23d177be3eb6 · 2025-04-24T11:54:47.000+02:00
Signed-off-by: Pierre Villard <pierre.villard.fr@gmail.com> This closes #9891.
diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstance.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstance.java
@@ -37,14 +37,20 @@
 import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.box.utils.BoxDate;
 import org.apache.nifi.serialization.RecordReader;
 import org.apache.nifi.serialization.RecordReaderFactory;
 import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
 
 import java.io.InputStream;
+import java.time.LocalDate;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static java.lang.String.valueOf;
@@ -223,20 +229,58 @@ private void processRecord(Record record, Metadata metadata, List<String> errors
             return;
         }
 
-        List<String> fieldNames = record.getSchema().getFieldNames();
+        final List<RecordField> fields = record.getSchema().getFields();
 
-        if (fieldNames.isEmpty()) {
+        if (fields.isEmpty()) {
             errors.add("Record has no fields");
             return;
         }
 
-        for (String fieldName : fieldNames) {
-            Object valueObj = record.getValue(fieldName);
-            String value = valueObj != null ? valueObj.toString() : null;
-            metadata.add("/" + fieldName, value);
+        for (final RecordField field : fields) {
+            addValueToMetadata(metadata, record, field);
         }
     }
 
+    private void addValueToMetadata(final Metadata metadata, final Record record, final RecordField field) {
+        if (record.getValue(field) == null) {
+            return;
+        }
+
+        final RecordFieldType fieldType = field.getDataType().getFieldType();
+        final String fieldName = field.getFieldName();
+        final String path = "/" + fieldName;
+
+        if (isNumber(fieldType)) {
+            metadata.add(path, record.getAsDouble(fieldName));
+        } else if (isDate(fieldType)) {
+            final LocalDate date = record.getAsLocalDate(fieldName, null);
+            metadata.add(path, BoxDate.of(date).format());
+        } else if (isArray(fieldType)) {
+            final List<String> values = Arrays.stream(record.getAsArray(fieldName))
+                    .filter(Objects::nonNull)
+                    .map(Object::toString)
+                    .toList();
+
+            metadata.add(path, values);
+        } else {
+            metadata.add(path, record.getAsString(fieldName));
+        }
+    }
+
+    private boolean isNumber(final RecordFieldType fieldType) {
+        final boolean isInteger = RecordFieldType.BIGINT.equals(fieldType) || RecordFieldType.BIGINT.isWiderThan(fieldType);
+        final boolean isFloat = RecordFieldType.DECIMAL.equals(fieldType) || RecordFieldType.DECIMAL.isWiderThan(fieldType);
+        return isInteger || isFloat;
+    }
+
+    private boolean isDate(final RecordFieldType fieldType) {
+        return RecordFieldType.DATE.equals(fieldType);
+    }
+
+    private boolean isArray(final RecordFieldType fieldType) {
+        return RecordFieldType.ARRAY.equals(fieldType);
+    }
+
     /**
      * Returns a BoxFile object for the given file ID.
      *
diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstance.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstance.java
@@ -37,11 +37,15 @@
 import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.box.utils.BoxDate;
 import org.apache.nifi.serialization.RecordReader;
 import org.apache.nifi.serialization.RecordReaderFactory;
 import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
 
 import java.io.InputStream;
+import java.time.LocalDate;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -216,8 +220,16 @@ private Map<String, Object> readDesiredState(final ProcessSession session,
 
             final Record record = recordReader.nextRecord();
             if (record != null) {
-                for (String fieldName : record.getSchema().getFieldNames()) {
-                    desiredState.put(fieldName, record.getValue(fieldName));
+                final List<RecordField> fields = record.getSchema().getFields();
+                for (final RecordField field : fields) {
+                    final String fieldName = field.getFieldName();
+                    final RecordFieldType type = field.getDataType().getFieldType();
+
+                    final Object value = RecordFieldType.DATE.equals(type)
+                            ? record.getAsLocalDate(fieldName, null) // Ensuring dates are read as LocalDate.
+                            : record.getValue(field);
+
+                    desiredState.put(fieldName, value);
                 }
             }
         }
@@ -269,13 +281,15 @@ private void updateField(final Metadata metadata,
             switch (value) {
                 case Number n -> metadata.replace(propertyPath, n.doubleValue());
                 case List<?> l -> metadata.replace(propertyPath, convertListToStringList(l, propertyPath));
+                case LocalDate d -> metadata.replace(propertyPath, BoxDate.of(d).format());
                 default -> metadata.replace(propertyPath, value.toString());
             }
         } else {
             // Add new field
             switch (value) {
                 case Number n -> metadata.add(propertyPath, n.doubleValue());
                 case List<?> l -> metadata.add(propertyPath, convertListToStringList(l, propertyPath));
+                case LocalDate d -> metadata.add(propertyPath, BoxDate.of(d).format());
                 default -> metadata.add(propertyPath, value.toString());
             }
         }
diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/utils/BoxDate.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/utils/BoxDate.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.box.utils;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+
+/**
+ * A wrapper class for formatting {@link LocalDate} to a string that is accepted by Box Metadata API.
+ */
+public final class BoxDate {
+
+    // The time part is always set to 0. https://developer.box.com/guides/metadata/fields/date/
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T00:00:00.000Z'");
+
+    private final LocalDate date;
+
+    private BoxDate(final LocalDate date) {
+        this.date = date;
+    }
+
+    public static BoxDate of(final LocalDate date) {
+        return new BoxDate(date);
+    }
+
+    public String format() {
+        return DATE_FORMATTER.format(date);
+    }
+}
diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstanceTest.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/CreateBoxFileMetadataInstanceTest.java
@@ -31,6 +31,13 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import java.text.ParseException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.Date;
+import java.util.List;
+
+import static java.time.ZoneOffset.UTC;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doThrow;
@@ -75,14 +82,20 @@ private void configureJsonRecordReader(TestRunner runner) throws InitializationE
     }
 
     @Test
-    void testSuccessfulMetadataCreation() {
+    void testSuccessfulMetadataCreation() throws ParseException {
         final String inputJson = """
                 {
                   "audience": "internal",
                   "documentType": "Q1 plans",
                   "competitiveDocument": "no",
                   "status": "active",
-                  "author": "Jones"
+                  "author": "Jones",
+                  "int": 1,
+                  "double": 1.234,
+                  "almostTenToThePowerOfThirty": 1000000000000000000000000000123,
+                  "array": [ "one", "two", "three" ],
+                  "intArray": [ 1, 2, 3 ],
+                  "date": "2025-01-01"
                 }""";
 
         testRunner.enqueue(inputJson);
@@ -97,6 +110,12 @@ void testSuccessfulMetadataCreation() {
         assertEquals("no", capturedMetadata.getValue("/competitiveDocument").asString());
         assertEquals("active", capturedMetadata.getValue("/status").asString());
         assertEquals("Jones", capturedMetadata.getValue("/author").asString());
+        assertEquals(1, capturedMetadata.getValue("/int").asInt());
+        assertEquals(1.234, capturedMetadata.getDouble("/double"));
+        assertEquals(1e30, capturedMetadata.getDouble("/almostTenToThePowerOfThirty")); // Precision loss is accepted.
+        assertEquals(List.of("one", "two", "three"), capturedMetadata.getMultiSelect("/array"));
+        assertEquals(List.of("1", "2", "3"), capturedMetadata.getMultiSelect("/intArray"));
+        assertEquals(createLegacyDate(2025, 1, 1), capturedMetadata.getDate("/date"));
 
         testRunner.assertAllFlowFilesTransferred(CreateBoxFileMetadataInstance.REL_SUCCESS, 1);
         final MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(CreateBoxFileMetadataInstance.REL_SUCCESS).getFirst();
@@ -157,4 +176,9 @@ void testTemplateNotFound() {
         flowFile.assertAttributeEquals(BoxFileAttributes.ERROR_MESSAGE, "API Error [404]");
     }
 
+    private static Date createLegacyDate(int year, int month, int day) {
+        final LocalDate date = LocalDate.of(year, month, day);
+        final Instant instant = date.atStartOfDay(UTC).toInstant();
+        return Date.from(instant);
+    }
 }
diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstanceTest.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/UpdateBoxFileMetadataInstanceTest.java
@@ -91,6 +91,7 @@ void setUp() throws Exception {
     private void configureJsonRecordReader(TestRunner runner) throws InitializationException {
         final JsonTreeReader readerService = new JsonTreeReader();
         runner.addControllerService("json-reader", readerService);
+        runner.setProperty(readerService, "Date Format", "yyyy-MM-dd");
         runner.enableControllerService(readerService);
     }
 
@@ -290,7 +291,8 @@ void testAddingDifferentDataTypes() {
                   "doubleField": 42.5,
                   "booleanField": true,
                   "listField": ["item1", "item2", "item3"],
-                  "emptyListField": []
+                  "emptyListField": [],
+                  "date": "2025-01-01"
                 }""";
 
         testRunner.enqueue(inputJson);
@@ -301,6 +303,7 @@ void testAddingDifferentDataTypes() {
         verify(mockMetadata).add("/numberField", 42.0); // Numbers are stored as doubles
         verify(mockMetadata).add("/doubleField", 42.5);
         verify(mockMetadata).add("/booleanField", "true"); // Booleans are stored as strings
+        verify(mockMetadata).add("/date", "2025-01-01T00:00:00.000Z"); // Dates have a specific format.
         // We need to use doAnswer/when to capture and verify list fields being added, but this is simpler
 
         verify(mockBoxFile).updateMetadata(any(Metadata.class));
diff --git a/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/utils/BoxDateTest.java b/nifi-extension-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/utils/BoxDateTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.box.utils;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.time.LocalDate;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class BoxDateTest {
+
+    @ParameterizedTest
+    @CsvSource({
+            "2025-01-01, 2025-01-01T00:00:00.000Z",
+            "2025-02-25, 2025-02-25T00:00:00.000Z",
+            "2025-11-10, 2025-11-10T00:00:00.000Z",
+            })
+    void format(final LocalDate date, final String expected) {
+        final BoxDate boxDate = BoxDate.of(date);
+        final String formatted = boxDate.format();
+
+        assertEquals(expected, formatted);
+    }
+}
