Make getConfigMapWithPrefix auto append tailing dot if missing (#8522)

xiangfu0 · web-flow · commit 900f01f288c8 · 2022-04-12T21:03:04.000-07:00
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
@@ -176,7 +176,7 @@ public static void copyURIToLocal(Map<String, String> batchConfigMap, URI source
     if (!PinotFSFactory.isSchemeSupported(sourceFileURIScheme)) {
       PinotFSFactory.register(sourceFileURIScheme, batchConfigMap.get(BatchConfigProperties.INPUT_FS_CLASS),
           IngestionConfigUtils.getInputFsProps(IngestionConfigUtils.getConfigMapWithPrefix(
-              batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX + IngestionConfigUtils.DOT_SEPARATOR)));
+              batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX)));
     }
     PinotFSFactory.create(sourceFileURIScheme).copyToLocalFile(sourceFileURI, destFile);
   }
diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
@@ -306,7 +306,7 @@ protected SegmentGenerationTaskSpec generateTaskSpec(Map<String, String> taskCon
     SegmentNameGeneratorSpec segmentNameGeneratorSpec = new SegmentNameGeneratorSpec();
     segmentNameGeneratorSpec.setType(taskConfigs.get(BatchConfigProperties.SEGMENT_NAME_GENERATOR_TYPE));
     segmentNameGeneratorSpec.setConfigs(IngestionConfigUtils.getConfigMapWithPrefix(taskConfigs,
-        BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX + IngestionConfigUtils.DOT_SEPARATOR));
+        BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX));
     taskSpec.setSegmentNameGeneratorSpec(segmentNameGeneratorSpec);
     taskSpec.setCustomProperty(BatchConfigProperties.INPUT_DATA_FILE_URI_KEY, inputFileURI.toString());
     return taskSpec;
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
@@ -117,25 +117,25 @@ public static String getBatchSegmentIngestionFrequency(TableConfig tableConfig)
    * Fetch the properties which belong to record reader, by removing the identifier prefix
    */
   public static Map<String, String> getRecordReaderProps(Map<String, String> batchConfigMap) {
-    return getConfigMapWithPrefix(batchConfigMap, BatchConfigProperties.RECORD_READER_PROP_PREFIX + DOT_SEPARATOR);
+    return getConfigMapWithPrefix(batchConfigMap, BatchConfigProperties.RECORD_READER_PROP_PREFIX);
   }
 
   /**
    * Fetch the properties which belong to segment name generator, by removing the identifier prefix
    */
   public static Map<String, String> getSegmentNameGeneratorProps(Map<String, String> batchConfigMap) {
     return getConfigMapWithPrefix(batchConfigMap,
-        BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX + DOT_SEPARATOR);
+        BatchConfigProperties.SEGMENT_NAME_GENERATOR_PROP_PREFIX);
   }
 
   public static PinotConfiguration getInputFsProps(Map<String, String> batchConfigMap) {
     return new PinotConfiguration(
-        getPropsWithPrefix(batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX + DOT_SEPARATOR));
+        getPropsWithPrefix(batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX));
   }
 
   public static PinotConfiguration getOutputFsProps(Map<String, String> batchConfigMap) {
     return new PinotConfiguration(
-        getPropsWithPrefix(batchConfigMap, BatchConfigProperties.OUTPUT_FS_PROP_PREFIX + DOT_SEPARATOR));
+        getPropsWithPrefix(batchConfigMap, BatchConfigProperties.OUTPUT_FS_PROP_PREFIX));
   }
 
   /**
@@ -160,6 +160,9 @@ public static Map<String, Object> getPropsWithPrefix(Map<String, String> batchCo
 
   public static Map<String, String> getConfigMapWithPrefix(Map<String, String> batchConfigMap, String prefix) {
     Map<String, String> props = new HashMap<>();
+    if (!prefix.endsWith(DOT_SEPARATOR)) {
+      prefix = prefix + DOT_SEPARATOR;
+    }
     for (String configKey : batchConfigMap.keySet()) {
       if (configKey.startsWith(prefix)) {
         String[] splits = configKey.split(prefix, 2);
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.spi.utils;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import java.util.HashMap;
 import java.util.Map;
@@ -146,4 +147,11 @@ public void testGetPushType() {
     segmentsValidationAndRetentionConfig.setSegmentPushType(null);
     Assert.assertNull(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig));
   }
+
+  @Test
+  public void testGetConfigMapWithPrefix() {
+    Map<String, String> testMap = ImmutableMap.of("k1", "v1", "k1.k2", "v2", "k1.k3", "v3", "k4", "v4");
+    Assert.assertEquals(2, IngestionConfigUtils.getConfigMapWithPrefix(testMap, "k1").size());
+    Assert.assertEquals(2, IngestionConfigUtils.getConfigMapWithPrefix(testMap, "k1.").size());
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -176,7 +176,7 @@ public static void copyURIToLocal(Map<String, String> batchConfigMap, URI source`
`176`	`176`	`if (!PinotFSFactory.isSchemeSupported(sourceFileURIScheme)) {`
`177`	`177`	`PinotFSFactory.register(sourceFileURIScheme, batchConfigMap.get(BatchConfigProperties.INPUT_FS_CLASS),`
`178`	`178`	`IngestionConfigUtils.getInputFsProps(IngestionConfigUtils.getConfigMapWithPrefix(`
`179`		`- batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX + IngestionConfigUtils.DOT_SEPARATOR)));`
	`179`	`+ batchConfigMap, BatchConfigProperties.INPUT_FS_PROP_PREFIX)));`
`180`	`180`	`}`
`181`	`181`	`PinotFSFactory.create(sourceFileURIScheme).copyToLocalFile(sourceFileURI, destFile);`
`182`	`182`	`}`