Skip to content

Commit 7a8aa6f

Browse files
committed
PARQUET-134 patch - Support file write mode
1 parent a635f21 commit 7a8aa6f

3 files changed

Lines changed: 91 additions & 6 deletions

File tree

‎parquet-hadoop/src/main/java/parquet/hadoop/ParquetFileWriter.java‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ public class ParquetFileWriter {
6666
public static final byte[] MAGIC = "PAR1".getBytes(Charset.forName("ASCII"));
6767
public static final int CURRENT_VERSION = 1;
6868

69+
// File creation modes
70+
public static enum Mode {
71+
CREATE,
72+
OVERWRITE
73+
}
74+
6975
private static final ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter();
7076

7177
private final MessageType schema;
@@ -141,17 +147,30 @@ private final STATE error() throws IOException {
141147
private STATE state = STATE.NOT_STARTED;
142148

143149
/**
144-
*
150+
* @param configuration Hadoop configuration
151+
* @param schema the schema of the data
152+
* @param file the file to write to
153+
* @throws IOException if the file can not be created
154+
*/
155+
public ParquetFileWriter(Configuration configuration, MessageType schema,
156+
Path file) throws IOException {
157+
this(configuration, schema, file, Mode.CREATE);
158+
}
159+
160+
/**
161+
* @param configuration Hadoop configuration
145162
* @param schema the schema of the data
146-
* @param out the file to write to
147-
* @param codec the codec to use to compress blocks
163+
* @param file the file to write to
164+
* @param mode file creation mode
148165
* @throws IOException if the file can not be created
149166
*/
150-
public ParquetFileWriter(Configuration configuration, MessageType schema, Path file) throws IOException {
167+
public ParquetFileWriter(Configuration configuration, MessageType schema,
168+
Path file, Mode mode) throws IOException {
151169
super();
152170
this.schema = schema;
153171
FileSystem fs = file.getFileSystem(configuration);
154-
this.out = fs.create(file, false);
172+
boolean overwriteFlag = (mode == Mode.OVERWRITE);
173+
this.out = fs.create(file, overwriteFlag);
155174
}
156175

157176
/**

‎parquet-hadoop/src/main/java/parquet/hadoop/ParquetWriter.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,11 +169,44 @@ public ParquetWriter(
169169
boolean validating,
170170
WriterVersion writerVersion,
171171
Configuration conf) throws IOException {
172+
this(file, ParquetFileWriter.Mode.CREATE, writeSupport,
173+
compressionCodecName, blockSize, pageSize, dictionaryPageSize,
174+
enableDictionary, validating, writerVersion, conf);
175+
}
176+
/**
177+
* Create a new ParquetWriter.
178+
*
179+
* @param file the file to create
180+
* @param mode file creation mode
181+
* @param writeSupport the implementation to write a record to a RecordConsumer
182+
* @param compressionCodecName the compression codec to use
183+
* @param blockSize the block size threshold
184+
* @param pageSize the page size threshold
185+
* @param dictionaryPageSize the page size threshold for the dictionary pages
186+
* @param enableDictionary to turn dictionary encoding on
187+
* @param validating to turn on validation using the schema
188+
* @param writerVersion version of parquetWriter from {@link ParquetProperties.WriterVersion}
189+
* @param conf Hadoop configuration to use while accessing the filesystem
190+
* @throws IOException
191+
*/
192+
public ParquetWriter(
193+
Path file,
194+
ParquetFileWriter.Mode mode,
195+
WriteSupport<T> writeSupport,
196+
CompressionCodecName compressionCodecName,
197+
int blockSize,
198+
int pageSize,
199+
int dictionaryPageSize,
200+
boolean enableDictionary,
201+
boolean validating,
202+
WriterVersion writerVersion,
203+
Configuration conf) throws IOException {
172204

173205
WriteSupport.WriteContext writeContext = writeSupport.init(conf);
174206
MessageType schema = writeContext.getSchema();
175207

176-
ParquetFileWriter fileWriter = new ParquetFileWriter(conf, schema, file);
208+
ParquetFileWriter fileWriter = new ParquetFileWriter(conf, schema, file,
209+
mode);
177210
fileWriter.start();
178211

179212
CodecFactory codecFactory = new CodecFactory(conf);

‎parquet-hadoop/src/test/java/parquet/hadoop/TestParquetFileWriter.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,39 @@ public class TestParquetFileWriter {
6060
private static final Log LOG = Log.getLog(TestParquetFileWriter.class);
6161
private String writeSchema;
6262

63+
@Test
64+
public void testWriteMode() throws Exception {
65+
File testDir = new File("target/test/TestParquetFileWriter/");
66+
testDir.mkdirs();
67+
File testFile = new File(testDir, "testParquetFile");
68+
testFile = testFile.getAbsoluteFile();
69+
testFile.createNewFile();
70+
MessageType schema = MessageTypeParser.parseMessageType(
71+
"message m { required group a {required binary b;} required group "
72+
+ "c { required int64 d; }}");
73+
Configuration conf = new Configuration();
74+
75+
ParquetFileWriter writer = null;
76+
boolean exceptionThrown = false;
77+
Path path = new Path(testFile.toURI());
78+
try {
79+
writer = new ParquetFileWriter(conf, schema, path,
80+
ParquetFileWriter.Mode.CREATE);
81+
} catch(IOException ioe1) {
82+
exceptionThrown = true;
83+
}
84+
assertTrue(exceptionThrown);
85+
exceptionThrown = false;
86+
try {
87+
writer = new ParquetFileWriter(conf, schema, path,
88+
ParquetFileWriter.Mode.OVERWRITE);
89+
} catch(IOException ioe2) {
90+
exceptionThrown = true;
91+
}
92+
assertTrue(!exceptionThrown);
93+
testFile.delete();
94+
}
95+
6396
@Test
6497
public void testWriteRead() throws Exception {
6598

0 commit comments

Comments
 (0)