diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetCodecFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetCodecFactory.java new file mode 100644 index 000000000000..87fe8b664def --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetCodecFactory.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.parquet.conf.ParquetConfiguration; +import org.apache.parquet.hadoop.BadConfigurationException; +import org.apache.parquet.hadoop.CodecFactory; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.hadoop.util.ConfigurationUtil; + +import java.lang.reflect.Constructor; + +/** + * A {@link CodecFactory} that instantiates codecs without Hadoop's {@code ReflectionUtils}. + * + *

{@code ReflectionUtils.newInstance} caches the constructor of every class it creates in a + * static map with strong keys. When Hadoop is loaded by a shared class loader, for example from + * {@code /opt/flink/lib} of a Flink session cluster, while Paimon comes with the job jar, the keys + * are Parquet's own codec classes (such as {@code ZstandardCodec}) loaded by the job class loader. + * The map then keeps every finished job's class loader, and its Metaspace, reachable for the + * lifetime of the process. + */ +public class ParquetCodecFactory extends CodecFactory { + + public ParquetCodecFactory(Configuration configuration, int pageSize) { + super(configuration, pageSize); + } + + public ParquetCodecFactory(ParquetConfiguration configuration, int pageSize) { + super(configuration, pageSize); + } + + @Override + protected CompressionCodec getCodec(CompressionCodecName codecName) { + String codecClassName = codecName.getHadoopCompressionCodecClassName(); + if (codecClassName == null) { + return null; + } + String codecCacheKey = cacheKey(codecName); + CompressionCodec codec = CODEC_BY_NAME.get(codecCacheKey); + if (codec != null) { + return codec; + } + + Class codecClass; + try { + try { + codecClass = Class.forName(codecClassName); + } catch (ClassNotFoundException e) { + // Try to load the class using the job classloader + codecClass = new Configuration(false).getClassLoader().loadClass(codecClassName); + } + } catch (ClassNotFoundException e) { + throw new BadConfigurationException("Class " + codecClassName + " was not found", e); + } + + // What ReflectionUtils.newInstance(codecClass, conf) does for a codec, without putting + // codecClass into its static constructor cache. + try { + Constructor constructor = codecClass.getDeclaredConstructor(); + constructor.setAccessible(true); + codec = (CompressionCodec) constructor.newInstance(); + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Failed to instantiate codec " + codecClassName, e); + } + if (codec instanceof Configurable) { + ((Configurable) codec).setConf(ConfigurationUtil.createHadoopConfiguration(conf)); + } + CODEC_BY_NAME.put(codecCacheKey, codec); + return codec; + } + + /** Same key as the private {@code CodecFactory#cacheKey}. */ + private String cacheKey(CompressionCodecName codecName) { + String level = null; + switch (codecName) { + case GZIP: + level = conf.get("zlib.compress.level"); + break; + case BROTLI: + level = conf.get("compression.brotli.quality"); + break; + case ZSTD: + level = conf.get("parquet.compression.codec.zstd.level"); + break; + default: + // compression level is not supported; ignore it + } + String codecClass = codecName.getHadoopCompressionCodecClassName(); + return level == null ? codecClass : codecClass + ":" + level; + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java index 575252095d3d..7546a1d81dd6 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java @@ -107,7 +107,8 @@ public static ParquetFileReader getParquetReader( public static ParquetReadOptions.Builder getParquetReadOptionsBuilder(Options options) { PlainParquetConfiguration parquetConfiguration = new PlainParquetConfiguration(options.toMap()); - return ParquetReadOptions.builder(parquetConfiguration); + return ParquetReadOptions.builder(parquetConfiguration) + .withCodecFactory(new ParquetCodecFactory(parquetConfiguration, 0)); } static void assertStatsClass( diff --git a/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java b/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java index ee092bbfb909..359a8920ea94 100644 --- a/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java +++ b/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java @@ -18,6 +18,8 @@ package org.apache.parquet.hadoop; +import org.apache.paimon.format.parquet.ParquetCodecFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.bytes.ByteBufferAllocator; @@ -664,7 +666,7 @@ public SELF withSizeStatisticsEnabled(boolean enabled) { public ParquetWriter build() throws IOException { ParquetProperties encodingProps = encodingPropsBuilder.build(); if (codecFactory == null) { - codecFactory = new CodecFactory(conf, encodingProps.getPageSizeThreshold()); + codecFactory = new ParquetCodecFactory(conf, encodingProps.getPageSizeThreshold()); } return new ParquetWriter<>( diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetCodecFactoryTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetCodecFactoryTest.java new file mode 100644 index 000000000000..f5cba07e28c3 --- /dev/null +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetCodecFactoryTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FormatReaderContext; +import org.apache.paimon.format.FormatWriter; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.parquet.hadoop.CodecFactory; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link ParquetCodecFactory}. */ +class ParquetCodecFactoryTest { + + @TempDir java.nio.file.Path tempDir; + + private final RowType rowType = RowType.of(DataTypes.INT(), DataTypes.STRING()); + + /** + * Hadoop's {@code ReflectionUtils} keeps every class it instantiates in a static constructor + * cache. With Hadoop on a shared class loader and Paimon in a job jar, a Parquet codec class in + * that cache keeps the job's class loader alive after the job finishes, so neither the write + * nor the read path may put one there. + */ + @ParameterizedTest + @ValueSource(strings = {"zstd", "snappy", "lz4_raw"}) + void testCodecClassStaysOutOfHadoopConstructorCache(String compression) throws Exception { + Class codecClass = + Class.forName( + CompressionCodecName.fromConf(compression) + .getHadoopCompressionCodecClassName()); + // Other tests in this JVM may have created the codec already. + hadoopConstructorCache().remove(codecClass); + parquetCodecCache().clear(); + + Path file = new Path(tempDir.toUri().toString(), compression + ".parquet"); + FileFormat format = FileFormat.fromIdentifier("parquet", new Options()); + LocalFileIO fileIO = LocalFileIO.create(); + try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { + FormatWriter writer = format.createWriterFactory(rowType).create(out, compression); + for (int i = 0; i < 100; i++) { + writer.addElement(GenericRow.of(i, BinaryString.fromString("v" + i))); + } + writer.close(); + } + assertThat(hadoopConstructorCache()).doesNotContainKey(codecClass); + + // Read with an empty codec cache, so the reader has to create the codec itself. + parquetCodecCache().clear(); + List ids = new ArrayList<>(); + try (RecordReader reader = + format.createReaderFactory(rowType, rowType, null) + .createReader( + new FormatReaderContext( + fileIO, file, fileIO.getFileSize(file), null, null))) { + reader.forEachRemaining(row -> ids.add(row.getInt(0))); + } + assertThat(ids).hasSize(100).startsWith(0, 1, 2).endsWith(99); + assertThat(hadoopConstructorCache()).doesNotContainKey(codecClass); + } + + @SuppressWarnings("unchecked") + private static Map, ?> hadoopConstructorCache() throws Exception { + Field field = ReflectionUtils.class.getDeclaredField("CONSTRUCTOR_CACHE"); + field.setAccessible(true); + return (Map, ?>) field.get(null); + } + + @SuppressWarnings("unchecked") + private static Map parquetCodecCache() throws Exception { + Field field = CodecFactory.class.getDeclaredField("CODEC_BY_NAME"); + field.setAccessible(true); + return (Map) field.get(null); + } +}