001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.juneau.marshall.parquet; 018 019import static org.apache.juneau.commons.utils.Shorts.*; 020 021import java.io.*; 022import java.util.zip.*; 023 024/** 025 * Parquet compression codecs supported by this implementation. 026 * 027 * <p> 028 * UNCOMPRESSED and GZIP support both directions. SNAPPY is <b>decode-only</b> (a dependency-free, 029 * pure-Java Snappy <i>block-format</i> decompressor) so Juneau can read spec-compliant Parquet from 030 * parquet-mr / Spark / Hive, which default to Snappy. LZ4, Zstd, Brotli, and LZO require external 031 * libraries and remain unsupported (hard error on read). 032 */ 033public enum CompressionCodec { 034 035 /** No compression. */ 036 UNCOMPRESSED(0) { 037 038 @Override 039 byte[] compress(byte[] data) { 040 return data; 041 } 042 043 @Override 044 byte[] decompress(byte[] data, int uncompressedSize) { 045 return data; 046 } 047 }, 048 049 /** Snappy block format — decode only (write is not supported). */ 050 SNAPPY(1) { 051 052 @Override 053 byte[] compress(byte[] data) throws IOException { 054 throw new IOException("Snappy compression (write) is not supported; use UNCOMPRESSED or GZIP."); 055 } 056 057 @Override 058 byte[] decompress(byte[] data, int uncompressedSize) throws IOException { 059 return SnappyBlockDecompressor.decompress(data, uncompressedSize); 060 } 061 }, 062 063 /** GZIP compression. */ 064 GZIP(2) { 065 066 @Override 067 byte[] compress(byte[] data) throws IOException { 068 var baos = new ByteArrayOutputStream(); 069 try (var gzip = new GZIPOutputStream(baos)) { 070 gzip.write(data); 071 } 072 return baos.toByteArray(); 073 } 074 075 @Override 076 byte[] decompress(byte[] data, int uncompressedSize) throws IOException { 077 // Decode incrementally into a buffer that grows with the bytes actually produced, rather than 078 // pre-allocating the declared uncompressedSize up front. A header may declare a large size it 079 // cannot actually deliver; growing on demand means such a mismatch is detected after allocating 080 // only what was really produced, not the (possibly huge) declared amount. 081 var out = new ByteArrayOutputStream(Math.min(Math.max(uncompressedSize, 0), 8192)); 082 var buf = new byte[8192]; 083 try (var gzip = new GZIPInputStream(new ByteArrayInputStream(data))) { 084 var total = 0; 085 int r; 086 while (total < uncompressedSize && (r = gzip.read(buf, 0, Math.min(buf.length, uncompressedSize - total))) > 0) { 087 out.write(buf, 0, r); 088 total += r; 089 } 090 if (total < uncompressedSize) 091 throw new IOException("GZIP decompression produced fewer bytes than expected"); 092 } 093 return out.toByteArray(); 094 } 095 }; 096 097 /** Thrift FileMetaData Codec enum value. */ 098 final int thriftValue; 099 100 /** 101 * Constructor. 102 * 103 * @param thriftValue The Thrift codec value. 104 */ 105 CompressionCodec(int thriftValue) { 106 this.thriftValue = thriftValue; 107 } 108 109 /** 110 * Resolves a codec name for <b>writing</b>. 111 * 112 * @param name The codec name (case-insensitive). Only <c>UNCOMPRESSED</c> and <c>GZIP</c> are writable. 113 * @return The matching codec. 114 * @throws IllegalArgumentException If the name is unknown or names a decode-only codec such as <c>SNAPPY</c>. 115 */ 116 static CompressionCodec forWrite(String name) { 117 for (var c : values()) 118 if (c.writable() && c.name().equalsIgnoreCase(name)) 119 return c; 120 throw forWriteError(name); 121 } 122 123 static IllegalArgumentException forWriteError(String name) { 124 return new IllegalArgumentException("Unsupported Parquet compression codec for writing: '" + name 125 + "'. Valid write codecs: UNCOMPRESSED, GZIP." 126 + (SNAPPY.name().equalsIgnoreCase(name) ? " SNAPPY is decode-only and cannot be written." : "")); 127 } 128 129 boolean writable() { 130 return this != SNAPPY; 131 } 132 133 abstract byte[] compress(byte[] data) throws IOException; 134 135 abstract byte[] decompress(byte[] data, int uncompressedSize) throws IOException; 136 137 /** 138 * Resolves a Thrift {@code CompressionCodec} enum value to a supported codec. 139 * 140 * <p> 141 * {@code UNCOMPRESSED} (0) and {@code GZIP} (2) support both directions; {@code SNAPPY} (1) is 142 * decode-only. Any other value — a known-but-unsupported codec (LZO/BROTLI/LZ4/ZSTD/LZ4_RAW) or an 143 * unrecognized id — is a hard error rather than a silent fallback to {@code UNCOMPRESSED}, which would 144 * otherwise treat still-compressed bytes as raw page data and surface silent garbage (GAP-5). 145 * 146 * @param value The Thrift codec enum value. 147 * @return The matching supported codec. 148 * @throws IOException If the codec is not supported by this implementation. 149 */ 150 static CompressionCodec fromThrift(int value) throws IOException { 151 for (var c : values()) 152 if (c.thriftValue == value) 153 return c; 154 throw ioex("Unsupported Parquet compression codec: %s. Only UNCOMPRESSED, GZIP, and SNAPPY (decode-only) are supported.", codecName(value)); 155 } 156 157 private static String codecName(int value) { 158 return switch (value) { 159 case 3 -> "LZO (3)"; 160 case 4 -> "BROTLI (4)"; 161 case 5 -> "LZ4 (5)"; 162 case 6 -> "ZSTD (6)"; 163 case 7 -> "LZ4_RAW (7)"; 164 default -> Integer.toString(value); 165 }; 166 } 167}