1// Protocol Buffers - Google's data interchange format 2// Copyright 2008 Google Inc. All rights reserved. 3// http://code.google.com/p/protobuf/ 4// 5// Redistribution and use in source and binary forms, with or without 6// modification, are permitted provided that the following conditions are 7// met: 8// 9// * Redistributions of source code must retain the above copyright 10// notice, this list of conditions and the following disclaimer. 11// * Redistributions in binary form must reproduce the above 12// copyright notice, this list of conditions and the following disclaimer 13// in the documentation and/or other materials provided with the 14// distribution. 15// * Neither the name of Google Inc. nor the names of its 16// contributors may be used to endorse or promote products derived from 17// this software without specific prior written permission. 18// 19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31// Author: brianolson@google.com (Brian Olson) 32// 33// This file contains the implementation of classes GzipInputStream and 34// GzipOutputStream. 35 36#include "config.h" 37 38#if HAVE_ZLIB 39#include <google/protobuf/io/gzip_stream.h> 40 41#include <google/protobuf/stubs/common.h> 42 43namespace google { 44namespace protobuf { 45namespace io { 46 47static const int kDefaultBufferSize = 65536; 48 49GzipInputStream::GzipInputStream( 50 ZeroCopyInputStream* sub_stream, Format format, int buffer_size) 51 : format_(format), sub_stream_(sub_stream), zerror_(Z_OK) { 52 zcontext_.zalloc = Z_NULL; 53 zcontext_.zfree = Z_NULL; 54 zcontext_.opaque = Z_NULL; 55 zcontext_.total_out = 0; 56 zcontext_.next_in = NULL; 57 zcontext_.avail_in = 0; 58 zcontext_.total_in = 0; 59 zcontext_.msg = NULL; 60 if (buffer_size == -1) { 61 output_buffer_length_ = kDefaultBufferSize; 62 } else { 63 output_buffer_length_ = buffer_size; 64 } 65 output_buffer_ = operator new(output_buffer_length_); 66 GOOGLE_CHECK(output_buffer_ != NULL); 67 zcontext_.next_out = static_cast<Bytef*>(output_buffer_); 68 zcontext_.avail_out = output_buffer_length_; 69 output_position_ = output_buffer_; 70} 71GzipInputStream::~GzipInputStream() { 72 operator delete(output_buffer_); 73 zerror_ = inflateEnd(&zcontext_); 74} 75 76static inline int internalInflateInit2( 77 z_stream* zcontext, GzipInputStream::Format format) { 78 int windowBitsFormat = 0; 79 switch (format) { 80 case GzipInputStream::GZIP: windowBitsFormat = 16; break; 81 case GzipInputStream::AUTO: windowBitsFormat = 32; break; 82 case GzipInputStream::ZLIB: windowBitsFormat = 0; break; 83 } 84 return inflateInit2(zcontext, /* windowBits */15 | windowBitsFormat); 85} 86 87int GzipInputStream::Inflate(int flush) { 88 if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) { 89 // previous inflate filled output buffer. don't change input params yet. 90 } else if (zcontext_.avail_in == 0) { 91 const void* in; 92 int in_size; 93 bool first = zcontext_.next_in == NULL; 94 bool ok = sub_stream_->Next(&in, &in_size); 95 if (!ok) { 96 zcontext_.next_out = NULL; 97 zcontext_.avail_out = 0; 98 return Z_STREAM_END; 99 } 100 zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in)); 101 zcontext_.avail_in = in_size; 102 if (first) { 103 int error = internalInflateInit2(&zcontext_, format_); 104 if (error != Z_OK) { 105 return error; 106 } 107 } 108 } 109 zcontext_.next_out = static_cast<Bytef*>(output_buffer_); 110 zcontext_.avail_out = output_buffer_length_; 111 output_position_ = output_buffer_; 112 int error = inflate(&zcontext_, flush); 113 return error; 114} 115 116void GzipInputStream::DoNextOutput(const void** data, int* size) { 117 *data = output_position_; 118 *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_); 119 output_position_ = zcontext_.next_out; 120} 121 122// implements ZeroCopyInputStream ---------------------------------- 123bool GzipInputStream::Next(const void** data, int* size) { 124 bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) 125 || (zerror_ == Z_BUF_ERROR); 126 if ((!ok) || (zcontext_.next_out == NULL)) { 127 return false; 128 } 129 if (zcontext_.next_out != output_position_) { 130 DoNextOutput(data, size); 131 return true; 132 } 133 if (zerror_ == Z_STREAM_END) { 134 if (zcontext_.next_out != NULL) { 135 // sub_stream_ may have concatenated streams to follow 136 zerror_ = inflateEnd(&zcontext_); 137 if (zerror_ != Z_OK) { 138 return false; 139 } 140 zerror_ = internalInflateInit2(&zcontext_, format_); 141 if (zerror_ != Z_OK) { 142 return false; 143 } 144 } else { 145 *data = NULL; 146 *size = 0; 147 return false; 148 } 149 } 150 zerror_ = Inflate(Z_NO_FLUSH); 151 if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) { 152 // The underlying stream's Next returned false inside Inflate. 153 return false; 154 } 155 ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) 156 || (zerror_ == Z_BUF_ERROR); 157 if (!ok) { 158 return false; 159 } 160 DoNextOutput(data, size); 161 return true; 162} 163void GzipInputStream::BackUp(int count) { 164 output_position_ = reinterpret_cast<void*>( 165 reinterpret_cast<uintptr_t>(output_position_) - count); 166} 167bool GzipInputStream::Skip(int count) { 168 const void* data; 169 int size; 170 bool ok = Next(&data, &size); 171 while (ok && (size < count)) { 172 count -= size; 173 ok = Next(&data, &size); 174 } 175 if (size > count) { 176 BackUp(size - count); 177 } 178 return ok; 179} 180int64 GzipInputStream::ByteCount() const { 181 return zcontext_.total_out + 182 (((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_)); 183} 184 185// ========================================================================= 186 187GzipOutputStream::Options::Options() 188 : format(GZIP), 189 buffer_size(kDefaultBufferSize), 190 compression_level(Z_DEFAULT_COMPRESSION), 191 compression_strategy(Z_DEFAULT_STRATEGY) {} 192 193GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) { 194 Init(sub_stream, Options()); 195} 196 197GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream, 198 const Options& options) { 199 Init(sub_stream, options); 200} 201 202void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream, 203 const Options& options) { 204 sub_stream_ = sub_stream; 205 sub_data_ = NULL; 206 sub_data_size_ = 0; 207 208 input_buffer_length_ = options.buffer_size; 209 input_buffer_ = operator new(input_buffer_length_); 210 GOOGLE_CHECK(input_buffer_ != NULL); 211 212 zcontext_.zalloc = Z_NULL; 213 zcontext_.zfree = Z_NULL; 214 zcontext_.opaque = Z_NULL; 215 zcontext_.next_out = NULL; 216 zcontext_.avail_out = 0; 217 zcontext_.total_out = 0; 218 zcontext_.next_in = NULL; 219 zcontext_.avail_in = 0; 220 zcontext_.total_in = 0; 221 zcontext_.msg = NULL; 222 // default to GZIP format 223 int windowBitsFormat = 16; 224 if (options.format == ZLIB) { 225 windowBitsFormat = 0; 226 } 227 zerror_ = deflateInit2( 228 &zcontext_, 229 options.compression_level, 230 Z_DEFLATED, 231 /* windowBits */15 | windowBitsFormat, 232 /* memLevel (default) */8, 233 options.compression_strategy); 234} 235 236GzipOutputStream::~GzipOutputStream() { 237 Close(); 238 if (input_buffer_ != NULL) { 239 operator delete(input_buffer_); 240 } 241} 242 243// private 244int GzipOutputStream::Deflate(int flush) { 245 int error = Z_OK; 246 do { 247 if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) { 248 bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_); 249 if (!ok) { 250 sub_data_ = NULL; 251 sub_data_size_ = 0; 252 return Z_BUF_ERROR; 253 } 254 GOOGLE_CHECK_GT(sub_data_size_, 0); 255 zcontext_.next_out = static_cast<Bytef*>(sub_data_); 256 zcontext_.avail_out = sub_data_size_; 257 } 258 error = deflate(&zcontext_, flush); 259 } while (error == Z_OK && zcontext_.avail_out == 0); 260 if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) { 261 // Notify lower layer of data. 262 sub_stream_->BackUp(zcontext_.avail_out); 263 // We don't own the buffer anymore. 264 sub_data_ = NULL; 265 sub_data_size_ = 0; 266 } 267 return error; 268} 269 270// implements ZeroCopyOutputStream --------------------------------- 271bool GzipOutputStream::Next(void** data, int* size) { 272 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) { 273 return false; 274 } 275 if (zcontext_.avail_in != 0) { 276 zerror_ = Deflate(Z_NO_FLUSH); 277 if (zerror_ != Z_OK) { 278 return false; 279 } 280 } 281 if (zcontext_.avail_in == 0) { 282 // all input was consumed. reset the buffer. 283 zcontext_.next_in = static_cast<Bytef*>(input_buffer_); 284 zcontext_.avail_in = input_buffer_length_; 285 *data = input_buffer_; 286 *size = input_buffer_length_; 287 } else { 288 // The loop in Deflate should consume all avail_in 289 GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed"; 290 } 291 return true; 292} 293void GzipOutputStream::BackUp(int count) { 294 GOOGLE_CHECK_GE(zcontext_.avail_in, count); 295 zcontext_.avail_in -= count; 296} 297int64 GzipOutputStream::ByteCount() const { 298 return zcontext_.total_in + zcontext_.avail_in; 299} 300 301bool GzipOutputStream::Flush() { 302 zerror_ = Deflate(Z_FULL_FLUSH); 303 // Return true if the flush succeeded or if it was a no-op. 304 return (zerror_ == Z_OK) || 305 (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 && 306 zcontext_.avail_out != 0); 307} 308 309bool GzipOutputStream::Close() { 310 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) { 311 return false; 312 } 313 do { 314 zerror_ = Deflate(Z_FINISH); 315 } while (zerror_ == Z_OK); 316 zerror_ = deflateEnd(&zcontext_); 317 bool ok = zerror_ == Z_OK; 318 zerror_ = Z_STREAM_END; 319 return ok; 320} 321 322} // namespace io 323} // namespace protobuf 324} // namespace google 325 326#endif // HAVE_ZLIB 327