Mercurial > hg > octave-thorsten
changeset 5269:a90ce2dc8b1e
[project @ 2005-04-06 19:20:09 by dbateman]
author | dbateman |
---|---|
date | Wed, 06 Apr 2005 19:20:50 +0000 |
parents | 7ca77747d680 |
children | a313e928afb1 |
files | liboctave/ChangeLog liboctave/Makefile.in src/ChangeLog src/Makefile.in src/load-save.cc src/load-save.h src/ls-mat5.cc src/ls-mat5.h src/zfstream.cc src/zfstream.h |
diffstat | 10 files changed, 1764 insertions(+), 105 deletions(-) [+] |
line wrap: on
line diff
--- a/liboctave/ChangeLog +++ b/liboctave/ChangeLog @@ -1,3 +1,7 @@ +2005-04-06 David Bateman <dbateman@free.fr> + + * Makefile.in: Link to UMFPACK_LIBS. + 2005-04-05 John W. Eaton <jwe@octave.org> * Array.cc (assignN): Avoid shadowed declaration in previous change.
--- a/liboctave/Makefile.in +++ b/liboctave/Makefile.in @@ -20,7 +20,8 @@ LINK_DEPS = \ -L../libcruft -L../glob -L. $(RLD_FLAG) \ - $(LIBCRUFT) $(BLAS_LIBS) $(FFTW_LIBS) $(LIBREADLINE) $(LIBS) $(FLIBS) + $(LIBCRUFT) $(BLAS_LIBS) $(FFTW_LIBS) $(UMFPACK_LIBS) $(LIBREADLINE) \ + $(LIBS) $(FLIBS) include $(srcdir)/COLAMD.files
--- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,52 @@ +2005-04-06 David Bateman <dbateman@free.fr> + + * Makefile.in: Link to UMFPACK_LIBS. Add zfstream.{cc,h} to build and + dist files. + + * zfstream.cc: New file for C++ binding for fstream like class for + zlib. + + * zfstream.h: Definition for fstream like C++ bindings to zlib. + + * load-save.cc (static bool check_gzip_magic (const std::string&)): + New function to look for GZIP magic + (static load_save_format get_file_format (const istream &file)): New + function split from old get_file_format but passed istream to allow + use with zlib istream. + (static load_save_format get_file_format (const std::string&, + const std::string&, bool)): Modify the test uncompressed file first, + then compressed version + (Fload) Allow -v6, -6, -v7 and -7 options. Split load code to allow + use of zlib. + (Fsave) Allow -zip, -z, -v6, -6, -v7 and -7 options. Split save code + to allow use of zlib. + + * load-save.h: add LS_MAT7_BINARY to load_save_format enum + + * ls-mat5.cc (read_mat5_binary_element): Test for miCOMPRESSED flag for + matlab v7 files, and decompress data in memory. Allow reading of matlab + logical variables either in mxDOUBLE_CLASS or mxUINT8_CLASS. + (int save_mat5_array_length(const double*, ...): New function to + calculate number of bytes used to save NDArray. + (int save_mat5_array_length(const Complex*, ...): New function to + calculate number of bytes used to save ComplexNDArray. + (int save_mat5_element_length): New function to find number of bytes + needed to save data element. + (save_mat5_binary_element): New input arguments, mat7_format and + compressing, that define if we are in a matlab v7 format and where we + are currently compressing the data element. If mat7_format use + miCOMPRESSED flag for matlab v7 files, and compress data in memory. + Add capability to save logical variables as mxUINT8_CLASS. If v7 + format maximum variable length is 63 characters and not 31. Use the + save_mat5_element_length function to pre-calculate the number of bytes + taken by a variable rather than use a negative seek to correct after + writing (zlib can't do negative seeking) + + * ls-mat5.h: Add to miCOMPRESSED, miUTF8, miUTF16 and miUTF32 flags to + the enum mat5_data_type. + (save_mat5_binary_element): Add args mat7_format and compressing to the + definition. + 2005-04-06 John W. Eaton <jwe@octave.org> * OPERATORS/op-scm-scm.cc: Delete #pragma implementation.
--- a/src/Makefile.in +++ b/src/Makefile.in @@ -96,7 +96,7 @@ lex.h load-save.h ls-hdf5.h ls-mat-ascii.h ls-mat4.h \ ls-mat5.h ls-oct-ascii.h ls-oct-binary.h ls-utils.h \ oct-fstrm.h oct-hist.h oct-iostrm.h oct-map.h oct-obj.h \ - oct-prcstrm.h oct-procbuf.h oct-stdstrm.h oct-stream.h \ + oct-prcstrm.h oct-procbuf.h oct-stdstrm.h oct-stream.h zfstream.h \ oct-strstrm.h oct-lvalue.h oct.h octave.h ops.h pager.h \ parse.h pr-output.h procstream.h sighandlers.h siglist.h \ sparse-xdiv.h sparse-xpow.h symtab.h sysdep.h \ @@ -169,7 +169,7 @@ ls-oct-binary.cc ls-utils.cc main.c mappers.cc matherr.c \ oct-fstrm.cc oct-hist.cc oct-iostrm.cc oct-map.cc \ oct-obj.cc oct-prcstrm.cc oct-procbuf.cc oct-stdstrm.cc \ - oct-stream.cc oct-strstrm.cc oct-lvalue.cc pager.cc \ + oct-stream.cc zfstream.cc oct-strstrm.cc oct-lvalue.cc pager.cc \ parse.y pr-output.cc procstream.cc sighandlers.cc \ siglist.c sparse-xdiv.cc sparse-xpow.cc strcasecmp.c \ strncase.c strfns.cc symtab.cc syscalls.cc sysdep.cc \ @@ -244,7 +244,7 @@ OCT_LINK_DEPS = \ -L../libcruft $(LIBCRUFT) -L../liboctave $(LIBOCTAVE) \ - -L. $(LIBOCTINTERP) $(BLAS_LIBS) $(FFTW_LIBS) $(LIBS) $(FLIBS) + -L. $(LIBOCTINTERP) $(BLAS_LIBS) $(FFTW_LIBS) $(UMFPACK_LIBS) $(LIBS) $(FLIBS) DISTFILES = Makefile.in ChangeLog mkdefs mkops mkgendoc \ DOCSTRINGS mkbuiltins mk-oct-links \ @@ -310,7 +310,7 @@ main.o $(DLD_STATIC_OBJ) \ $(OCTAVE_LFLAGS) \ $(OCTAVE_LIBS) \ - $(LEXLIB) $(BLAS_LIBS) $(FFTW_LIBS) $(LIBS) $(FLIBS) + $(LEXLIB) $(BLAS_LIBS) $(FFTW_LIBS) $(UMFPACK_LIBS) $(LIBS) $(FLIBS) stmp-pic: pic @if [ -f stmp-pic ]; then \
--- a/src/load-save.cc +++ b/src/load-save.cc @@ -80,6 +80,10 @@ #include "ls-oct-ascii.h" #include "ls-oct-binary.h" +#ifdef HAVE_ZLIB +#include "zfstream.h" +#endif + // Write octave-core file if Octave crashes or is killed by a signal. static bool Vcrash_dumps_octave_core; @@ -301,25 +305,28 @@ return 0; } +#ifdef HAVE_ZLIB +static bool +check_gzip_magic (const std::string& fname) +{ + bool retval = false; + std::ifstream file (fname.c_str ()); + OCTAVE_LOCAL_BUFFER (unsigned char, magic, 2); + + if (file.read (X_CAST (char *, magic), 2) && magic[0] == 0x1f && + magic[1] == 0x8b) + retval = true; + + file.close (); + return retval; +} +#endif + static load_save_format -get_file_format (const std::string& fname, const std::string& orig_fname) +get_file_format (std::istream& file) { load_save_format retval = LS_UNKNOWN; -#ifdef HAVE_HDF5 - // check this before we open the file - if (H5Fis_hdf5 (fname.c_str ()) > 0) - return LS_HDF5; -#endif /* HAVE_HDF5 */ - - std::ifstream file (fname.c_str ()); - - if (! file) - { - error ("load: couldn't open input file `%s'", orig_fname.c_str ()); - return retval; - } - oct_mach_info::float_format flt_fmt = oct_mach_info::flt_fmt_unknown; bool swap = false; @@ -358,26 +365,60 @@ if (! tmp.empty ()) retval = LS_ASCII; - else - { - // Try reading the file as numbers only, determining the - // number of rows and columns from the data. We don't - // even bother to check to see if the first item in the - // file is a number, so that get_complete_line() can - // skip any comments that might appear at the top of the - // file. - - retval = LS_MAT_ASCII; - } } } } - file.close (); + return retval; +} +static load_save_format +get_file_format (const std::string& fname, const std::string& orig_fname, + bool &use_zlib) +{ + load_save_format retval = LS_UNKNOWN; + +#ifdef HAVE_HDF5 + // check this before we open the file + if (H5Fis_hdf5 (fname.c_str ()) > 0) + return LS_HDF5; +#endif /* HAVE_HDF5 */ - if (retval == LS_UNKNOWN) - error ("load: unable to determine file format for `%s'", - orig_fname.c_str ()); + std::ifstream file (fname.c_str ()); + use_zlib = false; + + if (file) + { + retval = get_file_format (file); + file.close (); +#if HAVE_ZLIB + if (retval == LS_UNKNOWN && check_gzip_magic (fname)) + { + gzifstream gzfile (fname.c_str ()); + use_zlib = true; + + if (gzfile) + { + retval = get_file_format (gzfile); + gzfile.close (); + } + } + + if (retval == LS_UNKNOWN) + { + // Try reading the file as numbers only, determining the + // number of rows and columns from the data. We don't + // even bother to check to see if the first item in the + // file is a number, so that get_complete_line() can + // skip any comments that might appear at the top of the + // file. + + retval = LS_MAT_ASCII; + } + +#endif + } + else + error ("load: couldn't open input file `%s'", orig_fname.c_str ()); return retval; } @@ -430,6 +471,7 @@ #endif /* HAVE_HDF5 */ case LS_MAT5_BINARY: + case LS_MAT7_BINARY: name = read_mat5_binary_element (stream, orig_fname, swap, global, tc); break; @@ -588,7 +630,11 @@ \n\ @item -mat\n\ @itemx -mat-binary\n\ -Force Octave to assume the file is in @sc{Matlab}'s version 6 binary\n\ +@itemx -6\n\ +@itemx -v6\n\ +@itemx -7\n\ +@itemx -v7\n\ +Force Octave to assume the file is in @sc{Matlab}'s version 6 or 7 binary\n\ format.\n\ \n\ @item -V4\n\ @@ -666,10 +712,15 @@ { format = LS_BINARY; } - else if (argv[i] == "-mat-binary" || argv[i] == "-mat" || argv[i] == "-m") + else if (argv[i] == "-mat-binary" || argv[i] == "-mat" || argv[i] == "-m" + || argv[i] == "-6" || argv[i] == "-v6") { format = LS_MAT5_BINARY; } + else if (argv[i] == "7" || argv[i] == "-v7") + { + format = LS_MAT7_BINARY; + } else if (argv[i] == "-mat4-binary" || argv[i] == "-V4" || argv[i] == "-v4" || argv[i] == "-4") { @@ -734,6 +785,7 @@ else { std::string fname = file_ops::tilde_expand (argv[i]); + bool use_zlib = false; // Check if file exists, if it doesn't then also check with a // .mat extension @@ -754,7 +806,7 @@ } if (format == LS_UNKNOWN) - format = get_file_format (fname, orig_fname); + format = get_file_format (fname, orig_fname, use_zlib); #ifdef HAVE_HDF5 if (format == LS_HDF5) @@ -790,41 +842,83 @@ || format == LS_HDF5 #endif || format == LS_MAT_BINARY - || format == LS_MAT5_BINARY) + || format == LS_MAT5_BINARY + || format == LS_MAT7_BINARY) mode |= std::ios::binary; - std::ifstream file (fname.c_str (), mode); +#ifdef HAVE_ZLIB + if (use_zlib) + { + gzifstream file (fname.c_str (), mode); - if (file) - { - if (format == LS_BINARY) + if (file) { - if (read_binary_file_header (file, swap, flt_fmt) < 0) + if (format == LS_BINARY) + { + if (read_binary_file_header (file, swap, flt_fmt) < 0) + { + if (file) file.close (); + return retval; + } + } + else if (format == LS_MAT5_BINARY + || format == LS_MAT7_BINARY) { - file.close (); - return retval; + if (read_mat5_binary_file_header (file, swap, false) < 0) + { + if (file) file.close (); + return retval; + } } + + retval = do_load (file, orig_fname, force, format, + flt_fmt, list_only, swap, verbose, + argv, i, argc, nargout); + + file.close (); } - else if (format == LS_MAT5_BINARY) - { - if (read_mat5_binary_file_header (file, swap, false) < 0) - { - file.close (); - return retval; - } - } - - retval = do_load (file, orig_fname, force, format, - flt_fmt, list_only, swap, verbose, - argv, i, argc, nargout); - file.close (); + else + error ("load: couldn't open input file `%s'", + orig_fname.c_str ()); } else - error ("load: couldn't open input file `%s'", - orig_fname.c_str ()); +#endif + { + std::ifstream file (fname.c_str (), mode); + + if (file) + { + if (format == LS_BINARY) + { + if (read_binary_file_header (file, swap, flt_fmt) < 0) + { + if (file) file.close (); + return retval; + } + } + else if (format == LS_MAT5_BINARY + || format == LS_MAT7_BINARY) + { + if (read_mat5_binary_file_header (file, swap, false) < 0) + { + if (file) file.close (); + return retval; + } + } + + retval = do_load (file, orig_fname, force, format, + flt_fmt, list_only, swap, verbose, + argv, i, argc, nargout); + + file.close (); + } + else + error ("load: couldn't open input file `%s'", + orig_fname.c_str ()); + } } } - + return retval; } @@ -895,7 +989,11 @@ #endif /* HAVE_HDF5 */ case LS_MAT5_BINARY: - save_mat5_binary_element (os, tc, name, global, save_as_floats); + save_mat5_binary_element (os, tc, name, global, false, save_as_floats); + break; + + case LS_MAT7_BINARY: + save_mat5_binary_element (os, tc, name, global, true, save_as_floats); break; default: @@ -984,6 +1082,8 @@ retval = LS_BINARY; else if (fmt == "mat-binary" || fmt =="mat_binary") retval = LS_MAT5_BINARY; + else if (fmt == "mat7-binary" || fmt =="mat7_binary") + retval = LS_MAT7_BINARY; else if (fmt == "mat4-binary" || fmt =="mat4_binary") retval = LS_MAT_BINARY; #ifdef HAVE_HDF5 @@ -1014,6 +1114,7 @@ break; case LS_MAT5_BINARY: + case LS_MAT7_BINARY: { char const * versionmagic; TWO_BYTE_INT number = *(TWO_BYTE_INT *)"\x00\x01"; @@ -1172,7 +1273,8 @@ || format == LS_HDF5 #endif || format == LS_MAT_BINARY - || format == LS_MAT5_BINARY) + || format == LS_MAT5_BINARY + || format == LS_MAT7_BINARY) mode |= std::ios::binary; #ifdef HAVE_HDF5 @@ -1208,6 +1310,14 @@ } } +#ifdef HAVE_ZLIB +#define HAVE_ZLIB_HELP_STRING "" +#else /* ! HAVE_ZLIB */ +#define HAVE_ZLIB_HELP_STRING "\n\ +This option is not available, as this Octave executable was not linked with\n\ +the zlib library." +#endif /* ! HAVE ZLIB */ + DEFCMD (save, args, , "-*- texinfo -*-\n\ @deffn {Command} save options file v1 v2 @dots{}\n\ @@ -1236,9 +1346,21 @@ precision. You should use this format only if you know that all the\n\ values to be saved can be represented in single precision.\n\ \n\ -@item -mat\n\ +@item -V7\n\ +@itemx -v7\n\ +@itemx -7\n\ +@itemx -mat7-binary\n\ +Save the data in @sc{Matlab}'s v7 binary data format.\n" + +HAVE_ZLIB_HELP_STRING + +"\n\ +@item -V6\n\ +@item -v6\n\ +@itemx -6\n\ +@itemx -mat\n\ @itemx -mat-binary\n\ -Save the data in @sc{Matlab}'s binary data format.\n\ +Save the data in @sc{Matlab}'s v6 binary data format.\n\ \n\ @item -V4\n\ @itemx -v4\n\ @@ -1262,7 +1384,16 @@ @item -save-builtins\n\ Force Octave to save the values of built-in variables too. By default,\n\ Octave does not save built-in variables.\n\ -@end table\n\ +\n\ +@item -zip\n\ +@itemx -z\n\ +Use the gzip algorithm to compress the file. This works equally on files that\n\ +are compressed with gzip outside of octave, and gzip can equally be used to\n\ +convert the files for backward compatibility" + +HAVE_ZLIB_HELP_STRING + +"@end table\n\ \n\ The list of variables to save may include wildcard patterns containing\n\ the following special characters:\n\ @@ -1319,6 +1450,8 @@ bool append = false; + bool use_zlib = false; + int i; for (i = 1; i < argc; i++) { @@ -1352,10 +1485,19 @@ return retval; #endif /* ! HAVE_HDF5 */ } - else if (argv[i] == "-mat-binary" || argv[i] == "-mat" || argv[i] == "-m") + else if (argv[i] == "-mat-binary" || argv[i] == "-mat" + || argv[i] == "-m" || argv[i] == "-6" || argv[i] == "-v6" + || argv[i] == "-V6") { format = LS_MAT5_BINARY; } +#ifdef HAVE_ZLIB + else if (argv[i] == "-mat7-binary" || argv[i] == "-7" + || argv[i] == "-v7" || argv[i] == "-V7") + { + format = LS_MAT7_BINARY; + } +#endif else if (argv[i] == "-mat4-binary" || argv[i] == "-V4" || argv[i] == "-v4" || argv[i] == "-4") { @@ -1380,6 +1522,12 @@ { save_builtins = true; } +#ifdef HAVE_ZLIB + else if (argv[i] == "-zip" || argv[i] == "-z") + { + use_zlib = true; + } +#endif else break; } @@ -1436,7 +1584,8 @@ || format == LS_HDF5 #endif || format == LS_MAT_BINARY - || format == LS_MAT5_BINARY) + || format == LS_MAT5_BINARY + || format == LS_MAT7_BINARY) mode |= std::ios::binary; mode |= append ? std::ios::ate : std::ios::trunc; @@ -1464,21 +1613,51 @@ // don't insert any statements here! The brace below must go // with the "else" above! { - std::ofstream file (fname.c_str (), mode); - - if (file) +#ifdef HAVE_ZLIB + if (use_zlib) { - bool write_header_info - = ((file.rdbuf ())->pubseekoff (0, std::ios::cur) - == static_cast<std::streampos> (0)); + gzofstream file (fname.c_str (), mode); + + if (file) + { + bool write_header_info + = ((file.rdbuf ())->pubseekoff (0, std::ios::cur) + == static_cast<std::streampos> (0)); - save_vars (argv, i, argc, file, save_builtins, format, - save_as_floats, write_header_info); + save_vars (argv, i, argc, file, save_builtins, format, + save_as_floats, write_header_info); + + file.close (); + } + else + { + error ("save: couldn't open output file `%s'", + fname.c_str ()); + return retval; + } } else +#endif { - error ("save: couldn't open output file `%s'", fname.c_str ()); - return retval; + std::ofstream file (fname.c_str (), mode); + + if (file) + { + bool write_header_info + = ((file.rdbuf ())->pubseekoff (0, std::ios::cur) + == static_cast<std::streampos> (0)); + + save_vars (argv, i, argc, file, save_builtins, format, + save_as_floats, write_header_info); + + file.close (); + } + else + { + error ("save: couldn't open output file `%s'", + fname.c_str ()); + return retval; + } } } }
--- a/src/load-save.h +++ b/src/load-save.h @@ -36,6 +36,7 @@ LS_MAT_ASCII, LS_MAT_BINARY, LS_MAT5_BINARY, + LS_MAT7_BINARY, #ifdef HAVE_HDF5 LS_HDF5, #endif /* HAVE_HDF5 */
--- a/src/ls-mat5.cc +++ b/src/ls-mat5.cc @@ -69,6 +69,10 @@ #include "ls-utils.h" #include "ls-mat5.h" +#ifdef HAVE_ZLIB +#include <zlib.h> +#endif + #define PAD(l) (((l)<=4)?4:(((l)+7)/8)*8) #define TAGLENGTH(l) ((l)<=4?4:8) @@ -418,6 +422,48 @@ if (read_mat5_tag (is, swap, type, element_length)) return retval; // EOF +#if HAVE_ZLIB + if (type == miCOMPRESSED) + { + // If C++ allowed us direct access to the file descriptor of an ifstream + // in a uniform way, the code below could be vastly simplified, and + // additional copies of the data in memory wouldn't be needed!! + + OCTAVE_LOCAL_BUFFER (char, inbuf, element_length); + is.read (inbuf, element_length); + + // We uncompress the first 8 bytes of the header to get the buffer length + // This will fail with an error Z_MEM_ERROR + uLongf destLen = 8; + OCTAVE_LOCAL_BUFFER (unsigned int, tmp, 2); + if (uncompress (X_CAST (Bytef *, tmp), &destLen, + X_CAST (Bytef *, inbuf), element_length) != Z_MEM_ERROR) + { + // Why should I have to initialize outbuf as I'll just overwrite!! + destLen = tmp[1] + 8; + std::string outbuf (destLen, ' '); + + int err = uncompress (X_CAST (Bytef *, outbuf.c_str ()), &destLen, + X_CAST ( Bytef *, inbuf), element_length); + //if (uncompress (X_CAST (Bytef *, outbuf.c_str ()), &destLen, + // X_CAST ( Bytef *, inbuf), element_length) != Z_OK) + + if (err != Z_OK) + error ("load: error uncompressing data element"); + else + { + ISSTREAM gz_is (outbuf); + retval = read_mat5_binary_element (gz_is, filename, + swap, global, tc); + } + } + else + error ("load: error probing size of compressed data element"); + + return retval; + } +#endif + if (type != miMATRIX) { error ("load: invalid element type"); @@ -442,7 +488,7 @@ read_int (is, swap, flags); imag = (flags & 0x0800) != 0; // has an imaginary part? global = (flags & 0x0400) != 0; // global variable? - logicalvar = (flags & 0x0200) != 0; // we don't use this yet + logicalvar = (flags & 0x0200) != 0; // boolean ? arrayclass = (arrayclasstype)(flags & 0xff); read_int (is, swap, nnz); // number of non-zero in sparse @@ -737,7 +783,24 @@ break; case mxUINT8_CLASS: - OCTAVE_MAT5_INTEGER_READ (uint8NDArray); + { + OCTAVE_MAT5_INTEGER_READ (uint8NDArray); + + // logical variables can either be mxUINT8_CLASS or mxDOUBLE_CLASS, + // so chek if we have a logical variable and convert it + + if (logicalvar) + { + uint8NDArray in = tc.uint8_array_value (); + int nel = in.nelem (); + boolNDArray out (dims); + + for (int i = 0; i < nel; i++) + out (i) = static_cast<bool> (double (in (i))); + + tc = out; + } + } break; case mxINT16_CLASS: @@ -796,9 +859,22 @@ is.seekg (tmp_pos + static_cast<std::streamoff> (PAD (len))); - // imaginary data subelement - if (imag) + if (logicalvar) { + // logical variables can either be mxUINT8_CLASS or mxDOUBLE_CLASS, + // so chek if we have a logical variable and convert it + + boolNDArray out (dims); + + for (int i = 0; i < n; i++) + out (i) = static_cast<bool> (re (i)); + + tc = out; + } + else if (imag) + { + // imaginary data subelement + NDArray im (dims); if (read_mat5_tag (is, swap, type, len)) @@ -826,10 +902,12 @@ tc = ctmp; } else - tc = re; + { + tc = re; - if (arrayclass == mxCHAR_CLASS) - tc = tc.convert_to_str (false, true); + if (arrayclass == mxCHAR_CLASS) + tc = tc.convert_to_str (false, true); + } } } @@ -1108,33 +1186,287 @@ octave_value ov = cell(i); if (! save_mat5_binary_element (os, ov, "", mark_as_global, - save_as_floats)) + false, save_as_floats)) return false; } return true; } +int +save_mat5_array_length (const double* val, int nel, bool save_as_floats) +{ + if (nel > 0) + { + int size = 8; + + if (save_as_floats) + { + bool too_large_for_float = false; + for (int i = 0; i < nel; i++) + { + double tmp = val [i]; + + if (tmp > FLT_MAX || tmp < FLT_MIN) + { + too_large_for_float = true; + break; + } + } + + if (!too_large_for_float) + size = 4; + } + + // The code below is disabled since get_save_type currently doesn't + // deal with integer types. This will need to be activated if get_save_type + // is changed. + + // double max_val = val[0]; + // double min_val = val[0]; + // bool all_integers = true; + // + // for (int i = 0; i < nel; i++) + // { + // double val = val[i]; + // + // if (val > max_val) + // max_val = val; + // + // if (val < min_val) + // min_val = val; + // + // if (D_NINT (val) != val) + // { + // all_integers = false; + // break; + // } + // } + // + // if (all_integers) + // { + // if (max_val < 256 && min_val > -1) + // size = 1; + // else if (max_val < 65536 && min_val > -1) + // size = 2; + // else if (max_val < 4294967295UL && min_val > -1) + // size = 4; + // else if (max_val < 128 && min_val >= -128) + // size = 1; + // else if (max_val < 32768 && min_val >= -32768) + // size = 2; + // else if (max_val <= 2147483647L && min_val >= -2147483647L) + // size = 4; + // } + + return 8 + nel * size; + } + else + return 8; +} + +int +save_mat5_array_length (const Complex* val, int nel, bool save_as_floats) +{ + int ret; + + OCTAVE_LOCAL_BUFFER (double, tmp, nel); + + for (int i = 1; i < nel; i++) + tmp[i] = std::real (val[i]); + + ret = save_mat5_array_length (tmp, nel, save_as_floats); + + for (int i = 1; i < nel; i++) + tmp[i] = std::imag (val[i]); + + ret += save_mat5_array_length (tmp, nel, save_as_floats); + + return ret; +} + +int +save_mat5_element_length (const octave_value& tc, const std::string& name, + bool save_as_floats, bool mat7_format) +{ + int max_namelen = (mat7_format ? 63 : 31); + int len = name.length (); + std::string cname = tc.class_name (); + int ret = 32; + + if (len > 4) + ret += PAD (len > max_namelen ? max_namelen : len); + + ret += PAD (4 * tc.ndims ()); + + if (tc.is_string ()) + { + charMatrix chm = tc.char_matrix_value (); + ret += 8 + PAD (2 * chm.rows () * chm.cols ()); + } + else if (cname == "sparse") + { + if (tc.is_complex_type ()) + { + SparseComplexMatrix m = tc.sparse_complex_matrix_value (); + int nc = m.cols (); + int nnz = m.nnz (); + + ret += 16 + PAD (nnz * sizeof (int)) + PAD ((nc + 1) * sizeof (int)) + + save_mat5_array_length (m.data (), m.nelem (), save_as_floats); + } + else + { + SparseMatrix m = tc.sparse_matrix_value (); + int nc = m.cols (); + int nnz = m.nnz (); + + ret += 16 + PAD (nnz * sizeof (int)) + PAD ((nc + 1) * sizeof (int)) + + save_mat5_array_length (m.data (), m.nelem (), save_as_floats); + } + } + +#define INT_LEN(nel, size) \ + { \ + ret += 8; \ + int sz = nel * size; \ + if (sz > 4) \ + ret += PAD (sz); \ + } + + else if (cname == "int8") + INT_LEN (tc.int8_array_value ().nelem (), 1) + else if (cname == "int16") + INT_LEN (tc.int16_array_value ().nelem (), 2) + else if (cname == "int32") + INT_LEN (tc.int32_array_value ().nelem (), 4) + else if (cname == "int64") + INT_LEN (tc.int64_array_value ().nelem (), 8) + else if (cname == "uint8") + INT_LEN (tc.uint8_array_value ().nelem (), 1) + else if (cname == "uint16") + INT_LEN (tc.uint16_array_value ().nelem (), 2) + else if (cname == "uint32") + INT_LEN (tc.uint32_array_value ().nelem (), 4) + else if (cname == "uint64") + INT_LEN (tc.uint64_array_value ().nelem (), 8) + else if (tc.is_bool_type ()) + INT_LEN (tc.bool_array_value ().nelem (), 1) + else if (tc.is_real_scalar () || tc.is_real_matrix () || tc.is_range ()) + { + NDArray m = tc.array_value (); + ret += save_mat5_array_length (m.fortran_vec (), m.nelem (), + save_as_floats); + } + else if (tc.is_cell ()) + { + Cell cell = tc.cell_value (); + int nel = cell.nelem (); + + for (int i = 0; i < nel; i++) + ret += 8 + + save_mat5_element_length (cell (i), "", save_as_floats, mat7_format); + } + else if (tc.is_complex_scalar () || tc.is_complex_matrix ()) + { + ComplexNDArray m = tc.complex_array_value (); + ret += save_mat5_array_length (m.fortran_vec (), m.nelem (), + save_as_floats); + } + else if (tc.is_map ()) + { + int fieldcnt = 0; + const Octave_map m = tc.map_value (); + int nel = m.numel (); + + for (Octave_map::const_iterator i = m.begin (); i != m.end (); i++) + fieldcnt++; + + ret += 16 + fieldcnt * (max_namelen + 1); + + + for (int j = 0; j < nel; j++) + { + + for (Octave_map::const_iterator i = m.begin (); i != m.end (); i++) + { + Cell elts = m.contents (i); + + ret += 8 + save_mat5_element_length (elts (j), "", + save_as_floats, mat7_format); + } + } + } + else + ret = -1; + + return ret; +} + // save the data from TC along with the corresponding NAME on stream // OS in the MatLab version 5 binary format. Return true on success. bool save_mat5_binary_element (std::ostream& os, const octave_value& tc, const std::string& name, - bool mark_as_global, bool save_as_floats) + bool mark_as_global, bool mat7_format, + bool save_as_floats, bool compressing) { FOUR_BYTE_INT flags=0; FOUR_BYTE_INT nnz=0; std::streampos fixup, contin; std::string cname = tc.class_name (); + int max_namelen = (mat7_format ? 63 : 31); + +#ifdef HAVE_ZLIB + if (mat7_format && !compressing) + { + bool ret = false; + + OSSTREAM buf; + + // The code seeks backwards in the stream to fix the header. Can't + // do this with zlib, so use a stringstream. + ret = save_mat5_binary_element (buf, tc, name, mark_as_global, true, + save_as_floats, true); + + if (ret) + { + OSSTREAM_FREEZE (buf); + + // destLen must be 0.1% larger than source buffer + 12 bytes + uLongf srcLen = OSSTREAM_STR (buf).length (); + uLongf destLen = srcLen * 1001 / 1000 + 12; + OCTAVE_LOCAL_BUFFER (char, out_buf, destLen); + + if (compress (X_CAST (Bytef *, out_buf), &destLen, + X_CAST (Bytef *, OSSTREAM_C_STR (buf)), srcLen) == Z_OK) + { + write_mat5_tag (os, miCOMPRESSED, X_CAST(int, destLen)); + os.write (out_buf, destLen); + } + else + { + error ("save: error compressing data element"); + ret = false; + } + } + + return ret; + } +#endif // element type and length fixup = os.tellp (); - write_mat5_tag (os, miMATRIX, 99); // we don't know the real length yet + write_mat5_tag (os, miMATRIX, save_mat5_element_length + (tc, name, save_as_floats, mat7_format)); // array flags subelement write_mat5_tag (os, miUINT32, 8); + if (tc.is_bool_type ()) + flags |= 0x0200; + if (mark_as_global) flags |= 0x0400; @@ -1151,7 +1483,7 @@ flags |= mxINT32_CLASS; else if (cname == "int64") flags |= mxINT64_CLASS; - else if (cname == "uint8") + else if (cname == "uint8" || tc.is_bool_type ()) flags |= mxUINT8_CLASS; else if (cname == "uint16") flags |= mxUINT16_CLASS; @@ -1218,8 +1550,8 @@ { int namelen = name.length (); - if (namelen > 31) - namelen = 31; // only 31 char names permitted in mat file + if (namelen > max_namelen) + namelen = max_namelen; // only 31 or 63 char names permitted in mat file int paddedlength = PAD (namelen); @@ -1343,6 +1675,12 @@ write_mat5_integer_data (os, m.fortran_vec (), 8, m.nelem ()); } + else if (tc.is_bool_type ()) + { + uint8NDArray m (tc.bool_array_value ()); + + write_mat5_integer_data (os, m.fortran_vec (), 1, m.nelem ()); + } else if (tc.is_real_scalar () || tc.is_real_matrix () || tc.is_range ()) { NDArray m = tc.array_value (); @@ -1358,7 +1696,7 @@ } else if (tc.is_complex_scalar () || tc.is_complex_matrix ()) { - ComplexNDArray m_cmplx = tc.complex_matrix_value (); + ComplexNDArray m_cmplx = tc.complex_array_value (); write_mat5_array (os, ::real (m_cmplx), save_as_floats); write_mat5_array (os, ::imag (m_cmplx), save_as_floats); @@ -1370,8 +1708,8 @@ const Octave_map m = tc.map_value (); { - char buf[32]; - FOUR_BYTE_INT maxfieldnamelength = 32; + char buf[64]; + FOUR_BYTE_INT maxfieldnamelength = max_namelen + 1; int fieldcnt = 0; for (Octave_map::const_iterator i = m.begin (); i != m.end (); i++) @@ -1379,15 +1717,15 @@ write_mat5_tag (os, miINT32, 4); os.write ((char *)&maxfieldnamelength, 4); - write_mat5_tag (os, miINT8, fieldcnt*32); + write_mat5_tag (os, miINT8, fieldcnt*maxfieldnamelength); for (Octave_map::const_iterator i = m.begin (); i != m.end (); i++) { // write the name of each element std::string tstr = m.key (i); - memset (buf, 0, 32); - strncpy (buf, tstr.c_str (), 31); // only 31 char names permitted - os.write (buf, 32); + memset (buf, 0, max_namelen + 1); + strncpy (buf, tstr.c_str (), max_namelen); // only 31 or 63 char names permitted + os.write (buf, max_namelen + 1); } int len = m.numel (); @@ -1402,6 +1740,7 @@ bool retval2 = save_mat5_binary_element (os, elts(j), "", mark_as_global, + false, save_as_floats); if (! retval2) goto error_cleanup; @@ -1413,10 +1752,6 @@ gripe_wrong_type_arg ("save", tc, false); contin = os.tellp (); - os.seekp (fixup); - write_mat5_tag (os, miMATRIX, - static_cast<int>(contin - fixup) - 8); // the actual length - os.seekp (contin); return true;
--- a/src/ls-mat5.h +++ b/src/ls-mat5.h @@ -38,7 +38,11 @@ miRESERVE3, miINT64, // 64 bit signed miUINT64, // 64 bit unsigned - miMATRIX // MATLAB array + miMATRIX, // MATLAB array + miCOMPRESSED, // Compressed data + miUTF8, // Unicode UTF-8 Encoded Character Data + miUTF16, // Unicode UTF-16 Encoded Character Data + miUTF32, // Unicode UTF-32 Encoded Character Data }; extern int @@ -50,7 +54,8 @@ extern bool save_mat5_binary_element (std::ostream& os, const octave_value& tc, const std::string& name, - bool mark_as_global, bool save_as_floats); + bool mark_as_global, bool mat7_format, + bool save_as_floats, bool compressing = false); #endif
new file mode 100644 --- /dev/null +++ b/src/zfstream.cc @@ -0,0 +1,569 @@ +/* + +Copyright (C) 2005 Ludwig Schwardt, Kevin Ruland + + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; either version 2, or (at your option) any +later version. + +Octave is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, write to the Free +Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +*/ + +/* + + This file is adapted from the zlib 1.2.2 contrib/iostream3 code, + written by + + Ludwig Schwardt <schwardt@sun.ac.za> + original version by Kevin Ruland <kevin@rodin.wustl.edu> + +*/ + +#include "zfstream.h" + +#ifdef HAVE_ZLIB + +#include <cstring> // for strcpy, strcat, strlen (mode strings) +#include <cstdio> // for BUFSIZ + +// Internal buffer sizes (default and "unbuffered" versions) +#define BIGBUFSIZE BUFSIZ +#define SMALLBUFSIZE 1 + +/*****************************************************************************/ + +// Default constructor +gzfilebuf::gzfilebuf() +: file(NULL), io_mode(std::ios_base::openmode(0)), own_fd(false), + buffer(NULL), buffer_size(BIGBUFSIZE), own_buffer(true) +{ + // No buffers to start with + this->disable_buffer(); +} + +// Destructor +gzfilebuf::~gzfilebuf() +{ + // Sync output buffer and close only if responsible for file + // (i.e. attached streams should be left open at this stage) + this->sync(); + if (own_fd) + this->close(); + // Make sure internal buffer is deallocated + this->disable_buffer(); +} + +// Set compression level and strategy +int +gzfilebuf::setcompression(int comp_level, + int comp_strategy) +{ + return gzsetparams(file, comp_level, comp_strategy); +} + +// Open gzipped file +gzfilebuf* +gzfilebuf::open(const char *name, + std::ios_base::openmode mode) +{ + // Fail if file already open + if (this->is_open()) + return NULL; + // Don't support simultaneous read/write access (yet) + if ((mode & std::ios_base::in) && (mode & std::ios_base::out)) + return NULL; + + // Build mode string for gzopen and check it [27.8.1.3.2] + char char_mode[6] = "\0\0\0\0\0"; + if (!this->open_mode(mode, char_mode)) + return NULL; + + // Attempt to open file + if ((file = gzopen(name, char_mode)) == NULL) + return NULL; + + // On success, allocate internal buffer and set flags + this->enable_buffer(); + io_mode = mode; + own_fd = true; + return this; +} + +// Attach to gzipped file +gzfilebuf* +gzfilebuf::attach(int fd, + std::ios_base::openmode mode) +{ + // Fail if file already open + if (this->is_open()) + return NULL; + // Don't support simultaneous read/write access (yet) + if ((mode & std::ios_base::in) && (mode & std::ios_base::out)) + return NULL; + + // Build mode string for gzdopen and check it [27.8.1.3.2] + char char_mode[6] = "\0\0\0\0\0"; + if (!this->open_mode(mode, char_mode)) + return NULL; + + // Attempt to attach to file + if ((file = gzdopen(fd, char_mode)) == NULL) + return NULL; + + // On success, allocate internal buffer and set flags + this->enable_buffer(); + io_mode = mode; + own_fd = false; + return this; +} + +// Close gzipped file +gzfilebuf* +gzfilebuf::close() +{ + // Fail immediately if no file is open + if (!this->is_open()) + return NULL; + // Assume success + gzfilebuf* retval = this; + // Attempt to sync and close gzipped file + if (this->sync() == -1) + retval = NULL; + if (gzclose(file) < 0) + retval = NULL; + // File is now gone anyway (postcondition [27.8.1.3.8]) + file = NULL; + own_fd = false; + // Destroy internal buffer if it exists + this->disable_buffer(); + return retval; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +// Convert int open mode to mode string +bool +gzfilebuf::open_mode(std::ios_base::openmode mode, + char* c_mode) const +{ + bool testb = mode & std::ios_base::binary; + bool testi = mode & std::ios_base::in; + bool testo = mode & std::ios_base::out; + bool testt = mode & std::ios_base::trunc; + bool testa = mode & std::ios_base::app; + + // Check for valid flag combinations - see [27.8.1.3.2] (Table 92) + // Original zfstream hardcoded the compression level to maximum here... + // Double the time for less than 1% size improvement seems + // excessive though - keeping it at the default level + // To change back, just append "9" to the next three mode strings + if (!testi && testo && !testt && !testa) + strcpy(c_mode, "w"); + if (!testi && testo && !testt && testa) + strcpy(c_mode, "a"); + if (!testi && testo && testt && !testa) + strcpy(c_mode, "w"); + if (testi && !testo && !testt && !testa) + strcpy(c_mode, "r"); + // No read/write mode yet +// if (testi && testo && !testt && !testa) +// strcpy(c_mode, "r+"); +// if (testi && testo && testt && !testa) +// strcpy(c_mode, "w+"); + + // Mode string should be empty for invalid combination of flags + if (strlen(c_mode) == 0) + return false; + if (testb) + strcat(c_mode, "b"); + return true; +} + +// Determine number of characters in internal get buffer +std::streamsize +gzfilebuf::showmanyc() +{ + // Calls to underflow will fail if file not opened for reading + if (!this->is_open() || !(io_mode & std::ios_base::in)) + return -1; + // Make sure get area is in use + if (this->gptr() && (this->gptr() < this->egptr())) + return std::streamsize(this->egptr() - this->gptr()); + else + return 0; +} + +// Fill get area from gzipped file +gzfilebuf::int_type +gzfilebuf::underflow() +{ + // If something is left in the get area by chance, return it + // (this shouldn't normally happen, as underflow is only supposed + // to be called when gptr >= egptr, but it serves as error check) + if (this->gptr() && (this->gptr() < this->egptr())) + return traits_type::to_int_type(*(this->gptr())); + + // If the file hasn't been opened for reading, produce error + if (!this->is_open() || !(io_mode & std::ios_base::in)) + return traits_type::eof(); + + // Attempt to fill internal buffer from gzipped file + // (buffer must be guaranteed to exist...) + int bytes_read = gzread(file, buffer, buffer_size); + // Indicates error or EOF + if (bytes_read <= 0) + { + // Reset get area + this->setg(buffer, buffer, buffer); + return traits_type::eof(); + } + // Make all bytes read from file available as get area + this->setg(buffer, buffer, buffer + bytes_read); + + // Return next character in get area + return traits_type::to_int_type(*(this->gptr())); +} + +// Write put area to gzipped file +gzfilebuf::int_type +gzfilebuf::overflow(int_type c) +{ + // Determine whether put area is in use + if (this->pbase()) + { + // Double-check pointer range + if (this->pptr() > this->epptr() || this->pptr() < this->pbase()) + return traits_type::eof(); + // Add extra character to buffer if not EOF + if (!traits_type::eq_int_type(c, traits_type::eof())) + { + *(this->pptr()) = traits_type::to_char_type(c); + this->pbump(1); + } + // Number of characters to write to file + int bytes_to_write = this->pptr() - this->pbase(); + // Overflow doesn't fail if nothing is to be written + if (bytes_to_write > 0) + { + // If the file hasn't been opened for writing, produce error + if (!this->is_open() || !(io_mode & std::ios_base::out)) + return traits_type::eof(); + // If gzipped file won't accept all bytes written to it, fail + if (gzwrite(file, this->pbase(), bytes_to_write) != bytes_to_write) + return traits_type::eof(); + // Reset next pointer to point to pbase on success + this->pbump(-bytes_to_write); + } + } + // Write extra character to file if not EOF + else if (!traits_type::eq_int_type(c, traits_type::eof())) + { + // If the file hasn't been opened for writing, produce error + if (!this->is_open() || !(io_mode & std::ios_base::out)) + return traits_type::eof(); + // Impromptu char buffer (allows "unbuffered" output) + char_type last_char = traits_type::to_char_type(c); + // If gzipped file won't accept this character, fail + if (gzwrite(file, &last_char, 1) != 1) + return traits_type::eof(); + } + + // If you got here, you have succeeded (even if c was EOF) + // The return value should therefore be non-EOF + if (traits_type::eq_int_type(c, traits_type::eof())) + return traits_type::not_eof(c); + else + return c; +} + +// Assign new buffer +std::streambuf* +gzfilebuf::setbuf(char_type* p, + std::streamsize n) +{ + // First make sure stuff is sync'ed, for safety + if (this->sync() == -1) + return NULL; + // If buffering is turned off on purpose via setbuf(0,0), still allocate one... + // "Unbuffered" only really refers to put [27.8.1.4.10], while get needs at + // least a buffer of size 1 (very inefficient though, therefore make it bigger?) + // This follows from [27.5.2.4.3]/12 (gptr needs to point at something, it seems) + if (!p || !n) + { + // Replace existing buffer (if any) with small internal buffer + this->disable_buffer(); + buffer = NULL; + buffer_size = 0; + own_buffer = true; + this->enable_buffer(); + } + else + { + // Replace existing buffer (if any) with external buffer + this->disable_buffer(); + buffer = p; + buffer_size = n; + own_buffer = false; + this->enable_buffer(); + } + return this; +} + +// Write put area to gzipped file (i.e. ensures that put area is empty) +int +gzfilebuf::sync() +{ + return traits_type::eq_int_type(this->overflow(), traits_type::eof()) ? -1 : 0; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +// Allocate internal buffer +void +gzfilebuf::enable_buffer() +{ + // If internal buffer required, allocate one + if (own_buffer && !buffer) + { + // Check for buffered vs. "unbuffered" + if (buffer_size > 0) + { + // Allocate internal buffer + buffer = new char_type[buffer_size]; + // Get area starts empty and will be expanded by underflow as need arises + this->setg(buffer, buffer, buffer); + // Setup entire internal buffer as put area. + // The one-past-end pointer actually points to the last element of the buffer, + // so that overflow(c) can safely add the extra character c to the sequence. + // These pointers remain in place for the duration of the buffer + this->setp(buffer, buffer + buffer_size - 1); + } + else + { + // Even in "unbuffered" case, (small?) get buffer is still required + buffer_size = SMALLBUFSIZE; + buffer = new char_type[buffer_size]; + this->setg(buffer, buffer, buffer); + // "Unbuffered" means no put buffer + this->setp(0, 0); + } + } + else + { + // If buffer already allocated, reset buffer pointers just to make sure no + // stale chars are lying around + this->setg(buffer, buffer, buffer); + this->setp(buffer, buffer + buffer_size - 1); + } +} + +// Destroy internal buffer +void +gzfilebuf::disable_buffer() +{ + // If internal buffer exists, deallocate it + if (own_buffer && buffer) + { + // Preserve unbuffered status by zeroing size + if (!this->pbase()) + buffer_size = 0; + delete[] buffer; + buffer = NULL; + this->setg(0, 0, 0); + this->setp(0, 0); + } + else + { + // Reset buffer pointers to initial state if external buffer exists + this->setg(buffer, buffer, buffer); + if (buffer) + this->setp(buffer, buffer + buffer_size - 1); + else + this->setp(0, 0); + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +// Seek functions +gzfilebuf::pos_type +gzfilebuf::seekoff(off_type off, std::ios_base::seekdir way, + std::ios_base::openmode) +{ + pos_type ret = pos_type (off_type (-1)); + + if (this->is_open()) + { + off_type computed_off = off; + + if ((io_mode & std::ios_base::in) && way == std::ios_base::cur) + computed_off += this->gptr() - this->egptr(); + + if (way == std::ios_base::beg) + ret = pos_type (gzseek (file, computed_off, SEEK_SET)); + else if (way == std::ios_base::cur) + ret = pos_type (gzseek (file, computed_off, SEEK_CUR)); + else + // Can't seek from end of a gzipped file, so this will give -1 + ret = pos_type (gzseek (file, computed_off, SEEK_END)); + + if (io_mode & std::ios_base::in) + // Invalidates contents of the buffer + enable_buffer (); + else + // flush contents of buffer to file + overflow (); + } + + return ret; +} + +gzfilebuf::pos_type +gzfilebuf::seekpos(pos_type sp, std::ios_base::openmode) +{ + pos_type ret = pos_type (off_type (-1)); + + if (this->is_open ()) + { + ret = pos_type (gzseek (file, sp, SEEK_SET)); + + if (io_mode & std::ios_base::in) + // Invalidates contents of the buffer + enable_buffer (); + else + // flush contents of buffer to file + overflow (); + } + + return ret; +} + +/*****************************************************************************/ + +// Default constructor initializes stream buffer +gzifstream::gzifstream() +: std::istream(NULL), sb() +{ this->init(&sb); } + +// Initialize stream buffer and open file +gzifstream::gzifstream(const char* name, + std::ios_base::openmode mode) +: std::istream(NULL), sb() +{ + this->init(&sb); + this->open(name, mode); +} + +// Initialize stream buffer and attach to file +gzifstream::gzifstream(int fd, + std::ios_base::openmode mode) +: std::istream(NULL), sb() +{ + this->init(&sb); + this->attach(fd, mode); +} + +// Open file and go into fail() state if unsuccessful +void +gzifstream::open(const char* name, + std::ios_base::openmode mode) +{ + if (!sb.open(name, mode | std::ios_base::in)) + this->setstate(std::ios_base::failbit); + else + this->clear(); +} + +// Attach to file and go into fail() state if unsuccessful +void +gzifstream::attach(int fd, + std::ios_base::openmode mode) +{ + if (!sb.attach(fd, mode | std::ios_base::in)) + this->setstate(std::ios_base::failbit); + else + this->clear(); +} + +// Close file +void +gzifstream::close() +{ + if (!sb.close()) + this->setstate(std::ios_base::failbit); +} + +/*****************************************************************************/ + +// Default constructor initializes stream buffer +gzofstream::gzofstream() +: std::ostream(NULL), sb() +{ this->init(&sb); } + +// Initialize stream buffer and open file +gzofstream::gzofstream(const char* name, + std::ios_base::openmode mode) +: std::ostream(NULL), sb() +{ + this->init(&sb); + this->open(name, mode); +} + +// Initialize stream buffer and attach to file +gzofstream::gzofstream(int fd, + std::ios_base::openmode mode) +: std::ostream(NULL), sb() +{ + this->init(&sb); + this->attach(fd, mode); +} + +// Open file and go into fail() state if unsuccessful +void +gzofstream::open(const char* name, + std::ios_base::openmode mode) +{ + if (!sb.open(name, mode | std::ios_base::out)) + this->setstate(std::ios_base::failbit); + else + this->clear(); +} + +// Attach to file and go into fail() state if unsuccessful +void +gzofstream::attach(int fd, + std::ios_base::openmode mode) +{ + if (!sb.attach(fd, mode | std::ios_base::out)) + this->setstate(std::ios_base::failbit); + else + this->clear(); +} + +// Close file +void +gzofstream::close() +{ + if (!sb.close()) + this->setstate(std::ios_base::failbit); +} + +#endif // HAVE_ZLIB + +/* +;;; Local Variables: *** +;;; mode: C++ *** +;;; End: *** +*/
new file mode 100644 --- /dev/null +++ b/src/zfstream.h @@ -0,0 +1,516 @@ +/* + +Copyright (C) 2005 Ludwig Schwardt, Kevin Ruland + +This file is part of Octave. + +Octave is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; either version 2, or (at your option) any +later version. + +Octave is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Octave; see the file COPYING. If not, write to the Free +Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +*/ + +/* + + This file is adapted from the zlib 1.2.2 contrib/iostream3 code, + written by + + Ludwig Schwardt <schwardt@sun.ac.za> + original version by Kevin Ruland <kevin@rodin.wustl.edu> + +*/ + +#ifndef ZFSTREAM_H +#define ZFSTREAM_H + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#ifdef HAVE_ZLIB + +#include <istream> // not iostream, since we don't need cin/cout +#include <ostream> +#include "zlib.h" + +/*****************************************************************************/ + +/** + * @brief Gzipped file stream buffer class. + * + * This class implements basic_filebuf for gzipped files. It doesn't yet support + * seeking (allowed by zlib but slow/limited), putback and read/write access + * (tricky). Otherwise, it attempts to be a drop-in replacement for the standard + * file streambuf. +*/ +class gzfilebuf : public std::streambuf +{ +public: + // Default constructor. + gzfilebuf(); + + // Destructor. + virtual + ~gzfilebuf(); + + /** + * @brief Set compression level and strategy on the fly. + * @param comp_level Compression level (see zlib.h for allowed values) + * @param comp_strategy Compression strategy (see zlib.h for allowed values) + * @return Z_OK on success, Z_STREAM_ERROR otherwise. + * + * Unfortunately, these parameters cannot be modified separately, as the + * previous zfstream version assumed. Since the strategy is seldom changed, + * it can default and setcompression(level) then becomes like the old + * setcompressionlevel(level). + */ + int + setcompression(int comp_level, + int comp_strategy = Z_DEFAULT_STRATEGY); + + /** + * @brief Check if file is open. + * @return True if file is open. + */ + bool + is_open() const { return (file != NULL); } + + /** + * @brief Open gzipped file. + * @param name File name. + * @param mode Open mode flags. + * @return @c this on success, NULL on failure. + */ + gzfilebuf* + open(const char* name, + std::ios_base::openmode mode); + + /** + * @brief Attach to already open gzipped file. + * @param fd File descriptor. + * @param mode Open mode flags. + * @return @c this on success, NULL on failure. + */ + gzfilebuf* + attach(int fd, + std::ios_base::openmode mode); + + /** + * @brief Close gzipped file. + * @return @c this on success, NULL on failure. + */ + gzfilebuf* + close(); + +protected: + /** + * @brief Convert ios open mode int to mode string used by zlib. + * @return True if valid mode flag combination. + */ + bool + open_mode(std::ios_base::openmode mode, + char* c_mode) const; + + /** + * @brief Number of characters available in stream buffer. + * @return Number of characters. + * + * This indicates number of characters in get area of stream buffer. + * These characters can be read without accessing the gzipped file. + */ + virtual std::streamsize + showmanyc(); + + /** + * @brief Fill get area from gzipped file. + * @return First character in get area on success, EOF on error. + * + * This actually reads characters from gzipped file to stream + * buffer. Always buffered. + */ + virtual int_type + underflow(); + + /** + * @brief Write put area to gzipped file. + * @param c Extra character to add to buffer contents. + * @return Non-EOF on success, EOF on error. + * + * This actually writes characters in stream buffer to + * gzipped file. With unbuffered output this is done one + * character at a time. + */ + virtual int_type + overflow(int_type c = traits_type::eof()); + + /** + * @brief Installs external stream buffer. + * @param p Pointer to char buffer. + * @param n Size of external buffer. + * @return @c this on success, NULL on failure. + * + * Call setbuf(0,0) to enable unbuffered output. + */ + virtual std::streambuf* + setbuf(char_type* p, + std::streamsize n); + + /** + * @brief Flush stream buffer to file. + * @return 0 on success, -1 on error. + * + * This calls underflow(EOF) to do the job. + */ + virtual int + sync(); + + /** + * @brief Alters the stream positions. + * + * Each derived class provides its own appropriate behavior. + */ + virtual pos_type + seekoff(off_type off, std::ios_base::seekdir way, + std::ios_base::openmode mode = + std::ios_base::in|std::ios_base::out); + + /** + * @brief Alters the stream positions. + * + * Each derived class provides its own appropriate behavior. + */ + virtual pos_type + seekpos(pos_type sp, std::ios_base::openmode mode = + std::ios_base::in|std::ios_base::out); + +// +// Some future enhancements +// +// virtual int_type uflow(); +// virtual int_type pbackfail(int_type c = traits_type::eof()); + +private: + /** + * @brief Allocate internal buffer. + * + * This function is safe to call multiple times. It will ensure + * that a proper internal buffer exists if it is required. If the + * buffer already exists or is external, the buffer pointers will be + * reset to their original state. + */ + void + enable_buffer(); + + /** + * @brief Destroy internal buffer. + * + * This function is safe to call multiple times. It will ensure + * that the internal buffer is deallocated if it exists. In any + * case, it will also reset the buffer pointers. + */ + void + disable_buffer(); + + /** + * Underlying file pointer. + */ + gzFile file; + + /** + * Mode in which file was opened. + */ + std::ios_base::openmode io_mode; + + /** + * @brief True if this object owns file descriptor. + * + * This makes the class responsible for closing the file + * upon destruction. + */ + bool own_fd; + + /** + * @brief Stream buffer. + * + * For simplicity this remains allocated on the free store for the + * entire life span of the gzfilebuf object, unless replaced by setbuf. + */ + char_type* buffer; + + /** + * @brief Stream buffer size. + * + * Defaults to system default buffer size (typically 8192 bytes). + * Modified by setbuf. + */ + std::streamsize buffer_size; + + /** + * @brief True if this object owns stream buffer. + * + * This makes the class responsible for deleting the buffer + * upon destruction. + */ + bool own_buffer; +}; + +/*****************************************************************************/ + +/** + * @brief Gzipped file input stream class. + * + * This class implements ifstream for gzipped files. Seeking and putback + * is not supported yet. +*/ +class gzifstream : public std::istream +{ +public: + // Default constructor + gzifstream(); + + /** + * @brief Construct stream on gzipped file to be opened. + * @param name File name. + * @param mode Open mode flags (forced to contain ios::in). + */ + explicit + gzifstream(const char* name, + std::ios_base::openmode mode = std::ios_base::in); + + /** + * @brief Construct stream on already open gzipped file. + * @param fd File descriptor. + * @param mode Open mode flags (forced to contain ios::in). + */ + explicit + gzifstream(int fd, + std::ios_base::openmode mode = std::ios_base::in); + + /** + * Obtain underlying stream buffer. + */ + gzfilebuf* + rdbuf() const + { return const_cast<gzfilebuf*>(&sb); } + + /** + * @brief Check if file is open. + * @return True if file is open. + */ + bool + is_open() { return sb.is_open(); } + + /** + * @brief Open gzipped file. + * @param name File name. + * @param mode Open mode flags (forced to contain ios::in). + * + * Stream will be in state good() if file opens successfully; + * otherwise in state fail(). This differs from the behavior of + * ifstream, which never sets the state to good() and therefore + * won't allow you to reuse the stream for a second file unless + * you manually clear() the state. The choice is a matter of + * convenience. + */ + void + open(const char* name, + std::ios_base::openmode mode = std::ios_base::in); + + /** + * @brief Attach to already open gzipped file. + * @param fd File descriptor. + * @param mode Open mode flags (forced to contain ios::in). + * + * Stream will be in state good() if attach succeeded; otherwise + * in state fail(). + */ + void + attach(int fd, + std::ios_base::openmode mode = std::ios_base::in); + + /** + * @brief Close gzipped file. + * + * Stream will be in state fail() if close failed. + */ + void + close(); + +private: + /** + * Underlying stream buffer. + */ + gzfilebuf sb; +}; + +/*****************************************************************************/ + +/** + * @brief Gzipped file output stream class. + * + * This class implements ofstream for gzipped files. Seeking and putback + * is not supported yet. +*/ +class gzofstream : public std::ostream +{ +public: + // Default constructor + gzofstream(); + + /** + * @brief Construct stream on gzipped file to be opened. + * @param name File name. + * @param mode Open mode flags (forced to contain ios::out). + */ + explicit + gzofstream(const char* name, + std::ios_base::openmode mode = std::ios_base::out); + + /** + * @brief Construct stream on already open gzipped file. + * @param fd File descriptor. + * @param mode Open mode flags (forced to contain ios::out). + */ + explicit + gzofstream(int fd, + std::ios_base::openmode mode = std::ios_base::out); + + /** + * Obtain underlying stream buffer. + */ + gzfilebuf* + rdbuf() const + { return const_cast<gzfilebuf*>(&sb); } + + /** + * @brief Check if file is open. + * @return True if file is open. + */ + bool + is_open() { return sb.is_open(); } + + /** + * @brief Open gzipped file. + * @param name File name. + * @param mode Open mode flags (forced to contain ios::out). + * + * Stream will be in state good() if file opens successfully; + * otherwise in state fail(). This differs from the behavior of + * ofstream, which never sets the state to good() and therefore + * won't allow you to reuse the stream for a second file unless + * you manually clear() the state. The choice is a matter of + * convenience. + */ + void + open(const char* name, + std::ios_base::openmode mode = std::ios_base::out); + + /** + * @brief Attach to already open gzipped file. + * @param fd File descriptor. + * @param mode Open mode flags (forced to contain ios::out). + * + * Stream will be in state good() if attach succeeded; otherwise + * in state fail(). + */ + void + attach(int fd, + std::ios_base::openmode mode = std::ios_base::out); + + /** + * @brief Close gzipped file. + * + * Stream will be in state fail() if close failed. + */ + void + close(); + +private: + /** + * Underlying stream buffer. + */ + gzfilebuf sb; +}; + +/*****************************************************************************/ + +/** + * @brief Gzipped file output stream manipulator class. + * + * This class defines a two-argument manipulator for gzofstream. It is used + * as base for the setcompression(int,int) manipulator. +*/ +template<typename T1, typename T2> + class gzomanip2 + { + public: + // Allows insertor to peek at internals + template <typename Ta, typename Tb> + friend gzofstream& + operator<<(gzofstream&, + const gzomanip2<Ta,Tb>&); + + // Constructor + gzomanip2(gzofstream& (*f)(gzofstream&, T1, T2), + T1 v1, + T2 v2); + private: + // Underlying manipulator function + gzofstream& + (*func)(gzofstream&, T1, T2); + + // Arguments for manipulator function + T1 val1; + T2 val2; + }; + +/*****************************************************************************/ + +// Manipulator function thunks through to stream buffer +inline gzofstream& +setcompression(gzofstream &gzs, int l, int s = Z_DEFAULT_STRATEGY) +{ + (gzs.rdbuf())->setcompression(l, s); + return gzs; +} + +// Manipulator constructor stores arguments +template<typename T1, typename T2> + inline + gzomanip2<T1,T2>::gzomanip2(gzofstream &(*f)(gzofstream &, T1, T2), + T1 v1, + T2 v2) + : func(f), val1(v1), val2(v2) + { } + +// Insertor applies underlying manipulator function to stream +template<typename T1, typename T2> + inline gzofstream& + operator<<(gzofstream& s, const gzomanip2<T1,T2>& m) + { return (*m.func)(s, m.val1, m.val2); } + +// Insert this onto stream to simplify setting of compression level +inline gzomanip2<int,int> +setcompression(int l, int s = Z_DEFAULT_STRATEGY) +{ return gzomanip2<int,int>(&setcompression, l, s); } + +#endif // HAVE_ZLIB + +#endif // ZFSTREAM_H + +/* +;;; Local Variables: *** +;;; mode: C++ *** +;;; End: *** +*/ +