Skip to content

Commit 9820cc4

Browse files
JohanMabilleJohanMabille
authored
Merge pull request #133 from davidbrochart/file_support
Add xio_gdal_handler
2 parents cd03d91 + b0fd657 commit 9820cc4

18 files changed

+548
-81
lines changed

‎CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,11 @@ set(XTENSOR_IO_HEADERS
6262
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_aws_handler.hpp
6363
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_disk_handler.hpp
6464
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gcs_handler.hpp
65+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gdal_handler.hpp
6566
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gzip.hpp
67+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file_wrapper.hpp
68+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile_wrapper.hpp
69+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_stream_wrapper.hpp
6670
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xnpz.hpp
6771
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor-io.hpp
6872
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor_io_config.hpp

‎include/xtensor-io/xgdal.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ namespace xt
4646
}
4747

4848
/**
49-
* Get a band interleaved by yixel layout; index order = [row, column, band].
49+
* Get a band interleaved by pixel layout; index order = [row, column, band].
5050
*/
5151
inline layout layout_band_interleaved_pixel()
5252
{
@@ -346,7 +346,7 @@ namespace xt
346346
std::string driver_name;
347347

348348
/**
349-
* Options passed to to GDAL when the dataset is created (like COMPRESS=JPEG).
349+
* Options passed to GDAL when the dataset is created (like COMPRESS=JPEG).
350350
*/
351351
std::vector<std::string> creation_options;
352352

@@ -541,4 +541,4 @@ namespace xt
541541
} // namespace xt
542542

543543

544-
#endif // XTENSOR_IO_XGDAL_HPP
544+
#endif // XTENSOR_IO_XGDAL_HPP

‎include/xtensor-io/xio_aws_handler.hpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "xtensor/xarray.hpp"
55
#include "xtensor/xexpression.hpp"
66
#include "xfile_array.hpp"
7+
#include "xio_stream_wrapper.hpp"
78
#include <aws/core/Aws.h>
89
#include <aws/s3/S3Client.h>
910
#include <aws/s3/model/GetObjectRequest.h>
@@ -70,7 +71,8 @@ namespace xt
7071
request.SetKey(path2);
7172

7273
std::shared_ptr<Aws::IOStream> writer = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", path, std::ios_base::in | std::ios_base::binary);
73-
dump_file(*writer, expression, m_format_config);
74+
auto s = xostream_wrapper(*writer);
75+
dump_file(s, expression, m_format_config);
7476

7577
request.SetBody(writer);
7678

@@ -109,7 +111,8 @@ namespace xt
109111
}
110112

111113
auto& reader = outcome.GetResultWithOwnership().GetBody();
112-
load_file<ET>(reader, array, m_format_config);
114+
auto s = xistream_wrapper(reader);
115+
load_file<ET>(s, array, m_format_config);
113116
}
114117

115118
template <class C>

‎include/xtensor-io/xio_binary.hpp

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,17 @@
1515
#include "xtensor/xadapt.hpp"
1616
#include "xtensor-io.hpp"
1717
#include "xfile_array.hpp"
18+
#include "xio_stream_wrapper.hpp"
1819

1920
namespace xt
2021
{
2122
namespace detail
2223
{
23-
template <typename T>
24-
inline xt::svector<T> load_bin_file(std::istream& stream, bool as_big_endian)
24+
template <typename T, class I>
25+
inline xt::svector<T> load_bin(I& stream, bool as_big_endian)
2526
{
26-
std::string buffer{std::istreambuf_iterator<char>{stream}, {}};
27+
std::string buffer;
28+
stream.read_all(buffer);
2729
std::size_t uncompressed_size = buffer.size() / sizeof(T);
2830
xt::svector<T> uncompressed_buffer(uncompressed_size);
2931
std::copy((const T*)(buffer.data()), (const T*)(buffer.data()) + uncompressed_size, uncompressed_buffer.begin());
@@ -35,7 +37,7 @@ namespace xt
3537
}
3638

3739
template <class O, class E>
38-
inline void dump_bin_stream(O& stream, const xexpression<E>& e, bool as_big_endian)
40+
inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian)
3941
{
4042
using value_type = typename E::value_type;
4143
const E& ex = e.derived_cast();
@@ -56,7 +58,7 @@ namespace xt
5658
{
5759
uncompressed_buffer = reinterpret_cast<const char*>(eval_ex.data());
5860
}
59-
stream.write(uncompressed_buffer, std::streamsize(uncompressed_size));
61+
stream.write(uncompressed_buffer, uncompressed_size);
6062
stream.flush();
6163
}
6264
} // namespace detail
@@ -67,10 +69,17 @@ namespace xt
6769
* @param stream An output stream to which to dump the data
6870
* @param e the xexpression
6971
*/
72+
template <typename E, class O>
73+
inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
74+
{
75+
detail::dump_bin(stream, e, as_big_endian);
76+
}
77+
7078
template <typename E>
71-
inline void dump_bin(std::ostream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
79+
inline void dump_bin(std::ofstream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
7280
{
73-
detail::dump_bin_stream(stream, e, as_big_endian);
81+
auto s = xostream_wrapper(stream);
82+
detail::dump_bin(s, e, as_big_endian);
7483
}
7584

7685
/**
@@ -80,14 +89,21 @@ namespace xt
8089
* @param e the xexpression
8190
*/
8291
template <typename E>
83-
inline void dump_bin(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
92+
inline void dump_bin(const char* filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
8493
{
8594
std::ofstream stream(filename, std::ofstream::binary);
8695
if (!stream.is_open())
8796
{
8897
std::runtime_error("IO Error: failed to open file");
8998
}
90-
detail::dump_bin_stream(stream, e, as_big_endian);
99+
auto s = xostream_wrapper(stream);
100+
detail::dump_bin(s, e, as_big_endian);
101+
}
102+
103+
template <typename E>
104+
inline void dump_bin(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
105+
{
106+
dump_bin<E>(filename.c_str(), e, as_big_endian);
91107
}
92108

93109
/**
@@ -99,7 +115,8 @@ namespace xt
99115
inline std::string dump_bin(const xexpression<E>& e, bool as_big_endian=is_big_endian())
100116
{
101117
std::stringstream stream;
102-
detail::dump_bin_stream(stream, e, as_big_endian);
118+
auto s = xostream_wrapper(stream);
119+
detail::dump_bin(s, e, as_big_endian);
103120
return stream.str();
104121
}
105122

@@ -112,10 +129,10 @@ namespace xt
112129
* Fortran format
113130
* @return xarray with contents from binary file
114131
*/
115-
template <typename T, layout_type L = layout_type::dynamic>
116-
inline auto load_bin(std::istream& stream, bool as_big_endian=is_big_endian())
132+
template <typename T, layout_type L = layout_type::dynamic, class I>
133+
inline auto load_bin(I& stream, bool as_big_endian=is_big_endian())
117134
{
118-
xt::svector<T> uncompressed_buffer = detail::load_bin_file<T>(stream, as_big_endian);
135+
xt::svector<T> uncompressed_buffer = detail::load_bin<T>(stream, as_big_endian);
119136
std::vector<std::size_t> shape = {uncompressed_buffer.size()};
120137
auto array = adapt(std::move(uncompressed_buffer), shape);
121138
return array;
@@ -131,14 +148,21 @@ namespace xt
131148
* @return xarray with contents from binary file
132149
*/
133150
template <typename T, layout_type L = layout_type::dynamic>
134-
inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian())
151+
inline auto load_bin(const char* filename, bool as_big_endian=is_big_endian())
135152
{
136153
std::ifstream stream(filename, std::ifstream::binary);
137154
if (!stream.is_open())
138155
{
139-
std::runtime_error("load_bin: failed to open file " + filename);
156+
std::runtime_error(std::string("load_bin: failed to open file ") + filename);
140157
}
141-
return load_bin<T, L>(stream, as_big_endian);
158+
auto s = xistream_wrapper(stream);
159+
return load_bin<T, L>(s, as_big_endian);
160+
}
161+
162+
template <typename T, layout_type L = layout_type::dynamic>
163+
inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian())
164+
{
165+
return load_bin<T, L>(filename.c_str(), as_big_endian);
142166
}
143167

144168
struct xio_binary_config
@@ -170,8 +194,8 @@ namespace xt
170194
}
171195
};
172196

173-
template <class E>
174-
void load_file(std::istream& stream, xexpression<E>& e, const xio_binary_config& config)
197+
template <class E, class I>
198+
void load_file(I& stream, xexpression<E>& e, const xio_binary_config& config)
175199
{
176200
E& ex = e.derived_cast();
177201
auto shape = ex.shape();
@@ -186,8 +210,8 @@ namespace xt
186210
}
187211
}
188212

189-
template <class E>
190-
void dump_file(std::ostream& stream, const xexpression<E> &e, const xio_binary_config& config)
213+
template <class E, class O>
214+
void dump_file(O& stream, const xexpression<E> &e, const xio_binary_config& config)
191215
{
192216
dump_bin(stream, e, config.big_endian);
193217
}

‎include/xtensor-io/xio_blosc.hpp

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "xtensor-io.hpp"
1717
#include "xfile_array.hpp"
1818
#include "blosc.h"
19+
#include "xio_stream_wrapper.hpp"
1920

2021
namespace xt
2122
{
@@ -31,11 +32,12 @@ namespace xt
3132
}
3233
}
3334

34-
template <typename T>
35-
inline xt::svector<T> load_blosc_file(std::istream& stream, bool as_big_endian)
35+
template <typename T, class I>
36+
inline xt::svector<T> load_blosc(I& stream, bool as_big_endian)
3637
{
3738
init_blosc();
38-
std::string compressed_buffer{std::istreambuf_iterator<char>{stream}, {}};
39+
std::string compressed_buffer;
40+
stream.read_all(compressed_buffer);
3941
auto compressed_size = compressed_buffer.size();
4042
std::size_t uncompressed_size = 0;
4143
int res = blosc_cbuffer_validate(compressed_buffer.data(), compressed_size, &uncompressed_size);
@@ -60,7 +62,7 @@ namespace xt
6062
}
6163

6264
template <class O, class E>
63-
inline void dump_blosc_stream(O& stream, const xexpression<E>& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize)
65+
inline void dump_blosc(O& stream, const xexpression<E>& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize)
6466
{
6567
init_blosc();
6668
using value_type = typename E::value_type;
@@ -99,8 +101,8 @@ namespace xt
99101
{
100102
XTENSOR_THROW(std::runtime_error, "Blosc: compression error");
101103
}
102-
stream.write(compressed_buffer,
103-
std::streamsize(true_compressed_size));
104+
stream.write(compressed_buffer, std::streamsize(true_compressed_size));
105+
stream.flush();
104106
char_allocator.deallocate(compressed_buffer, max_compressed_size);
105107
}
106108
} // namespace detail
@@ -111,10 +113,17 @@ namespace xt
111113
* @param stream An output stream to which to dump the data
112114
* @param e the xexpression
113115
*/
116+
template <typename E, class O>
117+
inline void dump_blosc(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
118+
{
119+
detail::dump_blosc(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
120+
}
121+
114122
template <typename E>
115123
inline void dump_blosc(std::ostream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
116124
{
117-
detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
125+
auto s = xostream_wrapper(stream);
126+
detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
118127
}
119128

120129
/**
@@ -124,14 +133,21 @@ namespace xt
124133
* @param e the xexpression
125134
*/
126135
template <typename E>
127-
inline void dump_blosc(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
136+
inline void dump_blosc(const char* filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
128137
{
129138
std::ofstream stream(filename, std::ofstream::binary);
130139
if (!stream.is_open())
131140
{
132-
XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename);
141+
XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename);
133142
}
134-
detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
143+
auto s = xostream_wrapper(stream);
144+
detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
145+
}
146+
147+
template <typename E>
148+
inline void dump_blosc(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
149+
{
150+
dump_blosc<E>(filename.c_str(), e, as_big_endian, clevel, shuffle, cname, blocksize);
135151
}
136152

137153
/**
@@ -143,7 +159,8 @@ namespace xt
143159
inline std::string dump_blosc(const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
144160
{
145161
std::stringstream stream;
146-
detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
162+
auto s = xostream_wrapper(stream);
163+
detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
147164
return stream.str();
148165
}
149166

@@ -156,10 +173,10 @@ namespace xt
156173
* Fortran format
157174
* @return xarray with contents from blosc file
158175
*/
159-
template <typename T, layout_type L = layout_type::dynamic>
160-
inline auto load_blosc(std::istream& stream, bool as_big_endian=is_big_endian())
176+
template <typename T, layout_type L = layout_type::dynamic, class I>
177+
inline auto load_blosc(I& stream, bool as_big_endian=is_big_endian())
161178
{
162-
xt::svector<T> uncompressed_buffer = detail::load_blosc_file<T>(stream, as_big_endian);
179+
xt::svector<T> uncompressed_buffer = detail::load_blosc<T>(stream, as_big_endian);
163180
std::vector<std::size_t> shape = {uncompressed_buffer.size()};
164181
auto array = adapt(std::move(uncompressed_buffer), shape);
165182
return array;
@@ -175,14 +192,21 @@ namespace xt
175192
* @return xarray with contents from blosc file
176193
*/
177194
template <typename T, layout_type L = layout_type::dynamic>
178-
inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian())
195+
inline auto load_blosc(const char* filename, bool as_big_endian=is_big_endian())
179196
{
180197
std::ifstream stream(filename, std::ifstream::binary);
181198
if (!stream.is_open())
182199
{
183-
XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename);
200+
XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename);
184201
}
185-
return load_blosc<T, L>(stream, as_big_endian);
202+
auto s = xistream_wrapper(stream);;
203+
return load_blosc<T, L>(s, as_big_endian);
204+
}
205+
206+
template <typename T, layout_type L = layout_type::dynamic>
207+
inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian())
208+
{
209+
return load_blosc<T, L>(filename.c_str(), as_big_endian);
186210
}
187211

188212
struct xio_blosc_config
@@ -230,8 +254,8 @@ namespace xt
230254
}
231255
};
232256

233-
template <class E>
234-
void load_file(std::istream& stream, xexpression<E>& e, const xio_blosc_config& config)
257+
template <class E, class I>
258+
void load_file(I& stream, xexpression<E>& e, const xio_blosc_config& config)
235259
{
236260
E& ex = e.derived_cast();
237261
auto shape = ex.shape();
@@ -246,8 +270,8 @@ namespace xt
246270
}
247271
}
248272

249-
template <class E>
250-
void dump_file(std::ostream& stream, const xexpression<E> &e, const xio_blosc_config& config)
273+
template <class E, class O>
274+
void dump_file(O& stream, const xexpression<E> &e, const xio_blosc_config& config)
251275
{
252276
dump_blosc(stream, e, config.big_endian, config.clevel, config.shuffle, config.cname.c_str(), config.blocksize);
253277
}

0 commit comments

Comments
 (0)