diff --git a/.gitignore b/.gitignore index 78db7920..d4e7eb35 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,4 @@ test/project_path.h test/integrate/__pycache__/ test/integrate/js/ .vscode/ -hash_hex/ -build_PreRelease/ -build_debug/ \ No newline at end of file +hash_hex/ \ No newline at end of file diff --git a/build.sh b/build.sh index f7bb2beb..36a10497 100755 --- a/build.sh +++ b/build.sh @@ -38,6 +38,7 @@ create_signed_so() { ["person_first_match"]="example/personlist/first_match/enclave/enclave.config.xml" ["person_first_match_multi"]="example/multi_personlist/first_match/enclave/enclave.config.xml" ["person_first_match_multi_offchain"]="example/multi_personlist/first_match/enclave_for_offchain/enclave.config.xml" + ["convert_sealed_file"]="example/convert/enclave/enclave.config.xml" ) for filename in ${!enclave_libs[@]}; do diff --git a/doc/Optimized Path ORAM introduction.md b/doc/Optimized Path ORAM introduction.md deleted file mode 100644 index c297432a..00000000 --- a/doc/Optimized Path ORAM introduction.md +++ /dev/null @@ -1,124 +0,0 @@ -# Optimized Path ORAM 实现数据访问模式保护 - - - -## Optimized Path ORAM实现 - -- example/oram_personlist/plugin/person_reader_oram.cpp - - - 增加一个函数`int get_item_index_field(void *handle, char *buf, int *len)`，功能是读取原始未加密文件的每行数据中用来查询的索引字段。 -- toolkit/datahub/seal_oram_file.cpp - - 功能： - - 加密文件为自定义的格式ORAM Sealed File，该文件结构包含： - - 文件头header - - 包含ORAM树的重要参数：真实块数`block_num`、ORAM树高`level_num_L`、ORAM树节点(bucket)数`bucket_num_N`以及ORAM Sealed File中各数据结构的起始偏移量。 - - 索引表id map - - 存储的是索引字段哈希和数据块号`block_id`之间的映射关系。 - - ORAM树 - - 是一个以数组形式存储的完全二叉树； - - 其每个节点是一个bucket，一个bucket是由`Z`个数据块构成的（`Z`为常量）；每个数据块的结构包括：数据块号`block_id`、叶子节点序号`leaf label`、数据块中有效的行数`valid_item_num`以及加密的数据`encrypted_batch`； - - 其中数据块分为真实数据块和虚数据块，在虚数据块中，设置数据块号`block_id`、叶子节点序号`leaf_label`和数据块中有效的行数`valid_item_num`均为0，加密的数据`encrypted_batch`是随机字符填充的内容；在真实数据块中，数据块号`block_id`和叶子节点序号`leaf_label`均为正整数，其最大值取决于真实块数`block_num`和ORAM树高`level_num_L`，加密的数据`encrypted_batch`存储的是真实的加密数据。 - - 地址映射表position map - - 存储的是数据块号`block_id`和其映射的叶子节点序号`leaf label`之间的关系。 - - 暂存区Stash - - 存储的是每次数据访问操作后，根据Path ORAM协议的写回策略而无法写回ORAM树中的真实数据块，被称为溢出的块。 - - 步骤： - - 先循环读取行数据，以确定用来存储数据的batch数量(即真实块数`block_num`)和每个batch中包含的数据行数 - - 建立索引表id map - - 循环读取每行数据的索引字段，将其哈希和其所在的数据块的`block_id`的映射关系写入id map - - 索引字段对应的是查询阶段的查询参数 - - 建立文件头header - - 根据真实块数`block_num`确定ORAM树高`level_num_L`、ORAM树节点(bucket)数`bucket_num_N` - - 向ORAM Sealed File中写入文件头header、加密的索引表id map - - 建立并写入加密的ORAM树 - - 初始化ORAM树时，为了减少访问操作产生的溢出块数量，应尽可能将真实数据块写在ORAM树的底部。因此按照顺序在树的顶部写入虚数据块，再将所有真实数据块依次写入ORAM树的底部。 - - 在写入真实数据块，应该将数据块号`block_id`和其对应的叶子节点序号`leaf label`的映射关系存储在地址映射表position map。 - - 由于使用数组形式存储ORAM树，需要保证所有的数据块是相同大小的。因此若最后一个batch包含的真实行数据数量小于一个batch最大可以容纳的行数，需要使用随机字符补足行数，用数据块结构中的`valid_item_num`记录有效的行数据行数。 - - 写入加密的地址映射表position map - - 更新并写入文件头header中position map和stash的起始偏移量 -- include/ypc/core_t/analyzer/oram_sealed_data_provider.h - - 功能：实现path ORAM算法，获取到查询参数（索引字段）对应的数据块 - - 类`oram_sealed_data_provider`需要的初始化参数： - - 数据提供方的加密数据文件哈希`data_hash` - - 数据提供方枢私钥`private_key` - - 数据提供方枢公钥`pkey` - - 解密的参数`decrypted_param` - - 成员函数`access()` - - 功能：根据`decrypted_param`获取其所在的batch，实现了path ORAM算法获取目标块的过程。 - - 步骤： - 1. `download_oram_params()`函数发起一个OCALL`download_oram_params_OCALL`在`data_hash`对应的加密数据文件中，读取存储的ORAM树的相关参数信息； - 2. `get_block_id(block_id)`函数发起一个OCALL`get_block_id_OCALL`在`data_hash`对应的加密数据文件中，获取`decrypted_param`所在的数据块对应的数据块号`block_id`; - 3. `download_position_map()`函数发起一个OCALL`download_position_map_OCALL`在`data_hash`对应的加密数据文件中，读取加密的地址映射表position map，使用数据提供方枢私钥`private_key`来解密position map； - 4. 在地址映射表position map中获取`block_id`映射的叶节点`leaf`, 并使其映射一个新的随机的叶节点`new_leaf`； - 5. `download_path(leaf)`函数发起一个OCALL`download_path_OCALL`在`data_hash`对应的加密数据文件中，读取ORAM树根节点到leaf对应的叶节点的一条加密路径； - 6. `download_stash()`函数发起一个OCALL`download_stash_OCALL`在`data_hash`对应的加密数据文件中，读取上次访问数据操作后加密的溢出块数组，并使用数据提供方枢私钥`private_key`来解密出溢出块存储在暂存区`m_stash`; - 7. `decrypt_path()`函数使用数据提供方枢私钥`private_key`来解密路径，将解密的路径中的真实数据块添加到暂存区`m_stash`中； - 8. `access_in_stash(b_id, new_leaf)`函数在暂存区`m_stash`中查找数据块号`block_id`对应的数据块的数据记录在`m_items`中，`m_valid_item_num`记录目标数据块的有效行数； - 9. `rebuild_new_path(leaf)`函数根据`leaf`和暂存区`m_stash`中所有真实数据块映射的叶节点来重建一条写回ORAM树中的新路径。一个数据块在新路径中的位置是其映射的叶节点和leaf的最近公共祖先节点，如果最近公共祖先节点所在的bucket已经放满了Z个真实数据块，则沿着最近公共祖先节点到根节点的路径寻找空块，如果直到根节点也没找空块，则该数据块就是一个溢出块，它存放在暂存区`m_stash`中； - 10. `encrypt_path()`函数使用数据提供方枢公钥`pkey`来加密新路径； - 11. `update_position_map()`函数先使用数据提供方枢公钥`pkey`来加密更新后的地址映射表position map，然后发起一个OCALL`update_position_map_OCALL`在`data_hash`对应的加密数据文件中，写入更新后的加密position map； - 12. `upload_path(leaf)`函数发起一个OCALL`upload_path_OCALL`在`data_hash`对应的加密数据文件中，将加密后的新路径写回ORAM树中； - 13. `update_stash()`函数将暂存区`m_stash`中溢出的块使用数据提供方枢公钥`pkey`加密后，再发起一个OCALL`download_oram_params_OCALL`在`data_hash`对应的加密数据文件中，写入加密的溢出块数组。 - - 成员函数`process()` - - 功能：遍历目标块中的每行数据，将每行数据发送给数据使用方过滤，以找到查询参数所在的数据行。 -- include/ypc/core_t/analyzer/interface/data_interface.h - - 增加一个类模板对应数据源解析类型`oram_sealed_datasource_parser` - - `uint32_t init_data_source(const uint8_t *data_source_info, uint32_t len)` - - 从`data_source_info`中获取加密的查询参数`param_data`，加密查询参数使用的数据使用方的枢公钥`param_data_pkey` - - 用`param_data_pkey`请求对应的数据使用方枢私钥`param_private_key`，使用`param_private_key`来解密`param_data` - - 使用 数据提供方的加密数据文件哈希`data_hash`、数据提供方枢私钥`private_key`、数据提供方枢公钥`pkey`和解密的参数`decrypted_param`去初始化类`oram_sealed_data_provider` -- toolkit/analyzer/parsers/oram_parser.h - - 功能：定义了path ORAM算法中调用的所有OCALL函数的接口函数，这些OCALL接口函数其实是调用了`oramblockfile.h`中对加密文件ORAM sealed file中的读写方法。 - - 在`feed_datasource()`中调用`m_parser->init_data_source(data_info_bytes)`时传递的参数`data_info_bytes`需要包含数据使用方的加密请求参数。 -- include/ypc/core/oramblockfile.h - - 功能：提供了读写加密文件ORAM sealed file中各种数据结构的方法。 - - `bool get_block_id(bytes &item_index_field_hash, uint32_t *block_id)`：读取ORAM Sealed File中的索引表id map进入内存，获取索引字段哈希`item_index_field_hash`所在的数据块号`block_id`； - - `bool download_position_map(memref &posmap)`：读取ORAM Sealed File中加密的地址映射表position map； - - `bool update_position_map(uint8_t * position_map, uint32_t len)`：向ORAM Sealed File写入更新后的加密地址映射表position map； - - `bool download_path(uint32_t leaf, memref &en_path)`：读取从ORAM树根节点到叶节点`leaf`的路径； - - `bool upload_path(uint32_t leaf, uint8_t * encrpypted_path, uint32_t len)`：向ORAM Sealed File写入一条重建的从ORAM树根节点到叶节点`leaf`的新路径； - - `bool download_stash(memref &st)`：读取ORAM Sealed File中加密的暂存区Stash（只存储溢出数据块）； - - `bool update_stash(uint8_t * stash, uint32_t len)`：向ORAM Sealed File写入访问操作后的加密暂存区Stash（只写入溢出块）。 - -- include/ypc/core/oram_sealed_file.h - - - 是对include/ypc/core/oramblockfile.h类的一个封装。 - - - -## 测试 - -- `test/integrate/classic_job_oram.py` - - 类`classic_job`增加了一个成员变量`con_read_num`来设置连续读的次数； - - 类`classic_job`的方法`run(self)`：仅在第一次读的时候生成数据提供方枢钥、生成数据使用方枢钥和加密原始数据文件 - -- `test/integrate/test_findperson_oram.py`可以设置连续读的次数和随机的查询参数，初始化类`classic_job`，通过调用类`classic_job`的方法`run(self)`来测试连续查询的结果的正确性。 - - - -## 后续优化部分 - -- CMOV指令实现 - - ```cpp - std::vector m_stash; - std::vector m_position_map; - stbox::bytes m_encrypted_path; - std::vector m_decrypted_path; - ``` - - 以上使用了ntobject的换成结构体，调用OCALL的时候换成ntobject再序列化写到enclave以外（有必要吗？）就可以编写汇编函数了 - -- 默克尔树验证完整性 - - - `include/ypc/core_t/analyzer/interface/data_interface.h`中函数`check_actual_data_hash()`待实现 - -- 对每个查询参数字段都要建立索引 - -- 查询参数是唯一的，不唯一的时候如何做？ - -- 范围查询 - - - 数据文件在加密前按序排列，建立索引表id map时，block id也是按序排列的 - - 例如查询参数为10~100， 其对应的行数据在的block对应的ID可能为１～５ - - 拿到对应的块组合就可以找到查询参数对应的行数据组合 \ No newline at end of file diff --git a/example/convert/CMakeLists.txt b/example/convert/CMakeLists.txt new file mode 100644 index 00000000..10a5584d --- /dev/null +++ b/example/convert/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(enclave) +add_subdirectory(enclave2) \ No newline at end of file diff --git a/example/convert/common.h b/example/convert/common.h new file mode 100644 index 00000000..1622d7d6 --- /dev/null +++ b/example/convert/common.h @@ -0,0 +1,7 @@ +#pragma once +#include "common_t.h" +#include "ypc/core/blockfile.h" +#include "ypc/core/byte.h" +#include "ypc/corecommon/package.h" +typedef ypc::blockfile<0x82, 1024 * 1024, 256 * 64 * 1024> file_t; + diff --git a/example/convert/common_t.h b/example/convert/common_t.h new file mode 100644 index 00000000..9cb25648 --- /dev/null +++ b/example/convert/common_t.h @@ -0,0 +1,36 @@ +#pragma once +#include +#include + +define_nt(RYXXBZ, std::string, "RYXXBZ"); +define_nt(XM, std::string, "XM"); +define_nt(CYM, std::string, "CYM"); +define_nt(XBDM, std::string, "XBDM"); +define_nt(FWXXBZ, std::string, "FWXXBZ"); +define_nt(XP, std::string, "XP"); +define_nt(DWMC, std::string, "DWMC"); +define_nt(ZJHM, std::string, "ZJHM"); +define_nt(GJDM, std::string, "GJDM"); +define_nt(MZDM, std::string, "MZDM"); +define_nt(JGSSXDM, std::string, "JGSSXDM"); +define_nt(HKXZFLYDM, std::string, "HKXZFLYDM"); +define_nt(HLXDM, std::string, "HLXDM"); +define_nt(HJDZ_XZQHDM, std::string, "HJDZXZQHDM"); +define_nt(SJJZD_XZQHDM, std::string, "SJJZDXZQHDM"); +define_nt(SJJZD_QHNXXDZ, std::string, "SJJZDQHNXXDZ"); +define_nt(XLDM, std::string, "XLDM"); +define_nt(TSSFDM, std::string, "TSSFDM"); +define_nt(CSQR, std::string, "CSQR"); +define_nt(LXDH, std::string, "LXDH"); +define_nt(HYZKDM, std::string, "HYZKDM"); +define_nt(DJR_XM, std::string, "DJR_XM"); +define_nt(DJR_GMSFZHM, std::string, "DJRGMSFZHM"); +define_nt(DJR_LXDH, std::string, "DJRLXDH"); +define_nt(GXSJ, std::string, "GXSJ"); +define_nt(SJZT, std::string, "SJZT"); + +typedef ff::util::ntobject< + RYXXBZ, XM, CYM, XBDM, FWXXBZ, XP, DWMC, ZJHM, GJDM, MZDM, JGSSXDM, + HKXZFLYDM, HLXDM, HJDZ_XZQHDM, SJJZD_XZQHDM, SJJZD_QHNXXDZ, XLDM, TSSFDM, + CSQR, LXDH, HYZKDM, DJR_XM, DJR_GMSFZHM, DJR_LXDH, GXSJ, SJZT> + row_t; diff --git a/example/convert/enclave/CMakeLists.txt b/example/convert/enclave/CMakeLists.txt new file mode 100644 index 00000000..5e657449 --- /dev/null +++ b/example/convert/enclave/CMakeLists.txt @@ -0,0 +1,13 @@ +set(T_SRCS eparser.cpp) + +add_ypc_applet(convert_sealed_file + CRYPTO stdeth + SRCS ${T_SRCS}) + +if(SGX_MODE STREQUAL "Debug") + enclave_sign(convert_sealed_file KEY enclave_private.pem + CONFIG enclave.config.debug.xml) +else() + enclave_sign(convert_sealed_file KEY enclave_private.pem + CONFIG enclave.config.xml) +endif() diff --git a/example/convert/enclave/convert_parser.h b/example/convert/enclave/convert_parser.h new file mode 100644 index 00000000..d970ae49 --- /dev/null +++ b/example/convert/enclave/convert_parser.h @@ -0,0 +1,193 @@ +#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +class convert_parser { +public: + convert_parser() {} + convert_parser(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + + LOG(INFO) << "do convert_parse"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + + LOG(INFO) << "create id_map"; + + size_t batch_size = 0; + size_t item_size = 0; + uint64_t batch_num = 0; // the number of batch + uint64_t full_num = 0; + uint64_t last_num = 0; + uint64_t item_num = 0; + + std::vector id_map_array; + uint32_t batch_id = 1; + + hpda::processor::internal::filter_impl match2( + &converter, [&](const user_item_t &v) { + if(item_size == 0) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + item_size = item_data.size(); + } + + std::string item_index_field = v.get(); + + input_buf_t item_index_field_pkg; + item_index_field_pkg.set(item_index_field); + bytes item_index_field_bytes = ypc::make_bytes::for_package(item_index_field_pkg); + bytes item_index_field_hash; + crypto_ptr->hash_256(item_index_field_bytes, item_index_field_hash); + + std::shared_ptr k_v(new oram_ntt::id_map_pair()); + k_v->set(item_index_field_hash, batch_id); + id_map_array.push_back(*k_v); + + ++item_num; + + batch_size += item_size; + if (batch_size >= ypc::utc::max_item_size) { + + if(full_num == 0) { + full_num = item_num; + } + + item_num = 0; + batch_size = 0; + + ++batch_id; + ++batch_num; + } + + return false; + }); + match2.get_engine()->run(); + + if(item_num > 0) { + last_num = item_num; + ++batch_num; + } + + oram_ntt::id_map_t id_map_pkg; + id_map_pkg.set(id_map_array); + id_map_array.clear(); + bytes id_map_bytes = ypc::make_bytes::for_package(id_map_pkg); + + + LOG(INFO) << "write header"; + + ypc::oram::header osf_header{}; + osf_header.block_num = batch_num; + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size(); + + osf_header.item_size = item_size; + if(full_num > 0) { + osf_header.item_num_each_batch = full_num; + } else { + osf_header.item_num_each_batch = last_num; + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + LOG(INFO) << "write id map"; + + int32_t id_map_size = 8000000; + for(int i = 0; i <= id_map_bytes.size(); i += id_map_size) { + if(i + id_map_size <= id_map_bytes.size()) { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_size); + } else { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_bytes.size() - i); + } + + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + } + + + return result; + } + +protected: + ypc::data_source *m_source; +}; diff --git a/example/convert/enclave/enclave.config.debug.xml b/example/convert/enclave/enclave.config.debug.xml new file mode 100644 index 00000000..b3c13bc5 --- /dev/null +++ b/example/convert/enclave/enclave.config.debug.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x80000000 + 10 + 1 + + 0 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave/enclave.config.xml b/example/convert/enclave/enclave.config.xml new file mode 100644 index 00000000..08e3ae1b --- /dev/null +++ b/example/convert/enclave/enclave.config.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x80000000 + 10 + 1 + + 1 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave/enclave.lds b/example/convert/enclave/enclave.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave/enclave.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave/enclave_debug.lds b/example/convert/enclave/enclave_debug.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave/enclave_debug.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave/enclave_private.pem b/example/convert/enclave/enclave_private.pem new file mode 100644 index 00000000..529d07be --- /dev/null +++ b/example/convert/enclave/enclave_private.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4gIBAAKCAYEAroOogvsj/fZDZY8XFdkl6dJmky0lRvnWMmpeH41Bla6U1qLZ +AmZuyIF+mQC/cgojIsrBMzBxb1kKqzATF4+XwPwgKz7fmiddmHyYz2WDJfAjIveJ +ZjdMjM4+EytGlkkJ52T8V8ds0/L2qKexJ+NBLxkeQLfV8n1mIk7zX7jguwbCG1Pr +nEMdJ3Sew20vnje+RsngAzdPChoJpVsWi/K7cettX/tbnre1DL02GXc5qJoQYk7b +3zkmhz31TgFrd9VVtmUGyFXAysuSAb3EN+5VnHGr0xKkeg8utErea2FNtNIgua8H +ONfm9Eiyaav1SVKzPHlyqLtcdxH3I8Wg7yqMsaprZ1n5A1v/levxnL8+It02KseD +5HqV4rf/cImSlCt3lpRg8U5E1pyFQ2IVEC/XTDMiI3c+AR+w2jSRB3Bwn9zJtFlW +KHG3m1xGI4ck+Lci1JvWWLXQagQSPtZTsubxTQNx1gsgZhgv1JHVZMdbVlAbbRMC +1nSuJNl7KPAS/VfzAgEDAoIBgHRXxaynbVP5gkO0ug6Qw/E27wzIw4SmjsxG6Wpe +K7kfDeRskKxESdsA/xCrKkwGwhcx1iIgS5+Qscd1Yg+1D9X9asd/P7waPmWoZd+Z +AhlKwhdPsO7PiF3e1AzHhGQwsUTt/Y/aSI1MpHBvy2/s1h9mFCslOUxTmWw0oj/Q +ldIEgWeNR72CE2+jFIJIyml6ftnb6qzPiga8Bm48ubKh0kvySOqnkmnPzgh+JBD6 +JnBmtZbfPT97bwTT+N6rnPqOOApvfHPf15kWI8yDbprG1l4OCUaIUH1AszxLd826 +5IPM+8gINLRDP1MA6azECPjTyHXhtnSIBZCyWSVkc05vYmNXYUNiXWMajcxW9M02 +wKzFELO8NCEAkaTPxwo4SCyIjUxiK1LbQ9h8PSy4c1+gGP4LAMR8xqP4QKg6zdu9 +osUGG/xRe/uufgTBFkcjqBHtK5L5VI0jeNIUAgW/6iNbYXjBMJ0GfauLs+g1VsOm +WfdgXzsb9DYdMa0OXXHypmV4GwKBwQDUwQj8RKJ6c8cT4vcWCoJvJF00+RFL+P3i +Gx2DLERxRrDa8AVGfqaCjsR+3vLgG8V/py+z+dxZYSqeB80Qeo6PDITcRKoeAYh9 +xlT3LJOS+k1cJcEmlbbO2IjLkTmzSwa80fWexKu8/Xv6vv15gpqYl1ngYoqJM3pd +vzmTIOi7MKSZ0WmEQavrZj8zK4endE3v0eAEeQ55j1GImbypSf7Idh7wOXtjZ7WD +Dg6yWDrri+AP/L3gClMj8wsAxMV4ZR8CgcEA0fzDHkFa6raVOxWnObmRoDhAtE0a +cjUj976NM5yyfdf2MrKy4/RhdTiPZ6b08/lBC/+xRfV3xKVGzacm6QjqjZrUpgHC +0LKiZaMtccCJjLtPwQd0jGQEnKfMFaPsnhOc5y8qVkCzVOSthY5qhz0XNotHHFmJ +gffVgB0iqrMTvSL7IA2yqqpOqNRlhaYhNl8TiFP3gIeMtVa9rZy31JPgT2uJ+kfo +gV7sdTPEjPWZd7OshGxWpT6QfVDj/T9T7L6tAoHBAI3WBf2DFvxNL2KXT2QHAZ9t +k3imC4f7U+wSE6zILaDZyzygA4RUbwG0gv8/TJVn2P/Eynf76DuWHGlaiLWnCbSz +Az2DHBQBBaku409zDQym3j1ugMRjzzSQWzJg0SIyBH3hTmnYcn3+Uqcp/lEBvGW6 +O+rsXFt3pukqJmIV8HzLGGaLm62BHUeZf3dyWm+i3p/hQAL7Xvu04QW70xuGqdr5 +afV7p5eaeQIJXyGQJ0eylV/90+qxjMKiB1XYg6WYvwKBwQCL/ddpgOdHJGN8uRom +e7Zq0Csi3hGheMKlKbN3vcxT5U7MdyHtTZZOJbTvxKNNUNYH/8uD+PqDGNneb29G +BfGzvI3EASyLIcGZF3OhKwZd0jUrWk2y7Vhob91jwp2+t73vdMbkKyI4mHOuXvGv +fg95si9oO7EBT+Oqvhccd2J+F1IVXncccYnF4u5ZGWt5lLewN/pVr7MjjykeaHqN +t+rfnQam2psA6fL4zS2zTmZPzR2tnY8Y1GBTi0Ko1OKd1HMCgcAb5cB/7/AQlhP9 +yQa04PLH9ygQkKKptZp7dy5WcWRx0K/hAHRoi2aw1wZqfm7VBNu2SLcs90kCCCxp +6C5sfJi6b8NpNbIPC+sc9wsFr7pGo9SFzQ78UlcWYK2Gu2FxlMjonhka5hvo4zvg +WxlpXKEkaFt3gLd92m/dMqBrHfafH7VwOJY2zT3WIpjwuk0ZzmRg5p0pG/svVQEH +NZmwRwlopysbR69B/n1nefJ84UO50fLh5s5Zr3gBRwbWNZyzhXk= +-----END RSA PRIVATE KEY----- diff --git a/example/convert/enclave/eparser.cpp b/example/convert/enclave/eparser.cpp new file mode 100644 index 00000000..98e59883 --- /dev/null +++ b/example/convert/enclave/eparser.cpp @@ -0,0 +1,13 @@ +#include "ypc/core_t/analyzer/algo_wrapper.h" +#include "ypc/core_t/analyzer/macro.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/crypto/gmssl.h" + +#include "convert_parser.h" + +ypc::algo_wrapper> + pw; + +YPC_PARSER_IMPL(pw); \ No newline at end of file diff --git a/example/convert/enclave/user_type.h b/example/convert/enclave/user_type.h new file mode 100644 index 00000000..8920839d --- /dev/null +++ b/example/convert/enclave/user_type.h @@ -0,0 +1,7 @@ +#pragma once +#include "../common_t.h" +#include +#include +#include + +typedef row_t user_item_t; diff --git a/example/convert/enclave2/CMakeLists.txt b/example/convert/enclave2/CMakeLists.txt new file mode 100644 index 00000000..da106139 --- /dev/null +++ b/example/convert/enclave2/CMakeLists.txt @@ -0,0 +1,13 @@ +set(T_SRCS eparser.cpp) + +add_ypc_applet(convert_sealed_file2 + CRYPTO stdeth + SRCS ${T_SRCS}) + +if(SGX_MODE STREQUAL "Debug") + enclave_sign(convert_sealed_file2 KEY enclave_private.pem + CONFIG enclave.config.debug.xml) +else() + enclave_sign(convert_sealed_file2 KEY enclave_private.pem + CONFIG enclave.config.xml) +endif() diff --git a/example/convert/enclave2/convert_parser2.h b/example/convert/enclave2/convert_parser2.h new file mode 100644 index 00000000..e08f4186 --- /dev/null +++ b/example/convert/enclave2/convert_parser2.h @@ -0,0 +1,479 @@ +#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +bytes random_string(size_t len) { + std::string ret(len, '0'); + static std::default_random_engine generator; + static std::uniform_int_distribution distribution(int('a'), int('z')); + static auto rand = std::bind(distribution, generator); + + for (size_t i = 0; i < len; i++) { + ret[i] = rand(); + } + return bytes(ret.data(), ret.size()); +} + +bool push_dummy_block(std::vector& bucket_array, bytes &data_hash, + uint8_t count, uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + for(uint8_t i = 0; i < count; ++i) { + oram_ntt::block_t b_block; + + std::vector dummy_batch; + for(uint32_t j = 0; j < item_num_each_batch; ++j) { + bytes dummy_item = random_string(item_size); + dummy_batch.push_back(dummy_item); + bytes k_hash = data_hash + dummy_item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_dummy_batch; + bytes dummy_batch_str = + ypc::make_bytes::for_package(dummy_batch); + + // encrypt dummy batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, dummy_batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_dummy_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + b_block.set(0, 0, 0, encrypted_dummy_batch); + bucket_array.push_back(b_block); + } + return true; +} + +uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { + // leftmost leaf node + uint32_t leftmost_leaf_index = (1 << level_num_L) - 1; + if(bucket_index >= leftmost_leaf_index) { + return bucket_index - leftmost_leaf_index + 1; + } + + // randomly select a path to the leaf node + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 1); + + if(dis(gen) == 0) { + return get_leaf_label(2 * bucket_index + 1, level_num_L); + } + return get_leaf_label(2 * bucket_index + 2, level_num_L); +} + +bool push_real_block(std::vector& bucket_array, bytes &data_hash, + uint32_t& block_id_value, uint32_t bucket_index, + std::vector &position_map_array, uint8_t level_num_L, + std::vector &batch, uint32_t &batch_str_size, + uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + oram_ntt::block_t b_block; + uint32_t valid_item_num = batch.size(); + for(uint32_t i = 0; i < item_num_each_batch - valid_item_num; ++i) { + ypc::bytes item = random_string(item_size); + batch.push_back(item); + } + + for(auto &item : batch) { + ypc::bytes k_hash = data_hash + item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_batch; + ypc::bytes batch_str = + ypc::make_bytes::for_package(batch); + // encrypt batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + if(batch_str_size != encrypted_batch.size()) { + batch_str_size = encrypted_batch.size(); + } + + uint32_t b_leaf_label = get_leaf_label(bucket_index, level_num_L); + position_map_array[block_id_value] = b_leaf_label; + b_block.set + (block_id_value++, b_leaf_label, valid_item_num, encrypted_batch); + bucket_array.push_back(b_block); + + return true; +} + + +class convert_parser2 { +public: + convert_parser2() {} + convert_parser2(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + LOG(INFO) << "do convert_parse2"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + // 1. read header + ypc::oram::header osf_header{}; + + auto ret = stbox::ocall_cast(download_convert_params_ocall) + (&osf_header.block_num, &osf_header.oram_tree_filepos, &osf_header.item_num_each_batch, &osf_header.item_size); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "download_convert_params_ocall fail!"; + return result; + } + + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + + + // 2. write ORAM tree + LOG(INFO) << "write ORAM tree"; + std::vector data_hash_array; + + // from which bucket to start writing real blocks + uint8_t lastbucket_realblocknum = osf_header.block_num % ypc::oram::BucketSizeZ; + uint32_t bucket_index = 0; // bucket index in ORAM tree + uint32_t block_id_value = 1; // block_id_value <= osf_header.block_num + + + // 2.1 write buckets full of dummy blocks + LOG(INFO) << "write buckets full of dummy blocks"; + // osf.seekp(osf_header.oram_tree_filepos, osf.beg); + int64_t filepos = osf_header.oram_tree_filepos; + for(uint32_t i = 0; i < osf_header.bucket_num_N - real_bucket_num; ++i) { + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + bool retf = push_dummy_block(bucket_array, data_hash, ypc::oram::BucketSizeZ, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + return result; + } + + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + // secondary encryption on the serialized bucket + // in order to encrypt the mapping relationship between block_id and leaf_label + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + } + + + + std::vector batch; + std::vector position_map_array(osf_header.block_num + 1, 0); + + + LOG(INFO) << "write real blocks"; + // 2.2 write the bucket that contains both real and dummy blocks + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + int i = ypc::oram::BucketSizeZ; + + if(lastbucket_realblocknum != 0) { + --real_bucket_num; + push_dummy_block(bucket_array, data_hash, + ypc::oram::BucketSizeZ - lastbucket_realblocknum, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + i = lastbucket_realblocknum; + } + + bool break_flag = false; + int j = 0; + batch.clear(); + + + // 2.3 write buckets full of real blocks + hpda::processor::internal::filter_impl match3( + &converter, [&](const user_item_t &v) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + + if(break_flag) { + return false; + } + + batch.push_back(item_data); + ++j; + + if(j == osf_header.item_num_each_batch) { + + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return false; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return false; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return false; + } + + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + + i = ypc::oram::BucketSizeZ; + } + + batch.clear(); + j = 0; + } + + return false; + }); + match3.get_engine()->run(); + + if(break_flag) { + LOG(ERROR) << "write real blocks fail!"; + return result; + } + + // 2.4 write the last data block + // The number of valid rows in the last data block may be less than item_num_each_batch + if(j > 0) { + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return result; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return result; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + } + } + + + LOG(INFO) << "write real blocks done"; + + + // 3. write position_map + osf_header.position_map_filepos = filepos; + oram_ntt::position_map_t position_map_pkg; + position_map_pkg.set(position_map_array); + bytes position_map_bytes = ypc::make_bytes::for_package(position_map_pkg); + ypc::bytes encrypted_position_map_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, position_map_bytes, ypc::utc::crypto_prefix_arbitrary, encrypted_position_map_bytes); + if (status != 0u) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_position_map_bytes.data(), encrypted_position_map_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += encrypted_position_map_bytes.size(); + + + // 4. write merkle tree + osf_header.merkle_tree_filepos = filepos; + + for(int i = (1 << osf_header.level_num_L) - 2; i >= 0; --i) { + ypc::bytes k_hash = data_hash_array[i] + data_hash_array[2*i + 1]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + + k_hash = data_hash_array[i] + data_hash_array[2*i + 2]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + } + + for(auto &data_hash : data_hash_array) { + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, data_hash.data(), data_hash.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += data_hash.size(); + } + + + // 5. update and write osf_header + osf_header.stash_filepos = filepos; + osf_header.item_num_each_batch = 0; + osf_header.item_size = 0; + + ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + + LOG(INFO) << "convert_parse2 done"; + + return result; + } + + +protected: + ypc::data_source *m_source; +}; diff --git a/example/convert/enclave2/enclave.config.debug.xml b/example/convert/enclave2/enclave.config.debug.xml new file mode 100644 index 00000000..471a1132 --- /dev/null +++ b/example/convert/enclave2/enclave.config.debug.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x5000000 + 10 + 1 + + 0 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave2/enclave.config.xml b/example/convert/enclave2/enclave.config.xml new file mode 100644 index 00000000..df005eb0 --- /dev/null +++ b/example/convert/enclave2/enclave.config.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x5000000 + 10 + 1 + + 1 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave2/enclave.lds b/example/convert/enclave2/enclave.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave2/enclave.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave2/enclave_debug.lds b/example/convert/enclave2/enclave_debug.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave2/enclave_debug.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave2/enclave_private.pem b/example/convert/enclave2/enclave_private.pem new file mode 100644 index 00000000..529d07be --- /dev/null +++ b/example/convert/enclave2/enclave_private.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4gIBAAKCAYEAroOogvsj/fZDZY8XFdkl6dJmky0lRvnWMmpeH41Bla6U1qLZ +AmZuyIF+mQC/cgojIsrBMzBxb1kKqzATF4+XwPwgKz7fmiddmHyYz2WDJfAjIveJ +ZjdMjM4+EytGlkkJ52T8V8ds0/L2qKexJ+NBLxkeQLfV8n1mIk7zX7jguwbCG1Pr +nEMdJ3Sew20vnje+RsngAzdPChoJpVsWi/K7cettX/tbnre1DL02GXc5qJoQYk7b +3zkmhz31TgFrd9VVtmUGyFXAysuSAb3EN+5VnHGr0xKkeg8utErea2FNtNIgua8H +ONfm9Eiyaav1SVKzPHlyqLtcdxH3I8Wg7yqMsaprZ1n5A1v/levxnL8+It02KseD +5HqV4rf/cImSlCt3lpRg8U5E1pyFQ2IVEC/XTDMiI3c+AR+w2jSRB3Bwn9zJtFlW +KHG3m1xGI4ck+Lci1JvWWLXQagQSPtZTsubxTQNx1gsgZhgv1JHVZMdbVlAbbRMC +1nSuJNl7KPAS/VfzAgEDAoIBgHRXxaynbVP5gkO0ug6Qw/E27wzIw4SmjsxG6Wpe +K7kfDeRskKxESdsA/xCrKkwGwhcx1iIgS5+Qscd1Yg+1D9X9asd/P7waPmWoZd+Z +AhlKwhdPsO7PiF3e1AzHhGQwsUTt/Y/aSI1MpHBvy2/s1h9mFCslOUxTmWw0oj/Q +ldIEgWeNR72CE2+jFIJIyml6ftnb6qzPiga8Bm48ubKh0kvySOqnkmnPzgh+JBD6 +JnBmtZbfPT97bwTT+N6rnPqOOApvfHPf15kWI8yDbprG1l4OCUaIUH1AszxLd826 +5IPM+8gINLRDP1MA6azECPjTyHXhtnSIBZCyWSVkc05vYmNXYUNiXWMajcxW9M02 +wKzFELO8NCEAkaTPxwo4SCyIjUxiK1LbQ9h8PSy4c1+gGP4LAMR8xqP4QKg6zdu9 +osUGG/xRe/uufgTBFkcjqBHtK5L5VI0jeNIUAgW/6iNbYXjBMJ0GfauLs+g1VsOm +WfdgXzsb9DYdMa0OXXHypmV4GwKBwQDUwQj8RKJ6c8cT4vcWCoJvJF00+RFL+P3i +Gx2DLERxRrDa8AVGfqaCjsR+3vLgG8V/py+z+dxZYSqeB80Qeo6PDITcRKoeAYh9 +xlT3LJOS+k1cJcEmlbbO2IjLkTmzSwa80fWexKu8/Xv6vv15gpqYl1ngYoqJM3pd +vzmTIOi7MKSZ0WmEQavrZj8zK4endE3v0eAEeQ55j1GImbypSf7Idh7wOXtjZ7WD +Dg6yWDrri+AP/L3gClMj8wsAxMV4ZR8CgcEA0fzDHkFa6raVOxWnObmRoDhAtE0a +cjUj976NM5yyfdf2MrKy4/RhdTiPZ6b08/lBC/+xRfV3xKVGzacm6QjqjZrUpgHC +0LKiZaMtccCJjLtPwQd0jGQEnKfMFaPsnhOc5y8qVkCzVOSthY5qhz0XNotHHFmJ +gffVgB0iqrMTvSL7IA2yqqpOqNRlhaYhNl8TiFP3gIeMtVa9rZy31JPgT2uJ+kfo +gV7sdTPEjPWZd7OshGxWpT6QfVDj/T9T7L6tAoHBAI3WBf2DFvxNL2KXT2QHAZ9t +k3imC4f7U+wSE6zILaDZyzygA4RUbwG0gv8/TJVn2P/Eynf76DuWHGlaiLWnCbSz +Az2DHBQBBaku409zDQym3j1ugMRjzzSQWzJg0SIyBH3hTmnYcn3+Uqcp/lEBvGW6 +O+rsXFt3pukqJmIV8HzLGGaLm62BHUeZf3dyWm+i3p/hQAL7Xvu04QW70xuGqdr5 +afV7p5eaeQIJXyGQJ0eylV/90+qxjMKiB1XYg6WYvwKBwQCL/ddpgOdHJGN8uRom +e7Zq0Csi3hGheMKlKbN3vcxT5U7MdyHtTZZOJbTvxKNNUNYH/8uD+PqDGNneb29G +BfGzvI3EASyLIcGZF3OhKwZd0jUrWk2y7Vhob91jwp2+t73vdMbkKyI4mHOuXvGv +fg95si9oO7EBT+Oqvhccd2J+F1IVXncccYnF4u5ZGWt5lLewN/pVr7MjjykeaHqN +t+rfnQam2psA6fL4zS2zTmZPzR2tnY8Y1GBTi0Ko1OKd1HMCgcAb5cB/7/AQlhP9 +yQa04PLH9ygQkKKptZp7dy5WcWRx0K/hAHRoi2aw1wZqfm7VBNu2SLcs90kCCCxp +6C5sfJi6b8NpNbIPC+sc9wsFr7pGo9SFzQ78UlcWYK2Gu2FxlMjonhka5hvo4zvg +WxlpXKEkaFt3gLd92m/dMqBrHfafH7VwOJY2zT3WIpjwuk0ZzmRg5p0pG/svVQEH +NZmwRwlopysbR69B/n1nefJ84UO50fLh5s5Zr3gBRwbWNZyzhXk= +-----END RSA PRIVATE KEY----- diff --git a/example/convert/enclave2/eparser.cpp b/example/convert/enclave2/eparser.cpp new file mode 100644 index 00000000..1940f920 --- /dev/null +++ b/example/convert/enclave2/eparser.cpp @@ -0,0 +1,13 @@ +#include "ypc/core_t/analyzer/algo_wrapper.h" +#include "ypc/core_t/analyzer/macro.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/crypto/gmssl.h" + +#include "convert_parser2.h" + +ypc::algo_wrapper> + pw; + +YPC_PARSER_IMPL(pw); diff --git a/example/convert/enclave2/user_type.h b/example/convert/enclave2/user_type.h new file mode 100644 index 00000000..8920839d --- /dev/null +++ b/example/convert/enclave2/user_type.h @@ -0,0 +1,7 @@ +#pragma once +#include "../common_t.h" +#include +#include +#include + +typedef row_t user_item_t; diff --git a/example/oram_personlist/generator/main.cpp b/example/oram_personlist/generator/main.cpp index 509cba02..79e822c5 100644 --- a/example/oram_personlist/generator/main.cpp +++ b/example/oram_personlist/generator/main.cpp @@ -2,6 +2,7 @@ #include "ypc/core/byte.h" #include "ypc/corecommon/package.h" #include +#include // typedef ff::sql::table, person_list_meta, // RYXXBZ, XM, CYM, XBDM, FWXXBZ, XP, DWMC, ZJHM, GJDM, @@ -16,7 +17,8 @@ row_t create(const std::string &id) { row_t t; t.set("4028f c 856517466 f 0165174f 03680103"); - t.set("张三+" + id); + t.set("张三"); + // t.set("张三+" + id); t.set("4028f c856517466 f 0165174f 03680105"); t.set(id); t.set("156"); @@ -40,17 +42,6 @@ row_t create(const std::string &id) { return t; } -/* -void write_to_database() { - ff::sql::mysql engine("localhost", "test", "1234", - "testdb"); - person_list_table_t::create_table(&engine); - person_list_table_t::row_collection_type rs; - rs.push_back(create("19860726")); - - person_list_table_t::insert_or_replace_rows(&engine, rs); -}*/ - void write_to_file(const std::string &path, int num) { uint64_t id = 421003198607262336; file_t f; @@ -84,7 +75,7 @@ void check_file(const std::string &path) { } int main(int argc, char *argv[]) { - write_to_file("person_list_oram", 1 << 16); - check_file("person_list_oram"); + write_to_file("person_list", 7970); + check_file("person_list"); return 0; } diff --git a/include/ypc/core/oram_sealed_file.h b/include/ypc/core/oram_sealed_file.h index df59105a..6214ffc2 100644 --- a/include/ypc/core/oram_sealed_file.h +++ b/include/ypc/core/oram_sealed_file.h @@ -8,8 +8,7 @@ namespace ypc { namespace internal { class oram_sealed_file_base { public: - // TODO:幻数 - using oramblockfile_t = ypc::oram::oramblockfile<1024 * 1024 * 256, 5 * 32, 4>; + using oramblockfile_t = ypc::oram::oramblockfile<1024 * 1024 * 256>; oram_sealed_file_base(const std::string &file_path); @@ -17,28 +16,32 @@ namespace internal { virtual void reset() = 0; + virtual void open_for_write() = 0; + + virtual void open_for_read() = 0; + virtual bool download_oram_params(uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size) = 0; - virtual bool get_block_id(bytes &item_index_field_hash, uint32_t *block_id) = 0; + virtual bool get_block_id(const bytes &item_index_field_hash, uint32_t *block_id) = 0; virtual bool download_position_map(memref &posmap) = 0; - virtual bool update_position_map(uint8_t * position_map, uint32_t len) = 0; + virtual bool update_position_map(const uint8_t * position_map, uint32_t len) = 0; virtual bool download_path(uint32_t leaf, memref &en_path) = 0; - virtual bool upload_path(uint32_t leaf, uint8_t * encrpypted_path, uint32_t len) = 0; + virtual bool upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) = 0; virtual bool download_stash(memref &st) = 0; - virtual bool update_stash(uint8_t * stash, uint32_t len) = 0; + virtual bool update_stash(const uint8_t * stash, uint32_t len) = 0; virtual bool read_root_hash(memref &root_hash) = 0; virtual bool download_merkle_hash(uint32_t leaf, memref &merkle_hash) = 0; - virtual bool update_merkle_hash(uint32_t leaf, uint8_t * merkle_hash, uint32_t len) = 0; + virtual bool update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) = 0; public: @@ -56,19 +59,21 @@ class simple_oram_sealed_file : public internal::oram_sealed_file_base { public: simple_oram_sealed_file(const std::string &file_path); virtual void reset(); + virtual void open_for_write(); + virtual void open_for_read(); virtual bool download_oram_params(uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size); - virtual bool get_block_id(bytes &item_index_field_hash, uint32_t *block_id); + virtual bool get_block_id(const bytes &item_index_field_hash, uint32_t *block_id); virtual bool download_position_map(memref &posmap); - virtual bool update_position_map(uint8_t * position_map, uint32_t len); + virtual bool update_position_map(const uint8_t * position_map, uint32_t len); virtual bool download_path(uint32_t leaf, memref &en_path); - virtual bool upload_path(uint32_t leaf, uint8_t * encrpypted_path, uint32_t len); + virtual bool upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len); virtual bool download_stash(memref &st); - virtual bool update_stash(uint8_t * stash, uint32_t len); + virtual bool update_stash(const uint8_t * stash, uint32_t len); // virtual bool read_root_hash(bytes &root_hash); virtual bool read_root_hash(memref &root_hash); virtual bool download_merkle_hash(uint32_t leaf, memref &merkle_hash); - virtual bool update_merkle_hash(uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + virtual bool update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len); }; diff --git a/include/ypc/core/oramblockfile.h b/include/ypc/core/oramblockfile.h index 96944a68..90424373 100644 --- a/include/ypc/core/oramblockfile.h +++ b/include/ypc/core/oramblockfile.h @@ -16,17 +16,17 @@ using oram_ntt = ypc::oram::nt; namespace ypc { namespace oram { -template +template + // uint32_t OramBlockSizeLimit_t, uint8_t OramBucketSize_t> class oramblockfile { public: const static uint64_t BlockNumLimit = OramBlockNumLimit_t; - const static uint32_t DataSizeB = OramBlockSizeLimit_t; - const static uint8_t BucketSizeZ = OramBucketSize_t; + // const static uint32_t DataSizeB = OramBlockSizeLimit_t; + // const static uint8_t BucketSizeZ = OramBucketSize_t; const static uint32_t hash_size = 32; oramblockfile(const std::string &file_path): m_file(), m_file_path(file_path), m_header(), m_id_map() { - open_for_write(); + // open_for_write(); } oramblockfile(const oramblockfile &) = delete; @@ -67,7 +67,7 @@ class oramblockfile { void read_header() { m_file.seekg(0, m_file.beg); - m_file.read((char *)&m_header, sizeof(header)); + m_file.read((char *)&m_header, sizeof(m_header)); } void read_id_map() { @@ -103,7 +103,7 @@ class oramblockfile { return true; } - bool get_block_id(bytes &item_index_field_hash, uint32_t *block_id) { + bool get_block_id(const bytes &item_index_field_hash, uint32_t *block_id) { read_id_map(); if(m_id_map.find(item_index_field_hash) == m_id_map.end()) { return false; @@ -132,7 +132,7 @@ class oramblockfile { return m_header.batch_str_size; } - bool update_position_map(uint8_t * position_map, uint32_t len) { + bool update_position_map(const uint8_t * position_map, uint32_t len) { read_header(); m_file.clear(); @@ -173,7 +173,7 @@ class oramblockfile { return true; } - bool upload_path(uint32_t leaf, uint8_t * encrpypted_path, uint32_t len) { + bool upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) { read_header(); m_file.clear(); @@ -213,9 +213,9 @@ class oramblockfile { return true; } - bool update_stash(uint8_t * stash, uint32_t len) { - read_header(); - m_file.clear(); + bool update_stash(const uint8_t * stash, uint32_t len) { + // read_header(); + // m_file.clear(); m_header.stash_size = len; m_file.seekp(0, m_file.beg); @@ -293,7 +293,7 @@ class oramblockfile { return true; } - bool update_merkle_hash(uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { + bool update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) { read_header(); m_file.clear(); diff --git a/include/ypc/core_t/analyzer/algo_wrapper.h b/include/ypc/core_t/analyzer/algo_wrapper.h index 5570d0c9..a8951f05 100644 --- a/include/ypc/core_t/analyzer/algo_wrapper.h +++ b/include/ypc/core_t/analyzer/algo_wrapper.h @@ -18,6 +18,8 @@ #include "ypc/core_t/analyzer/internal/data_streams/noinput_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/raw_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h" #include "ypc/core_t/analyzer/internal/keymgr_session.h" #include "ypc/core_t/analyzer/internal/results/local_result.h" #include "ypc/core_t/analyzer/internal/results/offchain_result.h" diff --git a/include/ypc/core_t/analyzer/eparser_t_interface.h b/include/ypc/core_t/analyzer/eparser_t_interface.h index 1af95aaf..62de6487 100644 --- a/include/ypc/core_t/analyzer/eparser_t_interface.h +++ b/include/ypc/core_t/analyzer/eparser_t_interface.h @@ -58,6 +58,14 @@ sgx_status_t SGX_CDECL download_merkle_hash_OCALL(uint32_t *retval, const uint8_ sgx_status_t SGX_CDECL update_merkle_hash_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + + +sgx_status_t SGX_CDECL write_convert_data_structure(uint32_t *retval, int64_t filepos, + uint8_t * convert_data_bytes, uint32_t len); +sgx_status_t SGX_CDECL download_convert_params_ocall(uint32_t *retval, uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + + sgx_status_t SGX_CDECL km_session_request_oram_ocall(uint32_t *retval, sgx_dh_msg1_t *dh_msg1, uint32_t *session_id); diff --git a/include/ypc/core_t/analyzer/interface/algo_interface.h b/include/ypc/core_t/analyzer/interface/algo_interface.h index cd97df7a..af95fd58 100644 --- a/include/ypc/core_t/analyzer/interface/algo_interface.h +++ b/include/ypc/core_t/analyzer/interface/algo_interface.h @@ -13,10 +13,13 @@ #include "ypc/corecommon/allowance.h" #include "ypc/stbox/ebyte.h" #include "ypc/stbox/stx_status.h" +#include "ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h" +#include "ypc/corecommon/oram_types.h" namespace ypc { namespace internal { typedef nt ntt; +typedef oram::nt oram_ntt; template class DataAllowancePolicy, + template class ModelAllowancePolicy> +class algo_interface + : virtual public parser_var, + virtual public request_key_var, + virtual public result_var, + virtual public do_parse_interface, + virtual public keymgr_interface, + virtual public encrypted_param_var, + virtual public check_allowance_interface, + virtual public forward_interface { + typedef Crypto ecc; + typedef keymgr_interface keymgr_interface_t; + typedef request_key_var request_key_var_t; + typedef do_parse_interface do_parse_interface_t; + typedef check_allowance_interface + allowance_checker_t; + typedef forward_interface forward_interface_t; + +protected: + uint32_t parse_data_item_impl(const uint8_t *input_param, uint32_t len) { +#ifdef DEBUG + LOG(INFO) << "start parse data"; +#endif + ntt::param_t param = + make_package::type>::from_bytes( + input_param, len); + +#ifdef DEBUG + LOG(INFO) << "done unmarshal param, start request private key"; +#endif + + request_key_var_t::m_pkey4v = param.get(); + stbox::bytes dian_pkey; + auto ret = keymgr_interface_t::request_private_key_for_public_key( + request_key_var_t::m_pkey4v, request_key_var_t::m_private_key, + dian_pkey); + if (ret) { + LOG(ERROR) << "request_private_key failed: " << stbox::status_string(ret); + return ret; + } + +#ifdef DEBUG + LOG(INFO) << "start do_parse"; +#endif + result_var::m_result = do_parse_interface_t::do_parse( + request_key_var_t::m_pkey4v.data(), request_key_var_t::m_pkey4v.size()); + +#ifdef DEBUG + LOG(INFO) << "end parse "; +#endif + result_var::m_cost_gas = 0; + return stbox::stx_status::success; + } +}; + template class DataAllowancePolicy, diff --git a/include/ypc/core_t/analyzer/interface/data_interface.h b/include/ypc/core_t/analyzer/interface/data_interface.h index fcc2172a..8929861b 100644 --- a/include/ypc/core_t/analyzer/interface/data_interface.h +++ b/include/ypc/core_t/analyzer/interface/data_interface.h @@ -5,6 +5,7 @@ #include "ypc/core_t/analyzer/internal/data_streams/raw_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/sealed_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h" #include "ypc/core_t/analyzer/internal/keymgr_session.h" #include "ypc/core_t/analyzer/raw_data_provider.h" #include "ypc/core_t/analyzer/sealed_data_provider.h" @@ -165,6 +166,53 @@ class data_interface } }; +template +class data_interface + : virtual public data_source_var, + virtual public keymgr_interface, + virtual public keymgr_session { + typedef keymgr_interface keymgr_interface_t; + +public: + uint32_t init_data_source(const uint8_t *data_source_info, uint32_t len) { + using ntt = nt; + auto pkg = make_package::type>::from_bytes(data_source_info, len); + + auto ret = keymgr_session::init_keymgr_session(); + if (ret) { + LOG(ERROR) << "init_keymgr_session_convert failed: " << stbox::status_string(ret); + return ret; + } + + stbox::bytes private_key, dian_pkey; + ret = keymgr_interface_t::request_private_key_for_public_key( + pkg.get(), private_key, dian_pkey); + if (ret) { + LOG(ERROR) << "request_private_key_for_public_key failed: " + << stbox::status_string(ret); + return ret; + } + + m_ds_use_pkey = pkg.get() + pkg.get(); + + m_datasource.reset(new sealed_data_provider( + pkg.get(), private_key)); + + return stbox::stx_status::success; + } + + uint32_t check_actual_data_hash() { + auto p = m_datasource.get(); + if (p->data_hash() != p->expect_data_hash()) { + LOG(ERROR) << "expect " << p->expect_data_hash() << ", got " + << p->data_hash(); + return stbox::stx_status::data_hash_not_same_as_expect; + } + return stbox::stx_status::success; + } +}; + template class data_interface : virtual public data_source_var, diff --git a/include/ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h b/include/ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h new file mode 100644 index 00000000..a31cfd4c --- /dev/null +++ b/include/ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h @@ -0,0 +1,16 @@ +#pragma once +#include "ypc/core_t/analyzer/helper/parser_type_traits.h" +#include "ypc/stbox/ebyte.h" + +namespace ypc { +namespace internal { + +class convert_sealed_data_stream {}; + +} // namespace internal +using convert_sealed_data_stream = internal::convert_sealed_data_stream; + +template <> struct datasource_type_traits { + constexpr static uint32_t value = ypc::utc::single_sealed_datasource_parser; +}; +} // namespace ypc \ No newline at end of file diff --git a/include/ypc/core_t/analyzer/oasm_lib.h b/include/ypc/core_t/analyzer/oasm_lib.h index f14e2284..75375f35 100644 --- a/include/ypc/core_t/analyzer/oasm_lib.h +++ b/include/ypc/core_t/analyzer/oasm_lib.h @@ -1,4 +1,10 @@ #pragma once #include -extern "C" void omove(uint32_t i, uint32_t *item, uint32_t loc, uint32_t *leaf, uint32_t newLabel); \ No newline at end of file +extern "C" { + void omove(uint32_t i, uint32_t *item, uint32_t loc, uint32_t *leaf, uint32_t newLabel); + void oset_value(uint32_t *dest, uint32_t value, bool flag); + void oset_flag(bool *dest_flag, bool src_flag, bool flag); + void oset_bytes(uint8_t *dest_bytes, uint8_t *src_bytes, uint32_t bytes_size, bool flag); +} + diff --git a/include/ypc/core_t/analyzer/oram_sealed_data_provider.h b/include/ypc/core_t/analyzer/oram_sealed_data_provider.h index c70e9302..bb8020b1 100644 --- a/include/ypc/core_t/analyzer/oram_sealed_data_provider.h +++ b/include/ypc/core_t/analyzer/oram_sealed_data_provider.h @@ -22,7 +22,6 @@ typedef ypc::nt ntt; namespace ypc { template -// class oram_sealed_data_provider : public data_source_with_dhash { class oram_sealed_data_provider : public data_source_with_merkle_hash { typedef Crypto crypto; public: @@ -170,7 +169,8 @@ class oram_sealed_data_provider : public data_source_with_merkle_hash { return true; } - + + // oarray_search(m_position_map, block_id, &leaf, new_leaf, m_position_map.size()); void oarray_search(std::vector& array, uint32_t loc, uint32_t *leaf, uint32_t newLabel, uint32_t N_level) { for(uint32_t i=0; i < N_level; ++i) { @@ -406,13 +406,40 @@ class oram_sealed_data_provider : public data_source_with_merkle_hash { } for(oram_ntt::block_t e_block : block_array) { + bool pushed_flag = false; for(uint32_t k = 0; k < m_stash.size(); ++k) { - if(e_block.get() > 0 && m_stash[k].get() == 0) { - m_stash[k].set - (e_block.get(), e_block.get(), - e_block.get(), e_block.get()); - break; - } + uint32_t p_block_id = e_block.get(); + uint32_t s_block_id = m_stash[k].get(); + + bool flag = p_block_id > 0 && s_block_id == 0 && !pushed_flag; + + uint32_t p_leaf_label = e_block.get(); + uint32_t s_leaf_label = m_stash[k].get(); + + uint32_t p_valid_item_num = e_block.get(); + uint32_t s_valid_item_num = m_stash[k].get(); + + stbox::bytes p_encrypted_batch = e_block.get(); + stbox::bytes s_encrypted_batch = m_stash[k].get(); + + oset_value(&s_block_id, p_block_id, flag); + oset_value(&s_leaf_label, p_leaf_label, flag); + oset_value(&s_valid_item_num, p_valid_item_num, flag); + oset_bytes(s_encrypted_batch.data(), p_encrypted_batch.data(), p_encrypted_batch.size(), flag); + oset_flag(&pushed_flag, true, flag); + + m_stash[k].set + (s_block_id, s_leaf_label, s_valid_item_num, s_encrypted_batch); + + + + // if(e_block.get() > 0 && m_stash[k].get() == 0) { + // m_stash[k].set + // (e_block.get(), e_block.get(), + // e_block.get(), e_block.get()); + // break; + // } } stbox::bytes encrypted_batch = e_block.get(); @@ -456,95 +483,195 @@ class oram_sealed_data_provider : public data_source_with_merkle_hash { return true; } + void oset_value(uint32_t *dest_value, uint32_t src_value, bool move_flag) { + if(move_flag) { + *dest_value = src_value; + } + } + bool access_in_stash(uint32_t block_id, uint32_t new_leaf) { + stbox::bytes target_encrypted_batch_str(m_header.batch_str_size); for(uint32_t i = 0; i < m_stash.size(); ++i) { - if(m_stash[i].get() == block_id) { - m_valid_item_num = m_stash[i].get(); - if(m_valid_item_num == 0) { - LOG(ERROR) << "fail, the value of valid item num is 0 "; - return false; - } + bool flag = m_stash[i].get() == block_id; + uint32_t leaf_label = m_stash[i].get(); - stbox::bytes encrypted_batch_str = m_stash[i].get(); - stbox::bytes decrypted_batch_str; - uint32_t status = crypto::decrypt_message_with_prefix( - m_private_key, encrypted_batch_str, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); - if (status) { - LOG(ERROR) << "decrypt_message_with_prefix fail: " - << stbox::status_string(status); - return false; - } + oset_value(&leaf_label, new_leaf, flag); + oset_value(&m_valid_item_num, m_stash[i].get(), flag); + oset_bytes(target_encrypted_batch_str.data(), m_stash[i].get().data(), m_header.batch_str_size, flag); + m_stash[i].set(leaf_label); - try { - auto pkg = make_package::from_bytes(decrypted_batch_str); - m_items = pkg.get(); - if (m_items.size() == 0) { - LOG(ERROR) << "fail, m_items.size() == 0 "; - return false; - } + } - m_item_index = 0; - m_stash[i].set(new_leaf); - return true; - } catch (const std::exception &e) { - LOG(ERROR) << "make_package got: " << e.what(); - return false; - } + if(m_valid_item_num == 0) { + LOG(ERROR) << "fail, the value of valid item num is 0 "; + return false; + } + + stbox::bytes decrypted_batch_str; + uint32_t status = crypto::decrypt_message_with_prefix( + m_private_key, target_encrypted_batch_str, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); + if (status) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + try { + auto pkg = make_package::from_bytes(decrypted_batch_str); + m_items = pkg.get(); + if (m_items.size() == 0) { + LOG(ERROR) << "fail, m_items size should not be 0 "; + return false; } + m_item_index = 0; + return true; + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; } + + + // for(uint32_t i = 0; i < m_stash.size(); ++i) { + // if(m_stash[i].get() == block_id) { + // m_valid_item_num = m_stash[i].get(); + // if(m_valid_item_num == 0) { + // LOG(ERROR) << "fail, the value of valid item num is 0 "; + // return false; + // } + + // stbox::bytes encrypted_batch_str = m_stash[i].get(); + // stbox::bytes decrypted_batch_str; + // uint32_t status = crypto::decrypt_message_with_prefix( + // m_private_key, encrypted_batch_str, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); + // if (status) { + // LOG(ERROR) << "decrypt_message_with_prefix fail: " + // << stbox::status_string(status); + // return false; + // } + + // try { + // auto pkg = make_package::from_bytes(decrypted_batch_str); + // m_items = pkg.get(); + // if (m_items.size() == 0) { + // LOG(ERROR) << "fail, m_items size should not be 0 "; + // return false; + // } + + // m_item_index = 0; + // m_stash[i].set(new_leaf); + // return true; + // } catch (const std::exception &e) { + // LOG(ERROR) << "make_package got: " << e.what(); + // return false; + // } + + // } + + // } return false; } - uint8_t get_level(uint32_t leaf1, uint32_t leaf2) { + // uint8_t get_level(uint32_t leaf1, uint32_t leaf2) { + // uint32_t leaf1_index = leaf1 - 1 + (1 << m_header.level_num_L) - 1; + // uint32_t leaf2_index = leaf2 - 1 + (1 << m_header.level_num_L) - 1; + + // while(leaf1_index != leaf2_index) { + // leaf1_index = (leaf1_index - 1) / 2; + // leaf2_index = (leaf2_index - 1) / 2; + // } + + // return floor(log2(leaf1_index + 1)); + // } + + bool fit_in_path(uint32_t bucket_index, uint32_t leaf1, uint32_t leaf2) { uint32_t leaf1_index = leaf1 - 1 + (1 << m_header.level_num_L) - 1; uint32_t leaf2_index = leaf2 - 1 + (1 << m_header.level_num_L) - 1; while(leaf1_index != leaf2_index) { - leaf1_index = (leaf1_index - 1) / 2; - leaf2_index = (leaf2_index - 1) / 2; + leaf1_index = (leaf1_index - 1) / 2; + leaf2_index = (leaf2_index - 1) / 2; } - return floor(log2(leaf1_index + 1)); + return leaf1_index == bucket_index; } void rebuild_new_path(uint32_t leaf) { - for(auto &bu : m_decrypted_path) { - auto block_array = bu.get(); + uint32_t bucket_index = leaf - 1 + (1 << m_header.level_num_L) - 1; + for(int i = m_decrypted_path.size() - 1; i >= 0; --i) { + auto block_array = m_decrypted_path[i].get(); for(uint8_t j = 0; j < oram::BucketSizeZ; ++j) { - block_array[j].set(0, 0, 0); - } - bu.set(block_array); - } + uint32_t b_block_id = 0, b_leaf_label = 0, b_valid_item_num = 0; + stbox::bytes encrypted_batch = block_array[j].get(); + bool updated_flag = false; + for(uint32_t k = 0; k < m_stash.size(); ++k) { + uint32_t s_block_id = m_stash[k].get(); + uint32_t s_leaf_label = m_stash[k].get(); + uint32_t s_valid_item_num = m_stash[k].get(); + bool flag = s_block_id > 0 && !updated_flag + && fit_in_path(bucket_index, s_leaf_label, leaf); + + oset_value(&b_block_id, s_block_id, flag); + oset_value(&b_leaf_label, s_leaf_label, flag); + oset_value(&b_valid_item_num, s_valid_item_num, flag); + oset_bytes(encrypted_batch.data(), m_stash[k].get().data(), + encrypted_batch.size(), flag);\ + + oset_value(&s_block_id, 0, flag); + oset_value(&s_leaf_label, 0, flag); + oset_value(&s_valid_item_num, 0, flag); + + oset_flag(&updated_flag, true, flag); + + m_stash[k].set + (s_block_id, s_leaf_label, s_valid_item_num); + } - for(uint32_t i = 0; i < m_stash.size(); ++i) { - if(m_stash[i].get() > 0) { - uint8_t low_level = get_level(leaf, m_stash[i].get()); - for(int level = low_level; level >= 0; --level) { - for(uint8_t k = 0; k < oram::BucketSizeZ; ++k) { - if(m_decrypted_path[level].get()[k].get() == 0) { - m_decrypted_path[level].get()[k].set - - (m_stash[i].get(), m_stash[i].get(), - m_stash[i].get(), m_stash[i].get()); + block_array[j].set + (b_block_id, b_leaf_label, b_valid_item_num, encrypted_batch); - m_stash[i].set(0, 0, 0); + } + m_decrypted_path[i].set(block_array); + bucket_index = (bucket_index - 1) / 2; + } + + + // for(auto &bu : m_decrypted_path) { + // auto block_array = bu.get(); + // for(uint8_t j = 0; j < oram::BucketSizeZ; ++j) { + // block_array[j].set(0, 0, 0); + // } + // bu.set(block_array); + // } + + // for(uint32_t i = 0; i < m_stash.size(); ++i) { + // if(m_stash[i].get() > 0) { + // uint8_t low_level = get_level(leaf, m_stash[i].get()); + // for(int level = low_level; level >= 0; --level) { + // for(uint8_t k = 0; k < oram::BucketSizeZ; ++k) { + // if(m_decrypted_path[level].get()[k].get() == 0) { + // m_decrypted_path[level].get()[k].set + // + // (m_stash[i].get(), m_stash[i].get(), + // m_stash[i].get(), m_stash[i].get()); + + // m_stash[i].set(0, 0, 0); - break; + // break; - } - } + // } + // } - if(m_stash[i].get() == 0) { - break; - } - } - } - } + // if(m_stash[i].get() == 0) { + // break; + // } + // } + // } + // } } bool update_stash() { @@ -617,18 +744,6 @@ class oram_sealed_data_provider : public data_source_with_merkle_hash { return false; } - // TODO:虚块重新填充 - // for(uint32_t j = 0; j < items.size(); ++j) { - // std::string dummy_item_str; - // generate_random_str(dummy_item_str, item_size); - // bytes dummy_item(dummy_item_str); - // items[i] = dummy_item; - // stbox::bytes k_hash = data_hash + dummy_item; - // crypto::hash_256(k_hash, data_hash); - // } - - // 重新加密放入m_decrypted_path中 - } while(!m_merkle_hash_array[index].get()) { diff --git a/include/ypc/corecommon/oram_types.h b/include/ypc/corecommon/oram_types.h index 3f0515a0..538c145e 100644 --- a/include/ypc/corecommon/oram_types.h +++ b/include/ypc/corecommon/oram_types.h @@ -10,6 +10,24 @@ namespace oram { constexpr static uint8_t BucketSizeZ = 4; constexpr static uint32_t stash_size = 90; +struct header { + uint32_t block_num; + uint32_t bucket_num_N; + uint8_t level_num_L; + uint32_t bucket_str_size; + uint32_t batch_str_size; + long int id_map_filepos; + long int oram_tree_filepos; + long int position_map_filepos; + long int merkle_tree_filepos; + long int stash_filepos; + uint64_t stash_size; + + uint64_t item_num_each_batch; + uint64_t item_size; + +}; + template struct nt { define_nt(item_index_field_hash, BytesType); define_nt(block_id, uint32_t); @@ -39,19 +57,7 @@ template struct nt { }; -struct header { - uint32_t block_num; - uint32_t bucket_num_N; - uint8_t level_num_L; - uint32_t bucket_str_size; - uint32_t batch_str_size; - long int id_map_filepos; - long int oram_tree_filepos; - long int position_map_filepos; - long int merkle_tree_filepos; - long int stash_filepos; - uint64_t stash_size; -}; + } diff --git a/include/ypc/edl/eparser.edl b/include/ypc/edl/eparser.edl index d0b90bac..fd6bec3b 100644 --- a/include/ypc/edl/eparser.edl +++ b/include/ypc/edl/eparser.edl @@ -65,5 +65,7 @@ enclave { uint32_t leaf, [out] uint8_t ** merkle_hash, [out] uint32_t *len); uint32_t update_merkle_hash_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, [in, size = len] uint8_t * merkle_hash, uint32_t len); + uint32_t write_convert_data_structure(int64_t filepos, [in, size = len] uint8_t * convert_data_bytes, uint32_t len); + uint32_t download_convert_params_ocall([out] uint32_t *block_num, [out] long int *oram_tree_filepos, [out] uint64_t *item_num_each_batch, [out] uint64_t *item_size); }; }; diff --git a/include/ypc/stbox/stx_status.def b/include/ypc/stbox/stx_status.def index bdcaece3..54d36f99 100644 --- a/include/ypc/stbox/stx_status.def +++ b/include/ypc/stbox/stx_status.def @@ -45,6 +45,7 @@ ATT_STATUS(aes_rand_fail, 0xC6) ATT_STATUS(forward_target_not_set, 0xC7) ATT_STATUS(forward_allowance_invalid, 0xC8) ATT_STATUS(oram_sealed_file_error, 0xC9) +ATT_STATUS(convert_parser_error, 0xCA) ATT_STATUS(allowance_use_inconsistent_pkey, 0xD0) ATT_STATUS(model_allowance_miss, 0xD1) ATT_STATUS(data_allowance_miss, 0xD2) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 35e2890a..dac1fdb4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,4 +7,3 @@ add_subdirectory(core) add_subdirectory(crypto) add_subdirectory(keymgr) add_subdirectory(toolkit) -add_subdirectory(oram) diff --git a/test/integrate/classic_job_convert.py b/test/integrate/classic_job_convert.py new file mode 100644 index 00000000..42a8497d --- /dev/null +++ b/test/integrate/classic_job_convert.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 +import common +import json +from job_step import job_step + +def create_oram_file(oram_sealed_data_url): + cmd = "touch {}".format(oram_sealed_data_url) + output = common.execute_cmd(cmd) + return [cmd, output] + +class classic_job_convert: + def __init__(self, crypto, name, data_url, convert_parser_url, convert_parser2_url, oram_parser_url, plugin_url, con_read_num, config={}): + self.crypto = crypto + self.name = name + self.data_url = data_url + self.convert_parser_url = convert_parser_url + self.convert_parser2_url = convert_parser2_url + self.oram_parser_url = oram_parser_url + self.plugin_url = plugin_url + # self.input = input_param + self.all_outputs = list() + self.config = config + + self.input = "" + self.data_shukey_json = {} + self.shukey_json = {} + self.con_read_num = con_read_num + self.read_count = 0 + self.data_hash = "" + self.enclave_hash = "" + + + # def convert(self): + + + def run(self): + ''' + 1. call terminus to generate key + 2. call data provider to seal data + 3. call terminus to generate forward message + 4. call terminus to generate request + 5. call fid_analyzer + 6. call terminus to decrypt + ''' + self.all_outputs = list() + + # 0. generate key + data_key_file = self.name + ".data.key.json" + if self.read_count == 0: + self.data_shukey_json = job_step.gen_key(self.crypto, data_key_file) + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(data_key_file) + + + # 1. generate key + key_file = self.name + ".key.json" + if self.read_count == 0: + self.shukey_json = job_step.gen_key(self.crypto, key_file) + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(key_file) + + # 2. call data provider to seal data + sealed_data_url = self.name + ".sealed" + sealed_output = self.name + ".sealed.output" + oram_sealed_data_url = sealed_data_url + ".oram" + summary = {} + summary['data-url'] = self.data_url + summary['plugin-path'] = self.plugin_url + summary['sealed-data-url'] = sealed_data_url + summary['sealed-output'] = sealed_output + + if self.read_count == 0: + r = job_step.seal_data(self.crypto, self.data_url, self.plugin_url, + sealed_data_url, sealed_output, data_key_file) + self.data_hash = job_step.read_data_hash(sealed_output) + summary['data-hash'] = self.data_hash + print("done seal data with hash: {}, cmd: {}".format(self.data_hash, r[0])) + create_oram_file(oram_sealed_data_url) + + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(sealed_data_url) + self.all_outputs.append(sealed_output) + + # use first pkey + key = job_step.get_first_key(self.crypto) + pkey = key['public-key'] + summary['tee-pkey'] = key['public-key'] + + # 3. call terminus to generate forward message + forward_result = self.name + ".shukey.foward.json" + data_forward_json = job_step.forward_message( + self.crypto, data_key_file, pkey, "", forward_result) + if self.read_count == 0: + self.enclave_hash = job_step.read_parser_hash(self.convert_parser_url) + if self.read_count == 1: + self.enclave_hash = job_step.read_parser_hash(self.convert_parser2_url) + if self.read_count > 1: + self.enclave_hash = job_step.read_parser_hash(self.oram_parser_url) + self.all_outputs.append(forward_result) + + # 4.0 call terminus to generate forward message + param_key_forward_result = self.name + ".request.shukey.foward.json" + rq_forward_json = job_step.forward_message( + self.crypto, key_file, pkey, self.enclave_hash, param_key_forward_result) + self.all_outputs.append(param_key_forward_result) + + # 4.1 call terminus to generate request + param_output_url = self.name + "_param.json" + if self.read_count <= 1: + param_json = job_step.generate_request( + self.crypto, self.input, data_key_file, param_output_url, self.config) + else: + param_json = job_step.generate_request( + self.crypto, self.input, key_file, param_output_url, self.config) + summary['analyzer-input'] = param_json["encrypted-input"] + self.all_outputs.append(param_output_url) + + # 5. call fid_convert_analyzer 转换文件格式 + if self.read_count == 0: + input_obj = { + "input_data_url": sealed_data_url, + "input_data_hash": self.data_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = job_step.fid_convert_analyzer(self.shukey_json, self.data_shukey_json, rq_forward_json, self.enclave_hash, input_data, + self.convert_parser_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + + + if self.read_count == 1: + input_obj = { + "input_data_url": sealed_data_url, + "input_data_hash": self.data_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = job_step.fid_convert_analyzer(self.shukey_json, self.data_shukey_json, rq_forward_json, self.enclave_hash, input_data, + self.convert_parser2_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + + + # 6. call fid_oram_analyzer 使用ORAM格式的加密文件进行分析任务 + if self.read_count > 1: + root_hash_url = self.name + ".root_hash" + r = job_step.get_root_hash(oram_sealed_data_url, root_hash_url) + root_hash = r[1] + + input_obj = { + "input_data_url": oram_sealed_data_url, + # "input_data_hash": data_hash, + "input_data_hash": root_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = {} + # if(self.name.split('_')[-1] == 'oram'): + result_json = job_step.fid_oram_analyzer(self.shukey_json, rq_forward_json, self.enclave_hash, input_data, + self.oram_parser_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + self.all_outputs.append(root_hash_url) + + # 6. call terminus to decrypt + encrypted_result = summary["encrypted-result"] + decrypted_result = self.name + ".result" + self.result = job_step.decrypt_result( + self.crypto, encrypted_result, key_file, decrypted_result) + self.all_outputs.append(decrypted_result) + + + if 'remove-files' in self.config and self.config['remove-files']: + job_step.remove_files(self.all_outputs) + + self.read_count += 1 + diff --git a/test/integrate/classic_job_oram.py b/test/integrate/classic_job_oram.py index c71c7754..41a77ff3 100644 --- a/test/integrate/classic_job_oram.py +++ b/test/integrate/classic_job_oram.py @@ -5,7 +5,7 @@ from job_step import job_step -class classic_job: +class classic_job_oram: def __init__(self, crypto, name, data_url, parser_url, plugin_url, con_read_num, config={}): self.crypto = crypto self.name = name @@ -46,8 +46,8 @@ def run(self): self.all_outputs.append(key_file) # 2. call data provider to seal data - sealed_data_url = self.name + ".sealed" - sealed_output = self.name + ".sealed.output" + sealed_data_url = self.name + ".sealed.oram" + sealed_output = self.name + ".sealed.oram.output" summary = {} summary['data-url'] = self.data_url summary['plugin-path'] = self.plugin_url @@ -57,7 +57,7 @@ def run(self): if self.read_count == 0: r = job_step.oram_seal_data(self.crypto, self.data_url, self.plugin_url, sealed_data_url, sealed_output, data_key_file) - # TODO:需要在sealed_data_url中获取root hash ，作为fid_oram_analyzer的参数之一 + # 需要在sealed_data_url中获取root hash ，作为fid_oram_analyzer的参数之一 root_hash_url = self.name + ".root_hash" r = job_step.get_root_hash(sealed_data_url, root_hash_url) root_hash = r[1] diff --git a/test/integrate/generate_convert.py b/test/integrate/generate_convert.py new file mode 100644 index 00000000..4163cdde --- /dev/null +++ b/test/integrate/generate_convert.py @@ -0,0 +1,789 @@ +import os +import common +import subprocess + +def get_crypto_name(crypto): + if(crypto == "stdeth"): + crypto_name = "ypc::crypto::eth_sgx_crypto" + if(crypto == "gmssl"): + crypto_name = "ypc::crypto::gmssl_sgx_crypto" + return crypto_name + + +def compile_in_subdir(build_dir): + # build_dir + makefile_path = os.path.join(build_dir, 'Makefile') + if not os.path.isfile(makefile_path): + print(f"Error: makefile not found in {build_dir}") + return False + + # 执行编译命令 + try: + result = subprocess.run(['make', '-j8'], cwd = build_dir, check = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + print(result.stdout.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + print(f"Error: {e.stderr.decode('utf-8')}") + return False + + +def recompile(mode): + cmd = os.path.join(common.sdk_dir, "./build.sh compile-project {} -j8").format(mode) + output = common.execute_cmd(cmd) + return [cmd, output] + + +def run_test_convert_parser(): + cmd = 'python3 {}/test/integrate/test_convert_parser.py'.format(common.sdk_dir) + output = common.execute_cmd(cmd) + return [cmd, output] + + +# 检查目录是否存在 +def check_directory(directory_path): + if os.path.exists(directory_path): + return True + else: + print(f"\"{directory_path}\" does not exist.") + return False + +# 进入目录 +def enter_directory(directory_path): + if check_directory(directory_path): + os.chdir(directory_path) + return True + else: + return False + + +def gen_convert_parser(file_path, crypto_name, index_name): + content = '''#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +class convert_parser { +public: + convert_parser() {} + convert_parser(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + + LOG(INFO) << "do convert_parse"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + + LOG(INFO) << "create id_map"; + + size_t batch_size = 0; + size_t item_size = 0; + uint64_t batch_num = 0; // the number of batch + uint64_t full_num = 0; + uint64_t last_num = 0; + uint64_t item_num = 0; + + std::vector id_map_array; + uint32_t batch_id = 1; + + hpda::processor::internal::filter_impl match2( + &converter, [&](const user_item_t &v) { + if(item_size == 0) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + item_size = item_data.size(); + } + + std::string item_index_field = v.get<''' + + content = content + index_name + + content = content + '''>(); + + input_buf_t item_index_field_pkg; + item_index_field_pkg.set(item_index_field); + bytes item_index_field_bytes = ypc::make_bytes::for_package(item_index_field_pkg); + bytes item_index_field_hash; + crypto_ptr->hash_256(item_index_field_bytes, item_index_field_hash); + + std::shared_ptr k_v(new oram_ntt::id_map_pair()); + k_v->set(item_index_field_hash, batch_id); + id_map_array.push_back(*k_v); + + ++item_num; + + batch_size += item_size; + if (batch_size >= ypc::utc::max_item_size) { + + if(full_num == 0) { + full_num = item_num; + } + + item_num = 0; + batch_size = 0; + + ++batch_id; + ++batch_num; + } + + return false; + }); + match2.get_engine()->run(); + + if(item_num > 0) { + last_num = item_num; + ++batch_num; + } + + oram_ntt::id_map_t id_map_pkg; + id_map_pkg.set(id_map_array); + id_map_array.clear(); + bytes id_map_bytes = ypc::make_bytes::for_package(id_map_pkg); + + + LOG(INFO) << "write header"; + + ypc::oram::header osf_header{}; + osf_header.block_num = batch_num; + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size(); + + osf_header.item_size = item_size; + if(full_num > 0) { + osf_header.item_num_each_batch = full_num; + } else { + osf_header.item_num_each_batch = last_num; + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + LOG(INFO) << "write id map"; + + int32_t id_map_size = 8000000; + for(int i = 0; i <= id_map_bytes.size(); i += id_map_size) { + if(i + id_map_size <= id_map_bytes.size()) { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_size); + } else { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_bytes.size() - i); + } + + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + } + + + return result; + } + +protected: + ypc::data_source *m_source; +}; +''' + + with open(file_path, "w") as file: + file.write(content) + + + +def gen_convert_parser2(file_path2, crypto_name): + content = '''#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +bytes random_string(size_t len) { + std::string ret(len, '0'); + static std::default_random_engine generator; + static std::uniform_int_distribution distribution(int('a'), int('z')); + static auto rand = std::bind(distribution, generator); + + for (size_t i = 0; i < len; i++) { + ret[i] = rand(); + } + return bytes(ret.data(), ret.size()); +} + +bool push_dummy_block(std::vector& bucket_array, bytes &data_hash, + uint8_t count, uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + for(uint8_t i = 0; i < count; ++i) { + oram_ntt::block_t b_block; + + std::vector dummy_batch; + for(uint32_t j = 0; j < item_num_each_batch; ++j) { + bytes dummy_item = random_string(item_size); + dummy_batch.push_back(dummy_item); + bytes k_hash = data_hash + dummy_item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_dummy_batch; + bytes dummy_batch_str = + ypc::make_bytes::for_package(dummy_batch); + + // encrypt dummy batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, dummy_batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_dummy_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + b_block.set(0, 0, 0, encrypted_dummy_batch); + bucket_array.push_back(b_block); + } + return true; +} + +uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { + // leftmost leaf node + uint32_t leftmost_leaf_index = (1 << level_num_L) - 1; + if(bucket_index >= leftmost_leaf_index) { + return bucket_index - leftmost_leaf_index + 1; + } + + // randomly select a path to the leaf node + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 1); + + if(dis(gen) == 0) { + return get_leaf_label(2 * bucket_index + 1, level_num_L); + } + return get_leaf_label(2 * bucket_index + 2, level_num_L); +} + +bool push_real_block(std::vector& bucket_array, bytes &data_hash, + uint32_t& block_id_value, uint32_t bucket_index, + std::vector &position_map_array, uint8_t level_num_L, + std::vector &batch, uint32_t &batch_str_size, + uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + oram_ntt::block_t b_block; + uint32_t valid_item_num = batch.size(); + for(uint32_t i = 0; i < item_num_each_batch - valid_item_num; ++i) { + ypc::bytes item = random_string(item_size); + batch.push_back(item); + } + + for(auto &item : batch) { + ypc::bytes k_hash = data_hash + item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_batch; + ypc::bytes batch_str = + ypc::make_bytes::for_package(batch); + // encrypt batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + if(batch_str_size != encrypted_batch.size()) { + batch_str_size = encrypted_batch.size(); + } + + uint32_t b_leaf_label = get_leaf_label(bucket_index, level_num_L); + position_map_array[block_id_value] = b_leaf_label; + b_block.set + (block_id_value++, b_leaf_label, valid_item_num, encrypted_batch); + bucket_array.push_back(b_block); + + return true; +} + + +class convert_parser2 { +public: + convert_parser2() {} + convert_parser2(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + LOG(INFO) << "do convert_parse2"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + // 1. read header + ypc::oram::header osf_header{}; + + auto ret = stbox::ocall_cast(download_convert_params_ocall) + (&osf_header.block_num, &osf_header.oram_tree_filepos, &osf_header.item_num_each_batch, &osf_header.item_size); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "download_convert_params_ocall fail!"; + return result; + } + + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + + + // 2. write ORAM tree + LOG(INFO) << "write ORAM tree"; + std::vector data_hash_array; + + // from which bucket to start writing real blocks + uint8_t lastbucket_realblocknum = osf_header.block_num % ypc::oram::BucketSizeZ; + uint32_t bucket_index = 0; // bucket index in ORAM tree + uint32_t block_id_value = 1; // block_id_value <= osf_header.block_num + + + // 2.1 write buckets full of dummy blocks + LOG(INFO) << "write buckets full of dummy blocks"; + // osf.seekp(osf_header.oram_tree_filepos, osf.beg); + int64_t filepos = osf_header.oram_tree_filepos; + for(uint32_t i = 0; i < osf_header.bucket_num_N - real_bucket_num; ++i) { + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + bool retf = push_dummy_block(bucket_array, data_hash, ypc::oram::BucketSizeZ, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + return result; + } + + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + // secondary encryption on the serialized bucket + // in order to encrypt the mapping relationship between block_id and leaf_label + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + } + + + + std::vector batch; + std::vector position_map_array(osf_header.block_num + 1, 0); + + + LOG(INFO) << "write real blocks"; + // 2.2 write the bucket that contains both real and dummy blocks + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + int i = ypc::oram::BucketSizeZ; + + if(lastbucket_realblocknum != 0) { + --real_bucket_num; + push_dummy_block(bucket_array, data_hash, + ypc::oram::BucketSizeZ - lastbucket_realblocknum, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + i = lastbucket_realblocknum; + } + + bool break_flag = false; + int j = 0; + batch.clear(); + + + // 2.3 write buckets full of real blocks + hpda::processor::internal::filter_impl match3( + &converter, [&](const user_item_t &v) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + + if(break_flag) { + return false; + } + + batch.push_back(item_data); + ++j; + + if(j == osf_header.item_num_each_batch) { + + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return false; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return false; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return false; + } + + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + + i = ypc::oram::BucketSizeZ; + } + + batch.clear(); + j = 0; + } + + return false; + }); + match3.get_engine()->run(); + + if(break_flag) { + LOG(ERROR) << "write real blocks fail!"; + return result; + } + + // 2.4 write the last data block + // The number of valid rows in the last data block may be less than item_num_each_batch + if(j > 0) { + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return result; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return result; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + } + } + + + LOG(INFO) << "write real blocks done"; + + + // 3. write position_map + osf_header.position_map_filepos = filepos; + oram_ntt::position_map_t position_map_pkg; + position_map_pkg.set(position_map_array); + bytes position_map_bytes = ypc::make_bytes::for_package(position_map_pkg); + ypc::bytes encrypted_position_map_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, position_map_bytes, ypc::utc::crypto_prefix_arbitrary, encrypted_position_map_bytes); + if (status != 0u) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_position_map_bytes.data(), encrypted_position_map_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += encrypted_position_map_bytes.size(); + + + // 4. write merkle tree + osf_header.merkle_tree_filepos = filepos; + + for(int i = (1 << osf_header.level_num_L) - 2; i >= 0; --i) { + ypc::bytes k_hash = data_hash_array[i] + data_hash_array[2*i + 1]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + + k_hash = data_hash_array[i] + data_hash_array[2*i + 2]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + } + + for(auto &data_hash : data_hash_array) { + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, data_hash.data(), data_hash.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += data_hash.size(); + } + + + // 5. update and write osf_header + osf_header.stash_filepos = filepos; + osf_header.item_num_each_batch = 0; + osf_header.item_size = 0; + + ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + + LOG(INFO) << "convert_parse2 done"; + + return result; + } + + +protected: + ypc::data_source *m_source; +}; +''' + + with open(file_path2, "w") as file: + file.write(content) + + + + + + +if __name__ == "__main__": + + crypto = "stdeth" + index_name = "ZJHM" + mode = "debug" # or "prerelease" or "release" + + + crypto_name = get_crypto_name(crypto) + + # 根据以上变量，生成转换enclave中执行的转换算法定义文件 + file_path = os.path.join(common.sdk_dir, "example/convert/enclave/convert_parser.h") + file_path2 = os.path.join(common.sdk_dir, "example/convert/enclave2/convert_parser2.h") + + gen_convert_parser(file_path, crypto_name, index_name) + gen_convert_parser2(file_path2, crypto_name) + + build_dir = os.path.join(common.sdk_dir, 'build', mode) + + # 必须进入项目顶层目录执行 + if(enter_directory(common.sdk_dir)): + + # 重新编译 + if compile_in_subdir(build_dir) == False: + recompile(mode) + + # 执行转换enclave并测试转换后加密数据文件是否能被正常读写 + r = run_test_convert_parser() + print(r[1]) \ No newline at end of file diff --git a/test/integrate/generate_oram.py b/test/integrate/generate_oram.py new file mode 100644 index 00000000..a85b2dc8 --- /dev/null +++ b/test/integrate/generate_oram.py @@ -0,0 +1,173 @@ +import os +import common +import subprocess + +# 检查目录是否存在 +def check_directory(directory_path): + if os.path.exists(directory_path): + return True + else: + print(f"\"{directory_path}\" does not exist.") + return False + +# 进入目录 +def enter_directory(directory_path): + if check_directory(directory_path): + os.chdir(directory_path) + return True + else: + return False + +def compile_in_subdir(build_dir): + # build_dir + makefile_path = os.path.join(build_dir, 'Makefile') + if not os.path.isfile(makefile_path): + print(f"Error: makefile not found in {build_dir}") + return False + + # 执行编译命令 + try: + result = subprocess.run(['make', '-j8'], cwd = build_dir, check = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + print(result.stdout.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + print(f"Error: {e.stderr.decode('utf-8')}") + return False + +def recompile(mode): + cmd = os.path.join(common.sdk_dir, "./build.sh compile-project {} -j8").format(mode) + output = common.execute_cmd(cmd) + return [cmd, output] + +def run_test_findperson_oram(): + cmd = 'python3 {}/test/integrate/test_findperson_oram.py'.format(common.sdk_dir) + output = common.execute_cmd(cmd) + return [cmd, output] + + +def gen_person_reader_oram(file_path, index_name): + content = '''#include "person_reader_oram.h" +#include "ypc/common/limits.h" +#include "../common.h" +#include +#include +#include +#include +#include + +void *create_item_reader(const char *file_path, int len) { + + file_t *f = new file_t(); + f->open_for_read(file_path); + f->reset_read_item(); + return f; +} + +int reset_for_read(void *handle) { + if (!handle) { + return -1; + } + file_t *f = (file_t *)handle; + f->reset_read_item(); + return 0; +} + +int read_item_data(void *handle, char *buf, int *len) { + if (!handle) { + return -1; + } + if (!buf) { + return -1; + } + file_t *f = (file_t *)handle; + + ypc::memref r; + bool t = f->next_item(r); + if (t) { + memcpy(buf, r.data(), r.size()); + *len = r.size(); + r.dealloc(); + return 0; + } else { + *len = 0; + return -1; + } + + return 0; +} + +int close_item_reader(void *handle) { + file_t *f = (file_t *)handle; + f->close(); + return 0; +} + +uint64_t get_item_number(void *handle) { + file_t *f = (file_t *)handle; + return f->item_number(); +} + +int get_item_index_field(void *handle, char *buf, int *len) { + if (!handle) { + return -1; + } + if (!buf) { + return -1; + } + file_t *f = (file_t *)handle; + + ypc::memref r; + bool t = f->next_item(r); + if (t) { + typedef typename ypc::cast_obj_to_package::type pkg_t; + auto item_pkg = + ypc::make_package::from_bytes(ypc::bytes(r.data(), r.size())); + std::string index_field = item_pkg.get<''' + + content = content + index_name + + content = content + '''>(); + memcpy(buf, index_field.c_str(), index_field.size()); + *len = index_field.size(); + + r.dealloc(); + return 0; + } else { + *len = 0; + return -1; + } + + return 0; +}''' + + with open(file_path, "w") as file: + file.write(content) + + + + +if __name__ == "__main__": + + index_name = "ZJHM" + mode = "debug" # or "prerelease" or "release" + + file_path = os.path.join(common.sdk_dir, "example/oram_personlist/plugin/person_reader_oram.cpp") + + gen_person_reader_oram(file_path, index_name) + + build_dir = os.path.join(common.sdk_dir, 'build', mode) + + # 必须进入项目顶层目录执行 + if(enter_directory(common.sdk_dir)): + + # 重新编译 + if compile_in_subdir(build_dir) == False: + recompile(mode) + + # 执行转换enclave并测试转换后加密数据文件是否能被正常读写 + r = run_test_findperson_oram() + print(r[1]) + + + + diff --git a/test/integrate/job_step.py b/test/integrate/job_step.py index 4489d044..4c584b97 100644 --- a/test/integrate/job_step.py +++ b/test/integrate/job_step.py @@ -152,6 +152,44 @@ def fid_analyzer(shukey_json, rq_forward_json, enclave_hash, input_data, parser_ with open(parser_output_file) as of: return of.readlines() + def fid_convert_analyzer(shukey_json, data_shukey_json, rq_forward_json, enclave_hash, input_data, parser_url, dian_pkey, model, crypto, param_json, allowances, parser_input_file, parser_output_file): + parser_input = { + "shu_info": { + "shu_pkey": shukey_json["public-key"], + "encrypted_shu_skey": rq_forward_json["encrypted_skey"], + "shu_forward_signature": rq_forward_json["forward_sig"], + "enclave_hash": enclave_hash + }, + "input_data": input_data, + "parser_path": parser_url, + "keymgr_path": common.kmgr_enclave[crypto], + "parser_enclave_hash": enclave_hash, + "dian_pkey": dian_pkey, + "model": model, + "param": { + "crypto": crypto, + "param_data": param_json["encrypted-input"], + "public-key": data_shukey_json["public-key"], + } + } + if allowances: + parser_input['param']['allowances'] = allowances + with open(parser_input_file, "w") as of: + json.dump(parser_input, of) + param = { + "input": parser_input_file, + "output": parser_output_file + } + r = common.fid_analyzer(**param) + print("done fid_analyzer with cmd: {}".format(r[0])) + try: + with open(parser_output_file) as of: + return json.load(of) + except Exception as e: + # result is not json format + with open(parser_output_file) as of: + return of.readlines() + def fid_oram_analyzer(shukey_json, rq_forward_json, enclave_hash, input_data, parser_url, dian_pkey, model, crypto, param_json, allowances, parser_input_file, parser_output_file): parser_input = { "shu_info": { diff --git a/test/integrate/test_convert_parser.py b/test/integrate/test_convert_parser.py new file mode 100644 index 00000000..97af5cb0 --- /dev/null +++ b/test/integrate/test_convert_parser.py @@ -0,0 +1,53 @@ +from classic_job_oram import classic_job_oram +from classic_job_convert import classic_job_convert +import os +import common +import sys +import random +import re + +def gen_personlist(**kwargs): + cmd = os.path.join(common.bin_dir, "./personlist_gen_oram") + output = common.execute_cmd(cmd) + return [cmd, output] + + +if __name__ == "__main__": + name = "convert_person" + gen_personlist() + + crypto = "stdeth" + # crypto = "gmssl" + data = "person_list" + convert_parser = os.path.join(common.lib_dir, "convert_sealed_file.signed.so") + convert_parser2 = os.path.join(common.lib_dir, "convert_sealed_file2.signed.so") + oram_parser = os.path.join(common.lib_dir, "person_first_match_oram.signed.so") + plugin = os.path.join( + common.lib_dir, "libperson_reader{}.so".format(common.debug_postfix())) + + # con_read_num必须>=3，前两次转换成ORAM密封格式 + con_read_num = 6 + + cj = classic_job_convert(crypto, name, data, convert_parser, convert_parser2, oram_parser, plugin, con_read_num, { + 'request-use-js': True, + 'remove-files': True if len(sys.argv) < 2 else False, + }) + + id = 421003198607262336 + result = [] + for i in range(con_read_num): + ran_i = random.randint(0, 7970 - 1) + id_str = str(id + ran_i) + input_param = "\"[{\\\"type\\\":\\\"string\\\",\\\"value\\\":\\\"" + id_str + "\\\"}]\"" + + cj.input = input_param + cj.run() + + if(cj.read_count > 2): + result.append(cj.result) + + matches = re.findall(r"\d+", cj.result[0]) + if matches[0] != str(id_str): + print("not find target row!") + break + print(result) \ No newline at end of file diff --git a/test/integrate/test_findperson_oram.py b/test/integrate/test_findperson_oram.py index 97e0345d..262bacf2 100644 --- a/test/integrate/test_findperson_oram.py +++ b/test/integrate/test_findperson_oram.py @@ -1,4 +1,4 @@ -from classic_job_oram import classic_job +from classic_job_oram import classic_job_oram import os import common import sys @@ -16,14 +16,14 @@ def gen_personlist(**kwargs): gen_personlist() crypto = "stdeth" - data = "person_list_oram" + data = "person_list" parser = os.path.join(common.lib_dir, "person_first_match_oram.signed.so") plugin = os.path.join( common.lib_dir, "libperson_reader_oram{}.so".format(common.debug_postfix())) - con_read_num = 1 + con_read_num = 5 - cj = classic_job(crypto, name, data, parser, plugin, con_read_num, { + cj = classic_job_oram(crypto, name, data, parser, plugin, con_read_num, { 'request-use-js': True, 'remove-files': True if len(sys.argv) < 2 else False, }) @@ -31,7 +31,7 @@ def gen_personlist(**kwargs): id = 421003198607262336 result = [] for i in range(con_read_num): - ran_i = random.randint(0, 1 << 16 - 1) + ran_i = random.randint(0, 7970 - 1) id_str = str(id + ran_i) # id_str = str(id + i) input_param = "\"[{\\\"type\\\":\\\"string\\\",\\\"value\\\":\\\"" + id_str + "\\\"}]\"" @@ -41,7 +41,8 @@ def gen_personlist(**kwargs): # print("input_param is : ", input_param) # print("result is : ", cj.result) matches = re.findall(r"\d+", cj.result[0]) - if matches[0] != matches[1]: + print(str(id)) + if matches[0] != str(id_str): print("not find target row!") break print(result) \ No newline at end of file diff --git a/test/oram/CMakeLists.txt b/test/oram/CMakeLists.txt deleted file mode 100644 index 7d5453f6..00000000 --- a/test/oram/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(plugin) diff --git a/test/oram/performance/test_read_time.py b/test/oram/performance/test_read_time.py deleted file mode 100644 index 10e4ee49..00000000 --- a/test/oram/performance/test_read_time.py +++ /dev/null @@ -1,27 +0,0 @@ -def read_numbers_from_file(filename): - with open(filename, 'r') as file: - lines = file.readlines() - numbers = [float(line.strip()) for line in lines] - return numbers - -def calculate_average(numbers): - if not numbers: - return 0.0 - return sum(numbers) / len(numbers) - -def main(): - filename = "pathoram_execution_times.txt" - # filename = "trivialoram_execution_times.txt" - - try: - numbers = read_numbers_from_file(filename) - average = calculate_average(numbers) - print("Numbers in file:", numbers) - print("Average value:", average) - except FileNotFoundError: - print(f"Error: File '{filename}' not found.") - except ValueError: - print(f"Error: File '{filename}' contains non-numeric data.") - -if __name__ == "__main__": - main() diff --git a/test/oram/plugin/CMakeLists.txt b/test/oram/plugin/CMakeLists.txt deleted file mode 100644 index a7bdde7e..00000000 --- a/test/oram/plugin/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable(test_privacy_data_reader main.cpp) -target_link_libraries(test_privacy_data_reader core) \ No newline at end of file diff --git a/test/oram/plugin/main.cpp b/test/oram/plugin/main.cpp deleted file mode 100644 index 0e2338f1..00000000 --- a/test/oram/plugin/main.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "ypc/core/privacy_data_reader.h" -#include "ypc/corecommon/package.h" - - -int main(int argc, char *argv[]) { - std::string plugin = "/home/lumj/YeeZ-Privacy-Computing/./lib/libperson_reader_oram_debug.so"; - std::string file = "/home/lumj/YeeZ-Privacy-Computing/person_list"; - ypc::privacy_data_reader reader(plugin, file); - - uint64_t item_number = reader.get_item_number(); - - std::cout << item_number << std::endl; - - uint counter = 0; - // ypc::bytes item_data = reader.read_item_data(); - - // while (!item_data.empty() && counter < item_number) { - // item_data = reader.read_item_data(); - // std::cout << item_data.size() << std::endl; - // ++counter; - // } - - - - std::string item_index_field = reader.get_item_index_field(); - - while (!item_index_field.empty() && counter < item_number) { - - item_index_field = reader.get_item_index_field(); - ++counter; - } - - - - - return 0; -} \ No newline at end of file diff --git a/toolkit/analyzer/iodef.h b/toolkit/analyzer/iodef.h index 340e8bfb..f223e014 100644 --- a/toolkit/analyzer/iodef.h +++ b/toolkit/analyzer/iodef.h @@ -2,9 +2,11 @@ #include "ypc/core/byte.h" #include "ypc/core/ntjson.h" #include "ypc/corecommon/nt_cols.h" +#include "ypc/corecommon/oram_types.h" #include "ypc/corecommon/package.h" using ntt = ypc::nt; +using oram_ntt = ypc::oram::nt; define_nt(input_data_url, std::string); define_nt(input_data_hash, ypc::bytes); diff --git a/toolkit/analyzer/parsers/oram_parser.cpp b/toolkit/analyzer/parsers/oram_parser.cpp index e63767c1..d5a44a6f 100644 --- a/toolkit/analyzer/parsers/oram_parser.cpp +++ b/toolkit/analyzer/parsers/oram_parser.cpp @@ -6,6 +6,99 @@ #include "ypc/corecommon/package.h" #include +#define FIND_DATA_SOURCE \ + auto hash = ypc::bytes(data_hash, hash_size); \ + if (m_data_sources.find(hash) == m_data_sources.end()) \ + { \ + LOG(ERROR) << "data with hash: " << hash << " not found"; \ + return stbox::stx_status::data_source_not_found; \ + } \ + auto sosf = m_data_sources[hash]; + +#define DOWNLOAD_OCALL_RETURN(buffer, buffer_size) \ + if (ret) \ + { \ + *buffer = buffer##_mem.data(); \ + *buffer_size = buffer##_mem.size(); \ + return stbox::stx_status::success; \ + } \ + return stbox::stx_status::oram_sealed_file_error; + +#define DOWNLOAD_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + uint8_t **buffer, uint32_t *buffer_size) \ + { \ + FIND_DATA_SOURCE \ + ypc::memref buffer##_mem; \ + bool ret = sosf->name(buffer##_mem); \ + DOWNLOAD_OCALL_RETURN(buffer, buffer_size) \ + } + +#define DOWNLOAD_LEAF_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + uint32_t leaf, uint8_t **buffer, uint32_t *buffer_size) \ + { \ + FIND_DATA_SOURCE \ + ypc::memref buffer##_mem; \ + bool ret = sosf->name(leaf, buffer##_mem); \ + DOWNLOAD_OCALL_RETURN(buffer, buffer_size) \ + } + +#define NOMAL_RETURN \ + if (ret) \ + { \ + return stbox::stx_status::success; \ + } \ + return stbox::stx_status::oram_sealed_file_error; + +#define UPDATE_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + const uint8_t *buffer, uint32_t buffer_size) \ + { \ + FIND_DATA_SOURCE \ + bool ret = sosf->name(buffer, buffer_size); \ + NOMAL_RETURN \ + } + +#define UPDATE_LEAF_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + uint32_t leaf, const uint8_t *buffer, uint32_t buffer_size) \ + { \ + FIND_DATA_SOURCE \ + bool ret = sosf->name(leaf, buffer, buffer_size); \ + NOMAL_RETURN \ + } + +DOWNLOAD_OCALL(download_position_map, position_map, len) +DOWNLOAD_OCALL(download_stash, stash, len) +DOWNLOAD_LEAF_OCALL(download_path, encrpypted_path, len) +DOWNLOAD_LEAF_OCALL(download_merkle_hash, encrpypted_path, len) +UPDATE_OCALL(update_position_map, position_map, len) +UPDATE_OCALL(update_stash, stash, len) +UPDATE_LEAF_OCALL(upload_path, encrpypted_path, len) +UPDATE_LEAF_OCALL(update_merkle_hash, merkle_hash, len) + +uint32_t oram_parser::download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size) { + FIND_DATA_SOURCE + bool ret = sosf->download_oram_params(block_num, bucket_num_N, level_num_L, bucket_str_size, batch_str_size); + NOMAL_RETURN +} + +uint32_t oram_parser::get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size) { + + FIND_DATA_SOURCE + ypc::bytes item_index_field_hash(param_hash, param_hash_size); + bool ret = sosf->get_block_id(item_index_field_hash, block_id); + NOMAL_RETURN +} + + + + oram_parser::oram_parser(const input_param_t ¶m) : m_param(param) {} oram_parser::~oram_parser() = default; @@ -186,7 +279,7 @@ uint32_t oram_parser::feed_datasource() { auto sosf = std::make_shared(url); m_data_sources.insert(std::make_pair(data_hash, sosf)); - sosf->reset(); + sosf->open_for_write(); auto shu = item.get(); auto shu_skey = shu.get(); @@ -268,207 +361,199 @@ uint32_t oram_parser::feed_model() { } uint32_t oram_parser::feed_param() { return ypc::success; } -uint32_t oram_parser::download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, - uint32_t *bucket_str_size, uint32_t *batch_str_size) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - bool ret = sosf->download_oram_params(block_num, bucket_num_N, level_num_L, bucket_str_size, batch_str_size); - if(ret) { - return stbox::stx_status::success; - } +// uint32_t oram_parser::download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, +// uint32_t *bucket_str_size, uint32_t *batch_str_size) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// bool ret = sosf->download_oram_params(block_num, bucket_num_N, level_num_L, bucket_str_size, batch_str_size); +// if(ret) { +// return stbox::stx_status::success; +// } - // TODO:定义新的错误码 - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t *block_id, - const uint8_t *param_hash, uint32_t param_hash_size) { - - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t *block_id, +// const uint8_t *param_hash, uint32_t param_hash_size) { + +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; - sosf->reset(); - ypc::bytes item_index_field_hash(param_hash, param_hash_size); - bool ret = sosf->get_block_id(item_index_field_hash, block_id); - if(ret) { - return stbox::stx_status::success; - } +// sosf->reset(); +// ypc::bytes item_index_field_hash(param_hash, param_hash_size); +// bool ret = sosf->get_block_id(item_index_field_hash, block_id); +// if(ret) { +// return stbox::stx_status::success; +// } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint8_t ** position_map, uint32_t *len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - ypc::memref posmap; - bool ret = sosf->download_position_map(posmap); - if(ret) { - *position_map = posmap.data(); - *len = posmap.size(); - return stbox::stx_status::success; - } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint8_t * position_map, uint32_t len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - bool ret = sosf->update_position_map(position_map, len); - if(ret) { - return stbox::stx_status::success; - } +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint8_t ** position_map, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref posmap; +// bool ret = sosf->download_position_map(posmap); +// if(ret) { +// *position_map = posmap.data(); +// *len = posmap.size(); +// return stbox::stx_status::success; +// } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - ypc::memref en_path; - bool ret = sosf->download_path(leaf, en_path); - if(ret) { - *encrpypted_path = en_path.data(); - *len = en_path.size(); - return stbox::stx_status::success; - } +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// const uint8_t * position_map, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->update_position_map(position_map, len); +// if(ret) { +// return stbox::stx_status::success; +// } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint8_t ** stash, uint32_t *len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - ypc::memref st; - bool ret = sosf->download_stash(st); - if(ret) { - *stash = st.data(); - *len = st.size(); - return stbox::stx_status::success; - } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint8_t * stash, uint32_t len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - - auto sosf = m_data_sources[hash]; - sosf->reset(); - - bool ret = sosf->update_stash(stash, len); - if(ret) { - return stbox::stx_status::success; - } +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref en_path; +// bool ret = sosf->download_path(leaf, en_path); +// if(ret) { +// *encrpypted_path = en_path.data(); +// *len = en_path.size(); +// return stbox::stx_status::success; +// } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t leaf, uint8_t * encrpypted_path, uint32_t len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - bool ret = sosf->upload_path(leaf, encrpypted_path, len); - if(ret) { - return stbox::stx_status::success; - } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - ypc::memref me_hash; - bool ret = sosf->download_merkle_hash(leaf, me_hash); - if(ret) { - *merkle_hash = me_hash.data(); - *len = me_hash.size(); - return stbox::stx_status::success; - } +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint8_t ** stash, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref st; +// bool ret = sosf->download_stash(st); +// if(ret) { +// *stash = st.data(); +// *len = st.size(); +// return stbox::stx_status::success; +// } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} - -uint32_t oram_parser::update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { - auto hash = ypc::bytes(data_hash, hash_size); - if (m_data_sources.find(hash) == m_data_sources.end()) { - LOG(ERROR) << "data with hash: " << hash << " not found"; - return stbox::stx_status::data_source_not_found; - } - auto sosf = m_data_sources[hash]; - sosf->reset(); - - bool ret = sosf->update_merkle_hash(leaf, merkle_hash, len); - if(ret) { - return stbox::stx_status::success; - } +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// const uint8_t * stash, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } + +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->update_stash(stash, len); +// if(ret) { +// return stbox::stx_status::success; +// } - // return stbox::stx_status::sealed_file_reach_end; - return stbox::stx_status::oram_sealed_file_error; -} \ No newline at end of file +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->upload_path(leaf, encrpypted_path, len); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref me_hash; +// bool ret = sosf->download_merkle_hash(leaf, me_hash); +// if(ret) { +// *merkle_hash = me_hash.data(); +// *len = me_hash.size(); +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->update_merkle_hash(leaf, merkle_hash, len); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } \ No newline at end of file diff --git a/toolkit/analyzer/parsers/oram_parser.h b/toolkit/analyzer/parsers/oram_parser.h index 49b8a5dc..680aff1d 100644 --- a/toolkit/analyzer/parsers/oram_parser.h +++ b/toolkit/analyzer/parsers/oram_parser.h @@ -1,11 +1,11 @@ #pragma once #include "ypc/core/oram_sealed_file.h" - #include "iodef.h" #include "ypc/common/parser_type.h" #include "ypc/core/sgx/parser_sgx_module.h" #include "ypc/keymgr/default/keymgr_sgx_module.h" + class oram_parser { public: oram_parser(const input_param_t ¶m); @@ -26,7 +26,7 @@ class oram_parser { uint8_t ** position_map, uint32_t *len); virtual uint32_t update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint8_t * position_map, uint32_t len); + const uint8_t * position_map, uint32_t len); virtual uint32_t download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len); @@ -34,16 +34,16 @@ class oram_parser { virtual uint32_t download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint8_t ** stash, uint32_t *len); virtual uint32_t update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint8_t * stash, uint32_t len); + const uint8_t * stash, uint32_t len); virtual uint32_t upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t leaf, uint8_t * encrpypted_path, uint32_t len); + uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len); virtual uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); virtual uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, - uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + uint32_t leaf, const uint8_t * merkle_hash, uint32_t len); inline std::shared_ptr keymgr() const { return m_keymgr; } diff --git a/toolkit/analyzer/parsers/parser.cpp b/toolkit/analyzer/parsers/parser.cpp index 45d102d0..4c8b0ac3 100644 --- a/toolkit/analyzer/parsers/parser.cpp +++ b/toolkit/analyzer/parsers/parser.cpp @@ -179,6 +179,8 @@ uint32_t parser::feed_datasource() { m_data_sources.insert(std::make_pair(data_hash, ssf)); ssf->reset_read(); + m_oram_sealed_file_path = url + ".oram"; + auto shu = item.get(); auto shu_skey = shu.get(); auto shu_forward_sig = shu.get(); @@ -265,3 +267,65 @@ uint32_t parser::next_data_batch(const uint8_t *data_hash, uint32_t hash_size, void parser::free_data_batch(uint8_t *data) { delete[] data; } + +uint32_t parser::write_convert_data_structure(int64_t filepos, const uint8_t * convert_data_bytes, uint32_t len) { + + std::fstream m_oram_sealed_file = std::fstream(m_oram_sealed_file_path, std::ios::in | std::ios::out | std::ios::binary); + if(!m_oram_sealed_file.is_open()) { + LOG(ERROR) << "Failed to create oram sealed file: " + m_oram_sealed_file_path; + return stbox::stx_status::convert_parser_error; + } + + try { + m_oram_sealed_file.seekp(filepos, m_oram_sealed_file.beg); + m_oram_sealed_file.write((char *)convert_data_bytes, len); + } catch (const std::exception &e) { + LOG(ERROR) << "write_convert_data_structure got error: " << e.what(); + return stbox::stx_status::convert_parser_error; + } + + m_oram_sealed_file.close(); + + return stbox::stx_status::success; +} + + + + +uint32_t parser::download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size) { + + std::fstream m_oram_sealed_file = std::fstream(m_oram_sealed_file_path, std::ios::in | std::ios::binary); + if(!m_oram_sealed_file.is_open()) { + LOG(ERROR) << "Failed to create oram sealed file: " + m_oram_sealed_file_path; + return stbox::stx_status::convert_parser_error; + } + + ypc::oram::header osf_header{}; + + try { + m_oram_sealed_file.seekg(0, m_oram_sealed_file.beg); + m_oram_sealed_file.read((char *)&osf_header, sizeof(osf_header)); + + *block_num = osf_header.block_num; + *oram_tree_filepos = osf_header.oram_tree_filepos; + *item_num_each_batch = osf_header.item_num_each_batch; + *item_size = osf_header.item_size; + + + LOG(INFO) << "osf_header.block_num = " << osf_header.block_num; + LOG(INFO) << "osf_header.oram_tree_filepos = " << osf_header.oram_tree_filepos; + LOG(INFO) << "osf_header.item_num_each_batch = " << osf_header.item_num_each_batch; + LOG(INFO) << "osf_header.item_size = " << osf_header.item_size; + + + } catch (const std::exception &e) { + LOG(ERROR) << "download_convert_params_ocall got error: " << e.what(); + return stbox::stx_status::convert_parser_error; + } + + m_oram_sealed_file.close(); + + return stbox::stx_status::success; +} + diff --git a/toolkit/analyzer/parsers/parser.h b/toolkit/analyzer/parsers/parser.h index cfcdaefb..b4f664ee 100644 --- a/toolkit/analyzer/parsers/parser.h +++ b/toolkit/analyzer/parsers/parser.h @@ -15,6 +15,11 @@ class parser { virtual uint32_t parse(); + virtual uint32_t write_convert_data_structure(int64_t filepos, const uint8_t * convert_data_bytes, uint32_t len); + + virtual uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + virtual uint32_t next_data_batch(const uint8_t *data_hash, uint32_t hash_size, uint8_t **data, uint32_t *len); virtual void free_data_batch(uint8_t *data); @@ -38,4 +43,5 @@ class parser { std::unordered_map> m_data_sources; std::string m_result_str; + std::string m_oram_sealed_file_path; }; diff --git a/toolkit/analyzer/sgx_bridge.cpp b/toolkit/analyzer/sgx_bridge.cpp index 227a53c2..9a214490 100644 --- a/toolkit/analyzer/sgx_bridge.cpp +++ b/toolkit/analyzer/sgx_bridge.cpp @@ -54,6 +54,13 @@ uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + + + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len); +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + } uint32_t km_session_request_ocall(sgx_dh_msg1_t *dh_msg1, @@ -157,4 +164,15 @@ uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { return o_parser->update_merkle_hash_OCALL(data_hash, hash_size, leaf, merkle_hash, len); +} + + + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len) { + return g_parser->write_convert_data_structure(filepos, convert_data_bytes, len); +} + +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size) { + return g_parser->download_convert_params_ocall(block_num, oram_tree_filepos, item_num_each_batch, item_size); } \ No newline at end of file diff --git a/toolkit/datahub/get_root_hash.cpp b/toolkit/datahub/get_root_hash.cpp index 42129ad1..8b50e991 100644 --- a/toolkit/datahub/get_root_hash.cpp +++ b/toolkit/datahub/get_root_hash.cpp @@ -66,7 +66,9 @@ int main(int argc, char *argv[]) { std::string output = vm["output"].as(); auto sosf = std::make_shared(data_file); - sosf->reset(); + + sosf->open_for_write(); + ypc::memref me_hash; bool ret = sosf->read_root_hash(me_hash); ypc::bytes root_hash(me_hash.data(), me_hash.size()); diff --git a/toolkit/datahub/seal_oram_file.cpp b/toolkit/datahub/seal_oram_file.cpp index e859919b..c8472db9 100644 --- a/toolkit/datahub/seal_oram_file.cpp +++ b/toolkit/datahub/seal_oram_file.cpp @@ -64,7 +64,7 @@ ypc::bytes random_string(size_t len) { } void push_dummy_block(std::vector& bucket_array, ypc::bytes &data_hash, - uint8_t count, uint64_t item_num_each_batch, size_t item_size, + uint8_t count, uint64_t item_num_each_batch, uint64_t item_size, const crypto_ptr_t &crypto_ptr, const ypc::bytes &public_key) { for(uint8_t i = 0; i < count; ++i) { oram_ntt::block_t b_block; @@ -103,7 +103,7 @@ uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { // leftmost leaf node uint32_t leftmost_leaf_index = (1 << level_num_L) - 1; if(bucket_index >= leftmost_leaf_index) { - return bucket_index - leftmost_leaf_index + 1; + return bucket_index - leftmost_leaf_index + 1; } // randomly select a path to the leaf node @@ -112,7 +112,7 @@ uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { std::uniform_int_distribution<> dis(0, 1); if(dis(gen) == 0) { - return get_leaf_label(2 * bucket_index + 1, level_num_L); + return get_leaf_label(2 * bucket_index + 1, level_num_L); } return get_leaf_label(2 * bucket_index + 2, level_num_L); } @@ -121,7 +121,7 @@ void push_real_block(std::vector& bucket_array, ypc::bytes &d uint32_t& block_id_value, uint32_t bucket_index, std::vector &position_map_array, uint8_t level_num_L, std::vector &batch, uint32_t &batch_str_size, - uint64_t item_num_each_batch, size_t item_size, + uint64_t item_num_each_batch, uint64_t item_size, const crypto_ptr_t &crypto_ptr, const ypc::bytes &public_key) { oram_ntt::block_t b_block; uint32_t valid_item_num = batch.size(); @@ -184,7 +184,7 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi boost::progress_display pd(item_number); uint counter = 0; size_t batch_size = 0; - size_t item_size = item_data.size(); + uint64_t item_size = item_data.size(); // the number of batch uint64_t batch_num = 0; @@ -214,15 +214,14 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi if(item_num_each_batch != 0) { item_num_array.push_back(item_num_each_batch); item_num_each_batch = 0; - batch_size = 0; - ++batch_num; + batch_size = 0; + ++batch_num; } - - assert(batch_num == item_num_array.size()); + LOG(INFO) << "batch_num: " << batch_num; - // build id map - LOG(INFO) << "build id_map"; + // create id map + LOG(INFO) << "create id_map"; reader.reset_for_read(); counter = 0; @@ -259,24 +258,25 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi ++counter; } - assert(id_map_array.size() == item_number); oram_ntt::id_map_t id_map_pkg; id_map_pkg.set(id_map_array); bytes id_map_bytes = make_bytes::for_package(id_map_pkg); - // build header - LOG(INFO) << "build header"; - oram::header osf_header{}; + // create header + LOG(INFO) << "create header"; + + ypc::oram::header osf_header{}; osf_header.block_num = batch_num; uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; osf_header.id_map_filepos = sizeof(osf_header); - osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size(); - // write header, id map, invalid position map - LOG(INFO) << "write header, id map, invalid position map"; + + + // write header, id map + LOG(INFO) << "write header and id map"; std::fstream osf(oram_sealed_file_path, std::ios::out | std::ios::binary); if(!osf.is_open()) { throw std::runtime_error("Failed to create oram sealed file: " + oram_sealed_file_path); @@ -287,12 +287,11 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi osf.write((char *)id_map_bytes.data(), id_map_bytes.size()); - std::vector data_hash_array; - - // write ORAM tree - item_num_each_batch = item_num_array.front(); LOG(INFO) << "write ORAM tree"; + std::vector data_hash_array; + item_num_each_batch = item_num_array.front(); + // from which bucket to start writing real blocks uint8_t lastbucket_realblocknum = osf_header.block_num % oram::BucketSizeZ; uint32_t bucket_index = 0; // bucket index in ORAM tree @@ -372,6 +371,10 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi exit(1); } + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + osf.write((char *)encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); data_hash_array.push_back(data_hash); @@ -445,7 +448,6 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi // write merkle tree - assert(data_hash_array.size() == osf_header.bucket_num_N); osf_header.merkle_tree_filepos = osf.tellp(); for(int i = (1 << osf_header.level_num_L) - 2; i >= 0; --i) { @@ -470,6 +472,20 @@ uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugi osf.close(); + LOG(INFO) << "osf_header.block_num :" << osf_header.block_num; + LOG(INFO) << "osf_header.bucket_num_N :" << osf_header.bucket_num_N; + LOG(INFO) << "osf_header.level_num_L :" << osf_header.level_num_L; + LOG(INFO) << "osf_header.bucket_str_size :" << osf_header.bucket_str_size; + LOG(INFO) << "osf_header.batch_str_size :" << osf_header.batch_str_size; + LOG(INFO) << "osf_header.id_map_filepos :" << osf_header.id_map_filepos; + LOG(INFO) << "osf_header.oram_tree_filepos :" << osf_header.oram_tree_filepos; + LOG(INFO) << "osf_header.position_map_filepos :" << osf_header.position_map_filepos; + LOG(INFO) << "osf_header.merkle_tree_filepos :" << osf_header.merkle_tree_filepos; + LOG(INFO) << "osf_header.stash_filepos :" << osf_header.stash_filepos; + LOG(INFO) << "osf_header.stash_size :" << osf_header.stash_size; + LOG(INFO) << "osf_header.item_num_each_batch :" << osf_header.item_num_each_batch; + LOG(INFO) << "osf_header.item_size :" << osf_header.item_size; + return 0; } diff --git a/toolkit/ydump/nouse_bridge.cpp b/toolkit/ydump/nouse_bridge.cpp index be6dfc15..a76c2226 100644 --- a/toolkit/ydump/nouse_bridge.cpp +++ b/toolkit/ydump/nouse_bridge.cpp @@ -54,6 +54,12 @@ uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len); + +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + } @@ -149,4 +155,14 @@ uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { return 0; +} + + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len) { + return 0; +} + +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size) { + return 0; } \ No newline at end of file diff --git a/ypc/core/src/core/oram_sealed_file.cpp b/ypc/core/src/core/oram_sealed_file.cpp index ebcbf9a6..cfffbeee 100644 --- a/ypc/core/src/core/oram_sealed_file.cpp +++ b/ypc/core/src/core/oram_sealed_file.cpp @@ -11,6 +11,10 @@ simple_oram_sealed_file::simple_oram_sealed_file(const std::string &file_path) void simple_oram_sealed_file::reset() { m_file.reset(); } +void simple_oram_sealed_file::open_for_write() { m_file.open_for_write(); } + +void simple_oram_sealed_file::open_for_read() { m_file.open_for_read(); } + bool simple_oram_sealed_file::download_oram_params(uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size) { @@ -23,13 +27,13 @@ bool simple_oram_sealed_file::download_oram_params(uint32_t *block_num, uint32_t return true; } -bool simple_oram_sealed_file::get_block_id(bytes &item_index_field_hash, uint32_t *block_id) { +bool simple_oram_sealed_file::get_block_id(const bytes &item_index_field_hash, uint32_t *block_id) { return m_file.get_block_id(item_index_field_hash, block_id); } bool simple_oram_sealed_file::download_position_map(memref &posmap) {return m_file.download_position_map(posmap); } -bool simple_oram_sealed_file::update_position_map(uint8_t * position_map, uint32_t len) { +bool simple_oram_sealed_file::update_position_map(const uint8_t * position_map, uint32_t len) { return m_file.update_position_map(position_map, len); } @@ -37,13 +41,13 @@ bool simple_oram_sealed_file::download_path(uint32_t leaf, memref &en_path) { return m_file.download_path(leaf, en_path); } -bool simple_oram_sealed_file::upload_path(uint32_t leaf, uint8_t * encrpypted_path, uint32_t len) { +bool simple_oram_sealed_file::upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) { return m_file.upload_path(leaf, encrpypted_path, len); } bool simple_oram_sealed_file::download_stash(memref &st) { return m_file.download_stash(st); } -bool simple_oram_sealed_file::update_stash(uint8_t * stash, uint32_t len) { +bool simple_oram_sealed_file::update_stash(const uint8_t * stash, uint32_t len) { return m_file.update_stash(stash, len); } @@ -55,7 +59,7 @@ bool simple_oram_sealed_file::download_merkle_hash(uint32_t leaf, memref &merkle return m_file.download_merkle_hash(leaf, merkle_hash); } -bool simple_oram_sealed_file::update_merkle_hash(uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { +bool simple_oram_sealed_file::update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) { return m_file.update_merkle_hash(leaf, merkle_hash, len); } diff --git a/ypc/core/src/core/privacy_data_reader.cpp b/ypc/core/src/core/privacy_data_reader.cpp index bd31e31f..90d64bf1 100644 --- a/ypc/core/src/core/privacy_data_reader.cpp +++ b/ypc/core/src/core/privacy_data_reader.cpp @@ -119,8 +119,7 @@ std::string privacy_data_reader::get_item_index_field() { if (m_get_item_index_field == nullptr) { return std::string(); } - // We use static buf here to optimize memory usage. - // TODO:max_field_size + // We use static buf here to optimize memory usage. char buf[::ypc::utc::max_item_size] = {0}; int len = ::ypc::utc::max_item_size; diff --git a/ypc/core/src/core_t/analyzer/CMakeLists.txt b/ypc/core/src/core_t/analyzer/CMakeLists.txt index 6747af44..ba571bb7 100644 --- a/ypc/core/src/core_t/analyzer/CMakeLists.txt +++ b/ypc/core/src/core_t/analyzer/CMakeLists.txt @@ -9,24 +9,21 @@ set(src ${FF_PATH}/src/net/common/archive.cpp ${PROJECT_SOURCE_DIR}/hpda/src/engine/engine.cpp ${PROJECT_SOURCE_DIR}/hpda/src/engine/functor.cpp - # ${PROJECT_SOURCE_DIR}/ypc/core/src/core_t/analyzer/oblivious_functions.asm ) -add_custom_target(asm_target ALL COMMAND nasm -f elf64 ${ASM_FILE} -o ${PROJECT_SOURCE_DIR}/bin/oblivious_functions.o) +# TODO:oblivious_functions.o是需要作为enclave的可信库还是非可信库？ +add_custom_target(asm_target ALL COMMAND nasm -f elf64 ${ASM_FILE} -o ${PROJECT_SOURCE_DIR}/lib/oblivious_functions.o) -# set(CMAKE_ASM_NASM_COMPILER nasm) -# add_library(oblivious_functions -# SHARED -# ASM_FILE +# add_custom_target( +# clean_oblivious_functions +# COMMAND ${CMAKE_COMMAND} -E remove +# ${PROJECT_SOURCE_DIR}/lib/oblivious_functions.o # ) -# set_property(SOURCE ASM_FILE PROPERTY LANGUAGE C) -# set(CMAKE_ASM_NASM_FLAGS "-f elf64") +# add_dependencies(clean clean_oblivious_functions) + -# SET(CMAKE_ASM_NASM_SOURCE_FILE_EXTENSIONS asm) -# ENABLE_LANGUAGE(ASM_NASM) -# SET(ASM_NASMFLAGS "-f elf64 -g") add_trusted_library(analyzer_t analyzer @@ -47,7 +44,7 @@ target_include_directories(analyzer_t PUBLIC "$" ) -target_link_libraries(analyzer_t PUBLIC stbox_channel_t stbox_crypto_t ${PROJECT_SOURCE_DIR}/bin/oblivious_functions.o) +target_link_libraries(analyzer_t PUBLIC stbox_channel_t stbox_crypto_t ${PROJECT_SOURCE_DIR}/lib/oblivious_functions.o) install(TARGETS analyzer_t EXPORT mod_analyzer_t diff --git a/ypc/core/src/core_t/analyzer/oblivious_functions.asm b/ypc/core/src/core_t/analyzer/oblivious_functions.asm index e7145880..bc38ae5d 100644 --- a/ypc/core/src/core_t/analyzer/oblivious_functions.asm +++ b/ypc/core/src/core_t/analyzer/oblivious_functions.asm @@ -2,40 +2,146 @@ BITS 64 section .text global omove + global oset_value + global oset_flag + global oset_bytes omove: - ;command: - ;omove(i,&(array[i]),loc,leaf,newLabel) - ;Linux : rdi,rsi,rdx,rcx,r8,r9 + ;command: + ;omove(i,&(array[i]),loc,leaf,newLabel) + ;Linux : rdi,rsi,rdx,rcx,r8,r9 - ;If the callee wishes to use registers RBP, RBX, and R12-R15, - ;it must restore their original values before returning control to the caller. - ;All others must be saved by the caller if it wishes to preserve their values + ;If the callee wishes to use registers RBP, RBX, and R12-R15, + ;it must restore their original values before returning control to the caller. + ;All others must be saved by the caller if it wishes to preserve their values - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 - mov r13d, dword [rsi] ; r13d holds array[i] + mov ebx, dword [rcx] ; Move value in leaf to rbx + mov r13d, dword [rsi] ; r13d holds array[i] - mov ebx, dword [rcx] ; Move value in leaf to rbx - cmp edi, edx ; Compare i and loc + cmp edi, edx ; Compare i and loc - ;if i == loc - cmovz ebx, dword [rsi] ; move value pointed by rdx to rbx (array[i] to rbx ) - cmovz r13d, r8d ; move newLabel to array[i] + ;if i == loc + cmovz ebx, dword [rsi] ; move value pointed by rdx to rbx (array[i] to rbx ) + cmovz r13d, r8d ; move newLabel to array[i] - mov dword [rcx], ebx ; Push in leaflabel/prevleaf to leaf - mov dword [rsi], r13d ; Push in newLabel/array[i] to array[i] + mov dword [rcx], ebx ; Push in leaflabel/prevleaf to leaf + mov dword [rsi], r13d ; Push in newLabel/array[i] to array[i] - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx - ret \ No newline at end of file + ret + +oset_value: + ; oset_value(&dest, value, flag); + ; Linux : rdi,rsi,rdx,rcx,r8,r9 + ; Callee-saved : RBP, RBX, and R12–R15 + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov ebx, dword [rdi] + + cmp edx, 1 + + cmovz ebx, esi + + mov dword [rdi], ebx + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret + +oset_flag: + ; oset_flag(&dest_flag, src_flag, flag); + ; Linux : rdi,rsi,rdx,rcx,r8,r9 + ; Callee-saved : RBP, RBX, and R12–R15 + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov bl, byte [rdi] + + cmp dl, 1 + + cmovz bx, si + + mov byte [rdi], bl + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret + +oset_bytes: + ; oset_bytes(dest_bytes.data(), src_bytes.data(), bytes_size, flag) + ; Linux : rdi,rsi,rdx,rcx,r8,r9 + ; Callee-saved : RBP, RBX, and R12–R15 + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + + ; RCX will be lost for loop, store flag from rcx to rbp (1 byte , so bpl) + mov bpl, cl + + ; Oblivious evaluation of flag + cmp bpl, 1 + + mov ecx, edx + + loop_start: + cmp bpl, 1 + + mov r14b, byte [rdi] + mov r15b, byte [rsi] + + cmovz r14, r15 + + mov byte [rdi], r14b + + inc rdi + inc rsi + + loop loop_start + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret \ No newline at end of file