Skip to content
This repository was archived by the owner on Sep 9, 2025. It is now read-only.

Commit c344ae9

Browse files
gglin001gglin001
authored
Merge pull request PaddlePaddle#830 from graphcore/develop_to_develop-ipu_e3658aecb5660a2c63d321d5d87d5afb2298191d
[AUTO PR] Pulling [develop] into [develop-ipu] [2022-06-23 18:39:00+08:00]
2 parents e3658ae + e3d94fc commit c344ae9

File tree

96 files changed

+5859
-2026
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+5859
-2026
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,4 +80,3 @@ repos:
8080
hooks:
8181
- id: cmakelint
8282
args: [--config=./tools/codestyle/.cmakelintrc]
83-
# exclude files which need to be fixed

‎paddle/fluid/distributed/test/graph_node_test.cc‎

Lines changed: 47 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -228,32 +228,36 @@ void testFeatureNodeSerializeFloat64() {
228228
// void testCache();
229229
void testGraphToBuffer();
230230

231-
std::string edges[] = {
232-
std::string("37\t45\t0.34"), std::string("37\t145\t0.31"),
233-
std::string("37\t112\t0.21"), std::string("96\t48\t1.4"),
234-
std::string("96\t247\t0.31"), std::string("96\t111\t1.21"),
235-
std::string("59\t45\t0.34"), std::string("59\t145\t0.31"),
236-
std::string("59\t122\t0.21"), std::string("97\t48\t0.34"),
237-
std::string("97\t247\t0.31"), std::string("97\t111\t0.21")};
231+
const char* edges[] = {"37\t45\t0.34",
232+
"37\t145\t0.31",
233+
"37\t112\t0.21",
234+
"96\t48\t1.4",
235+
"96\t247\t0.31",
236+
"96\t111\t1.21",
237+
"59\t45\t0.34",
238+
"59\t145\t0.31",
239+
"59\t122\t0.21",
240+
"97\t48\t0.34",
241+
"97\t247\t0.31",
242+
"97\t111\t0.21"};
238243
char edge_file_name[] = "edges.txt";
239244

240-
std::string nodes[] = {
241-
std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"),
242-
std::string("user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd"),
243-
std::string("user\t59\ta 0.11\tb 11 14"),
244-
std::string("user\t97\ta 0.11\tb 12 11"),
245-
std::string("item\t45\ta 0.21"),
246-
std::string("item\t145\ta 0.21"),
247-
std::string("item\t112\ta 0.21"),
248-
std::string("item\t48\ta 0.21"),
249-
std::string("item\t247\ta 0.21"),
250-
std::string("item\t111\ta 0.21"),
251-
std::string("item\t46\ta 0.21"),
252-
std::string("item\t146\ta 0.21"),
253-
std::string("item\t122\ta 0.21"),
254-
std::string("item\t49\ta 0.21"),
255-
std::string("item\t248\ta 0.21"),
256-
std::string("item\t113\ta 0.21")};
245+
const char* nodes[] = {"user\t37\ta 0.34\tb 13 14\tc hello\td abc",
246+
"user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd",
247+
"user\t59\ta 0.11\tb 11 14",
248+
"user\t97\ta 0.11\tb 12 11",
249+
"item\t45\ta 0.21",
250+
"item\t145\ta 0.21",
251+
"item\t112\ta 0.21",
252+
"item\t48\ta 0.21",
253+
"item\t247\ta 0.21",
254+
"item\t111\ta 0.21",
255+
"item\t46\ta 0.21",
256+
"item\t146\ta 0.21",
257+
"item\t122\ta 0.21",
258+
"item\t49\ta 0.21",
259+
"item\t248\ta 0.21",
260+
"item\t113\ta 0.21"};
257261
char node_file_name[] = "nodes.txt";
258262

259263
void prepare_file(char file_name[], bool load_edge) {
@@ -335,7 +339,8 @@ ::paddle::distributed::PSParameter GetWorkerProto() {
335339

336340
/*-------------------------------------------------------------------------*/
337341

338-
std::string ip_ = "127.0.0.1", ip2 = "127.0.0.1";
342+
const char* ip_ = "127.0.0.1";
343+
const char* ip2 = "127.0.0.1";
339344
uint32_t port_ = 5209, port2 = 5210;
340345

341346
std::vector<std::string> host_sign_list_;
@@ -382,8 +387,10 @@ void RunServer2() {
382387
}
383388

384389
void RunClient(
385-
std::map<uint64_t, std::vector<paddle::distributed::Region>>& dense_regions,
386-
int index, paddle::distributed::PsBaseService* service) {
390+
const std::map<uint64_t, std::vector<paddle::distributed::Region>>&
391+
dense_regions,
392+
int index,
393+
paddle::distributed::PsBaseService* service) {
387394
::paddle::distributed::PSParameter worker_proto = GetWorkerProto();
388395
paddle::distributed::PaddlePSEnvironment _ps_env;
389396
auto servers_ = host_sign_list_.size();
@@ -533,8 +540,8 @@ void RunBrpcPushSparse() {
533540
VLOG(0) << "second bound";
534541
client1.load_node_file(std::string("user"), std::string(node_file_name));
535542
client1.load_node_file(std::string("item"), std::string(node_file_name));
536-
client1.load_edge_file(std::string("user2item"), std::string(edge_file_name),
537-
0);
543+
client1.load_edge_file(
544+
std::string("user2item"), std::string(edge_file_name), 0);
538545
nodes.clear();
539546
VLOG(0) << "start to pull graph list";
540547
nodes = client1.pull_graph_list(std::string("user"), 0, 1, 4, 1);
@@ -551,8 +558,8 @@ void RunBrpcPushSparse() {
551558
std::cout << "check pull graph list by step " << test_step << std::endl;
552559
for (int server_id = 0; server_id < 2; server_id++) {
553560
for (int start_step = 0; start_step < test_step; start_step++) {
554-
nodes = client1.pull_graph_list(std::string("item"), server_id,
555-
start_step, 12, test_step);
561+
nodes = client1.pull_graph_list(
562+
std::string("item"), server_id, start_step, 12, test_step);
556563
for (auto g : nodes) {
557564
count_item_nodes.insert(g.get_id());
558565
}
@@ -570,14 +577,15 @@ void RunBrpcPushSparse() {
570577
std::vector<int64_t> node_ids;
571578
node_ids.push_back(96);
572579
node_ids.push_back(37);
573-
res = client1.batch_sample_neighbors(std::string("user2item"), node_ids, 4,
574-
true, false);
580+
res = client1.batch_sample_neighbors(
581+
std::string("user2item"), node_ids, 4, true, false);
575582

576583
ASSERT_EQ(res.first[1].size(), 1);
577584
std::vector<int64_t> nodes_ids = client2.random_sample_nodes("user", 0, 6);
578585
ASSERT_EQ(nodes_ids.size(), 2);
579-
ASSERT_EQ(true, (nodes_ids[0] == 59 && nodes_ids[1] == 37) ||
580-
(nodes_ids[0] == 37 && nodes_ids[1] == 59));
586+
ASSERT_EQ(true,
587+
(nodes_ids[0] == 59 && nodes_ids[1] == 37) ||
588+
(nodes_ids[0] == 37 && nodes_ids[1] == 59));
581589

582590
VLOG(0) << "start to test get node feat";
583591
// Test get node feat
@@ -598,14 +606,14 @@ void RunBrpcPushSparse() {
598606

599607
node_feat[1][0] = "helloworld";
600608

601-
client1.set_node_feat(std::string("user"), node_ids, feature_names,
602-
node_feat);
609+
client1.set_node_feat(
610+
std::string("user"), node_ids, feature_names, node_feat);
603611

604612
// sleep(5);
605613
node_feat =
606614
client1.get_node_feat(std::string("user"), node_ids, feature_names);
607615
VLOG(0) << "get_node_feat: " << node_feat[1][0];
608-
ASSERT_TRUE(node_feat[1][0] == "helloworld");
616+
ASSERT_EQ(node_feat[1][0], "helloworld");
609617

610618
// Test string
611619
node_ids.clear();
@@ -698,7 +706,7 @@ void testGraphToBuffer() {
698706
s.set_feature(0, std::string("hhhh"));
699707
s.set_id(65);
700708
int size = s.get_size(true);
701-
char str[size];
709+
char str[size]; // NOLINT
702710
s.to_buffer(str, true);
703711
s1.recover_from_buffer(str);
704712
ASSERT_EQ(s.get_id(), s1.get_id());

‎paddle/fluid/framework/archive.h‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
14-
1514
#pragma once
1615

1716
#if defined _WIN32 || defined __APPLE__
@@ -92,17 +91,21 @@ class ArchiveBase {
9291

9392
char* Buffer() { return buffer_; }
9493

95-
void SetReadBuffer(char* buffer, size_t length,
94+
void SetReadBuffer(char* buffer,
95+
size_t length,
9696
std::function<void(char*)>&& deleter) {
9797
SetBuffer(buffer, length, length, std::move(deleter));
9898
}
9999

100-
void SetWriteBuffer(char* buffer, size_t capacity,
100+
void SetWriteBuffer(char* buffer,
101+
size_t capacity,
101102
std::function<void(char*)>&& deleter) {
102103
SetBuffer(buffer, 0, capacity, std::move(deleter));
103104
}
104105

105-
void SetBuffer(char* buffer, size_t length, size_t capacity,
106+
void SetBuffer(char* buffer,
107+
size_t length,
108+
size_t capacity,
106109
std::function<void(char*)>&& deleter) {
107110
CHECK(length <= capacity);
108111
FreeBuffer();
@@ -324,9 +327,10 @@ class Archive<BinaryArchiveType> : public ArchiveBase {
324327
size_t temp = Limit() - Finish();
325328
int len = snprintf(Finish(), temp, fmt, args...);
326329
CHECK(len >= 0); // NOLINT
327-
if ((size_t)len >= temp) {
330+
if (static_cast<size_t>(len) >= temp) {
328331
PrepareWrite(len + 1);
329-
CHECK(snprintf(Finish(), (size_t)len + 1, fmt, args...) == len);
332+
CHECK(snprintf(Finish(), static_cast<size_t>(len) + 1, fmt, args...) ==
333+
len);
330334
}
331335
AdvanceFinish(len);
332336
}
@@ -351,7 +355,7 @@ Archive<AR>& operator>>(Archive<AR>& ar, T (&p)[N]) {
351355
template <class AR, class T>
352356
Archive<AR>& operator<<(Archive<AR>& ar, const std::vector<T>& p) {
353357
#ifdef _LINUX
354-
ar << (size_t)p.size();
358+
ar << static_cast<size_t>(p.size());
355359
#else
356360
ar << (uint64_t)p.size();
357361
#endif
@@ -377,7 +381,7 @@ Archive<AR>& operator>>(Archive<AR>& ar, std::vector<T>& p) {
377381
template <class AR, class T>
378382
Archive<AR>& operator<<(Archive<AR>& ar, const std::valarray<T>& p) {
379383
#ifdef _LINUX
380-
ar << (size_t)p.size();
384+
ar << static_cast<size_t>(p.size());
381385
#else
382386
ar << (uint64_t)p.size();
383387
#endif
@@ -402,7 +406,7 @@ Archive<AR>& operator>>(Archive<AR>& ar, std::valarray<T>& p) {
402406

403407
inline BinaryArchive& operator<<(BinaryArchive& ar, const std::string& s) {
404408
#ifdef _LINUX
405-
ar << (size_t)s.length();
409+
ar << static_cast<size_t>(s.length());
406410
#else
407411
ar << (uint64_t)s.length();
408412
#endif
@@ -482,28 +486,32 @@ Archive<AR>& operator<<(Archive<AR>& ar, const std::tuple<T...>& x) {
482486

483487
#ifdef _LINUX
484488
template <class AR, class... T>
485-
Archive<AR>& DeserializeTuple(Archive<AR>& ar, std::tuple<T...>& x, // NOLINT
489+
Archive<AR>& DeserializeTuple(const Archive<AR>& ar,
490+
std::tuple<T...>& x, // NOLINT
486491
std::integral_constant<size_t, 0> n) {
487492
return ar;
488493
}
489494
#else
490495
template <class AR, class... T>
491-
Archive<AR>& DeserializeTuple(Archive<AR>& ar, std::tuple<T...>& x, // NOLINT
496+
Archive<AR>& DeserializeTuple(const Archive<AR>& ar,
497+
std::tuple<T...>& x, // NOLINT
492498
std::integral_constant<uint64_t, 0> n) {
493499
return ar;
494500
}
495501
#endif
496502

497503
#ifdef _LINUX
498504
template <class AR, class... T, size_t N>
499-
Archive<AR>& DeserializeTuple(Archive<AR>& ar, std::tuple<T...>& x, // NOLINT
505+
Archive<AR>& DeserializeTuple(const Archive<AR>& ar,
506+
std::tuple<T...>& x, // NOLINT
500507
std::integral_constant<size_t, N> n) {
501508
return DeserializeTuple(ar, x, std::integral_constant<size_t, N - 1>()) >>
502509
std::get<N - 1>(x);
503510
}
504511
#else
505512
template <class AR, class... T, uint64_t N>
506-
Archive<AR>& DeserializeTuple(Archive<AR>& ar, std::tuple<T...>& x, // NOLINT
513+
Archive<AR>& DeserializeTuple(const Archive<AR>& ar,
514+
std::tuple<T...>& x, // NOLINT
507515
std::integral_constant<uint64_t, N> n) {
508516
return DeserializeTuple(ar, x, std::integral_constant<uint64_t, N - 1>()) >>
509517
std::get<N - 1>(x);
@@ -529,7 +537,7 @@ Archive<AR>& operator>>(Archive<AR>& ar, std::tuple<T...>& x) {
529537
template <class AR, class KEY, class VALUE, class... ARGS> \
530538
Archive<AR>& operator<<(Archive<AR>& ar, \
531539
const MAP_TYPE<KEY, VALUE, ARGS...>& p) { \
532-
ar << (size_t)p.size(); \
540+
ar << static_cast<size_t>(p.size()); \
533541
for (auto it = p.begin(); it != p.end(); ++it) { \
534542
ar << *it; \
535543
} \
@@ -579,7 +587,7 @@ ARCHIVE_REPEAT(std::unordered_multimap, p.reserve(size))
579587
#define ARCHIVE_REPEAT(SET_TYPE, RESERVE_STATEMENT) \
580588
template <class AR, class KEY, class... ARGS> \
581589
Archive<AR>& operator<<(Archive<AR>& ar, const SET_TYPE<KEY, ARGS...>& p) { \
582-
ar << (size_t)p.size(); \
590+
ar << static_cast<size_t>(p.size()); \
583591
for (auto it = p.begin(); it != p.end(); ++it) { \
584592
ar << *it; \
585593
} \

‎paddle/fluid/framework/heter_util.h‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,17 @@ distributed under the License is distributed on an "AS IS" BASIS,
1111
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
See the License for the specific language governing permissions and
1313
limitations under the License. */
14-
1514
#pragma once
1615

1716
#include <fstream>
17+
#include <map>
1818
#include <memory>
1919
#include <mutex> // NOLINT
2020
#include <string>
2121
#include <thread> // NOLINT
2222
#include <unordered_map> // NOLINT
2323
#include <unordered_set> // NOLINT
24+
#include <utility>
2425
#include <vector>
2526
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
2627
#include "bthread/bthread.h"
@@ -76,9 +77,13 @@ class HeterTask {
7677
<< std::endl;
7778
}
7879
}
79-
void PackTask(Scope* scope, int taskid, DataFeed* reader, int cur_batch,
80+
void PackTask(Scope* scope,
81+
int taskid,
82+
DataFeed* reader,
83+
int cur_batch,
8084
const ProgramDesc& program);
81-
void PackGpuTask(Scope* thread_scope, DataFeed* reader,
85+
void PackGpuTask(Scope* thread_scope,
86+
DataFeed* reader,
8287
const ProgramDesc& program);
8388

8489
Scope* scope_{nullptr};
@@ -145,7 +150,7 @@ class HeterObjectPool {
145150

146151
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
147152
struct BthreadMutextGuard {
148-
BthreadMutextGuard(bthread_mutex_t* rho) {
153+
explicit BthreadMutextGuard(bthread_mutex_t* rho) {
149154
mutex_ = rho;
150155
bthread_mutex_lock(mutex_);
151156
}
@@ -220,7 +225,7 @@ class HeterList {
220225

221226
void SetCap(int num) { cap_ = num; }
222227

223-
bool TryPut(K& key, T& value) {
228+
bool TryPut(const K& key, const T& value) {
224229
std::unique_lock<std::mutex> lock(mutex_);
225230
cond_.wait(lock, [this] { return size < cap_; });
226231
if (task_map_.find(key) != task_map_.end()) {
@@ -236,7 +241,7 @@ class HeterList {
236241
}
237242
}
238243

239-
bool Put(K& key, T& value) {
244+
bool Put(const K& key, const T& value) {
240245
std::unique_lock<std::mutex> lock(mutex_);
241246
cond_.wait(lock, [this] { return size < cap_; });
242247
HeterNode<K, T>* node = new HeterNode<K, T>;

0 commit comments

Comments
 (0)