Skip to content

Commit b23ff6d

Browse files
udesouudesou
committed
Minor fixes to support sticky
1 parent 31d7be0 commit b23ff6d

File tree

5 files changed

+91
-175
lines changed

5 files changed

+91
-175
lines changed

‎src/gc-mmtk.c

+7-77
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,7 @@ inline void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) {
878878
STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
879879
if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) {
880880
mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj);
881+
}
881882
}
882883

883884
JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int osize, size_t align, void *ty)
@@ -1094,6 +1095,11 @@ void jl_gc_notify_image_load(const char* img_data, size_t len)
10941095
mmtk_set_vm_space((void*)img_data, len);
10951096
}
10961097

1098+
void jl_gc_notify_image_alloc(const char* img_data, size_t len)
1099+
{
1100+
mmtk_immortal_region_post_alloc((void*)img_data, len);
1101+
}
1102+
10971103
// ========================================================================= //
10981104
// Code specific to stock that is not supported by MMTk
10991105
// ========================================================================= //
@@ -1143,7 +1149,7 @@ JL_DLLEXPORT void jl_gc_queue_root(const struct _jl_value_t *ptr) JL_NOTSAFEPOIN
11431149
{
11441150
jl_task_t *ct = jl_current_task;
11451151
jl_ptls_t ptls = ct->ptls;
1146-
mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, parent, (const void*) 0);
1152+
mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, ptr, (const void*) 0);
11471153
}
11481154

11491155
JL_DLLEXPORT void jl_gc_queue_multiroot(const struct _jl_value_t *root, const void *stored,
@@ -1225,82 +1231,6 @@ JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p)
12251231
return NULL;
12261232
}
12271233

1228-
JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destdata,
1229-
jl_genericmemory_t *src, char* srcdata,
1230-
size_t n) JL_NOTSAFEPOINT
1231-
{
1232-
jl_datatype_t *dt = (jl_datatype_t*)jl_typetagof(dest);
1233-
if (dt != (jl_datatype_t*)jl_typetagof(src))
1234-
jl_exceptionf(jl_argumenterror_type, "jl_genericmemory_copyto requires source and dest to have same type");
1235-
const jl_datatype_layout_t *layout = dt->layout;
1236-
if (layout->flags.arrayelem_isboxed) {
1237-
_Atomic(void*) * dest_p = (_Atomic(void*)*)destdata;
1238-
_Atomic(void*) * src_p = (_Atomic(void*)*)srcdata;
1239-
jl_value_t *owner = jl_genericmemory_owner(dest);
1240-
jl_gc_wb(owner, NULL);
1241-
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
1242-
jl_value_t *src_owner = jl_genericmemory_owner(src);
1243-
ssize_t done = 0;
1244-
if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) {
1245-
if (dest_p < src_p || dest_p > src_p + n) {
1246-
for (; done < n; done++) { // copy forwards
1247-
void *val = jl_atomic_load_relaxed(src_p + done);
1248-
jl_atomic_store_release(dest_p + done, val);
1249-
// `val` is young or old-unmarked
1250-
if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
1251-
jl_gc_queue_root(owner);
1252-
break;
1253-
}
1254-
}
1255-
src_p += done;
1256-
dest_p += done;
1257-
} else {
1258-
for (; done < n; done++) { // copy backwards
1259-
void *val = jl_atomic_load_relaxed(src_p + n - done - 1);
1260-
jl_atomic_store_release(dest_p + n - done - 1, val);
1261-
// `val` is young or old-unmarked
1262-
if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
1263-
jl_gc_queue_root(owner);
1264-
break;
1265-
}
1266-
}
1267-
}
1268-
n -= done;
1269-
}
1270-
}
1271-
return memmove_refs(dest_p, src_p, n);
1272-
}
1273-
size_t elsz = layout->size;
1274-
char *src_p = srcdata;
1275-
int isbitsunion = layout->flags.arrayelem_isunion;
1276-
if (isbitsunion) {
1277-
char *sourcetypetagdata = jl_genericmemory_typetagdata(src);
1278-
char *desttypetagdata = jl_genericmemory_typetagdata(dest);
1279-
memmove(desttypetagdata+(size_t)destdata, sourcetypetagdata+(size_t)srcdata, n);
1280-
srcdata = (char*)src->ptr + elsz*(size_t)srcdata;
1281-
destdata = (char*)dest->ptr + elsz*(size_t)destdata;
1282-
}
1283-
if (layout->first_ptr != -1) {
1284-
memmove_refs((_Atomic(void*)*)destdata, (_Atomic(void*)*)srcdata, n * elsz / sizeof(void*));
1285-
jl_value_t *owner = jl_genericmemory_owner(dest);
1286-
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
1287-
jl_value_t *src_owner = jl_genericmemory_owner(src);
1288-
if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) {
1289-
dt = (jl_datatype_t*)jl_tparam1(dt);
1290-
for (size_t done = 0; done < n; done++) { // copy forwards
1291-
char* s = (char*)src_p+done*elsz;
1292-
if (*((jl_value_t**)s+layout->first_ptr) != NULL)
1293-
jl_gc_queue_multiroot(owner, s, dt);
1294-
}
1295-
}
1296-
}
1297-
}
1298-
else {
1299-
memmove(destdata, srcdata, n * elsz);
1300-
}
1301-
}
1302-
1303-
13041234
#ifdef __cplusplus
13051235
}
13061236
#endif

‎src/gc-wb-mmtk.h

+14
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ extern "C" {
1313

1414
extern void mmtk_object_reference_write_post(void* mutator, const void* parent, const void* ptr);
1515
extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, const void* ptr);
16+
extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;
1617

1718
#define MMTK_OBJECT_BARRIER (1)
1819
// Stickyimmix needs write barrier. Immix does not need write barrier.
@@ -64,6 +65,19 @@ STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_
6465
mmtk_gc_wb_fast(parent, (void*)0);
6566
}
6667

68+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
69+
jl_genericmemory_t *src, _Atomic(void*) * src_p,
70+
size_t* n) JL_NOTSAFEPOINT
71+
{
72+
mmtk_gc_wb_fast(dest_owner, (void*)0);
73+
}
74+
75+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
76+
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
77+
{
78+
mmtk_gc_wb_fast(owner, (void*)0);
79+
}
80+
6781

6882
#ifdef __cplusplus
6983
}

‎src/gc-wb-stock.h

+54
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,60 @@ STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_
4343
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
4444
}
4545

46+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
47+
jl_genericmemory_t *src, _Atomic(void*) * src_p,
48+
size_t* n) JL_NOTSAFEPOINT
49+
{
50+
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
51+
jl_value_t *src_owner = jl_genericmemory_owner(src);
52+
size_t done = 0;
53+
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
54+
if (dest_p < src_p || dest_p > src_p + (*n)) {
55+
for (; done < (*n); done++) { // copy forwards
56+
void *val = jl_atomic_load_relaxed(src_p + done);
57+
jl_atomic_store_release(dest_p + done, val);
58+
// `val` is young or old-unmarked
59+
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
60+
jl_gc_queue_root(dest_owner);
61+
break;
62+
}
63+
}
64+
src_p += done;
65+
dest_p += done;
66+
}
67+
else {
68+
for (; done < (*n); done++) { // copy backwards
69+
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
70+
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
71+
// `val` is young or old-unmarked
72+
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
73+
jl_gc_queue_root(dest_owner);
74+
break;
75+
}
76+
}
77+
}
78+
(*n) -= done;
79+
}
80+
}
81+
}
82+
83+
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
84+
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
85+
{
86+
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
87+
jl_value_t *src_owner = jl_genericmemory_owner(src);
88+
size_t elsz = dt->layout->size;
89+
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
90+
dt = (jl_datatype_t*)jl_tparam1(dt);
91+
for (size_t done = 0; done < n; done++) { // copy forwards
92+
char* s = (char*)src_p+done*elsz;
93+
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
94+
jl_gc_queue_multiroot(owner, s, dt);
95+
}
96+
}
97+
}
98+
}
99+
46100
#ifdef __cplusplus
47101
}
48102
#endif

‎src/julia.h

+12-96
Original file line numberDiff line numberDiff line change
@@ -1182,12 +1182,6 @@ JL_DLLEXPORT void jl_free_stack(void *stkbuf, size_t bufsz);
11821182
// thread-local allocator of the current thread.
11831183
JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value);
11841184

1185-
#ifndef MMTK_GC
1186-
#include "gc-wb-stock.h"
1187-
#else
1188-
#include "gc-wb-mmtk.h"
1189-
#endif
1190-
11911185
JL_DLLEXPORT void jl_gc_safepoint(void);
11921186
JL_DLLEXPORT int jl_safepoint_suspend_thread(int tid, int waitstate);
11931187
JL_DLLEXPORT void jl_safepoint_suspend_all_threads(struct _jl_task_t *ct);
@@ -1250,6 +1244,18 @@ STATIC_INLINE jl_value_t *jl_svecset(
12501244
#define jl_array_maxsize(a) (((jl_array_t*)(a))->ref.mem->length)
12511245
#define jl_array_len(a) (jl_array_ndims(a) == 1 ? jl_array_nrows(a) : jl_array_maxsize(a))
12521246

1247+
JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT;
1248+
#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack))
1249+
1250+
STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
1251+
1252+
// write barriers
1253+
#ifndef MMTK_GC
1254+
#include "gc-wb-stock.h"
1255+
#else
1256+
#include "gc-wb-mmtk.h"
1257+
#endif
1258+
12531259
/*
12541260
how - allocation style
12551261
0 = data is inlined
@@ -1306,94 +1312,6 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set(
13061312
}
13071313
#endif
13081314

1309-
// GC write barriers
1310-
1311-
STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
1312-
{
1313-
// parent and ptr isa jl_value_t*
1314-
if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset
1315-
(jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young
1316-
jl_gc_queue_root((jl_value_t*)parent);
1317-
}
1318-
1319-
STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
1320-
{
1321-
// if ptr is old
1322-
if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) {
1323-
jl_gc_queue_root((jl_value_t*)ptr);
1324-
}
1325-
}
1326-
1327-
STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
1328-
{
1329-
// 3 == GC_OLD_MARKED
1330-
// ptr is an immutable object
1331-
if (__likely(jl_astaggedvalue(parent)->bits.gc != 3))
1332-
return; // parent is young or in remset
1333-
if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3))
1334-
return; // ptr is old and not in remset (thus it does not point to young)
1335-
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr);
1336-
const jl_datatype_layout_t *ly = dt->layout;
1337-
if (ly->npointers)
1338-
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
1339-
}
1340-
1341-
STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
1342-
1343-
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
1344-
jl_genericmemory_t *src, _Atomic(void*) * src_p,
1345-
size_t* n) JL_NOTSAFEPOINT
1346-
{
1347-
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
1348-
jl_value_t *src_owner = jl_genericmemory_owner(src);
1349-
size_t done = 0;
1350-
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
1351-
if (dest_p < src_p || dest_p > src_p + (*n)) {
1352-
for (; done < (*n); done++) { // copy forwards
1353-
void *val = jl_atomic_load_relaxed(src_p + done);
1354-
jl_atomic_store_release(dest_p + done, val);
1355-
// `val` is young or old-unmarked
1356-
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
1357-
jl_gc_queue_root(dest_owner);
1358-
break;
1359-
}
1360-
}
1361-
src_p += done;
1362-
dest_p += done;
1363-
}
1364-
else {
1365-
for (; done < (*n); done++) { // copy backwards
1366-
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
1367-
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
1368-
// `val` is young or old-unmarked
1369-
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
1370-
jl_gc_queue_root(dest_owner);
1371-
break;
1372-
}
1373-
}
1374-
}
1375-
(*n) -= done;
1376-
}
1377-
}
1378-
}
1379-
1380-
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
1381-
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
1382-
{
1383-
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
1384-
jl_value_t *src_owner = jl_genericmemory_owner(src);
1385-
size_t elsz = dt->layout->size;
1386-
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
1387-
dt = (jl_datatype_t*)jl_tparam1(dt);
1388-
for (size_t done = 0; done < n; done++) { // copy forwards
1389-
char* s = (char*)src_p+done*elsz;
1390-
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
1391-
jl_gc_queue_multiroot(owner, s, dt);
1392-
}
1393-
}
1394-
}
1395-
}
1396-
13971315
STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT
13981316
{
13991317
jl_genericmemory_t *m_ = (jl_genericmemory_t*)m;
@@ -2407,8 +2325,6 @@ JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e JL_MAYBE_UNROOTED);
24072325
JL_DLLEXPORT void JL_NORETURN jl_rethrow(void);
24082326
JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED);
24092327
JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e, jl_task_t *ct);
2410-
JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT;
2411-
#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack))
24122328

24132329
extern JL_DLLIMPORT int jl_task_gcstack_offset;
24142330
extern JL_DLLIMPORT int jl_task_ptls_offset;

‎src/staticdata.c

+4-2
Original file line numberDiff line numberDiff line change
@@ -4231,9 +4231,10 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i
42314231
char *sysimg;
42324232
int success = !needs_permalloc;
42334233
ios_seek(f, datastartpos);
4234-
if (needs_permalloc)
4234+
if (needs_permalloc) {
42354235
sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0);
4236-
else
4236+
jl_gc_notify_image_alloc(sysimg, len);
4237+
} else
42374238
sysimg = &f->buf[f->bpos];
42384239
if (needs_permalloc)
42394240
success = ios_readall(f, sysimg, len) == len;
@@ -4355,6 +4356,7 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname)
43554356
ios_seek_end(&f);
43564357
size_t len = ios_pos(&f);
43574358
char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0);
4359+
jl_gc_notify_image_alloc(sysimg, len);
43584360
ios_seek(&f, 0);
43594361
if (ios_readall(&f, sysimg, len) != len)
43604362
jl_errorf("Error reading system image file.");

0 commit comments

Comments
 (0)