Skip to content

Commit 61572a6

Browse files
JeffBezansonJeffBezanson
authored
fix excess array object alignment (#41287)
1 parent bc6da93 commit 61572a6

File tree

6 files changed

+59
-30
lines changed

6 files changed

+59
-30
lines changed

‎src/array.c

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -114,20 +114,23 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
114114
}
115115

116116
int ndimwords = jl_array_ndimwords(ndims);
117-
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
117+
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
118118
if (tot <= ARRAY_INLINE_NBYTES) {
119-
if (isunboxed && elsz >= 4)
120-
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align data area
119+
// align data area
120+
if (tot >= ARRAY_CACHE_ALIGN_THRESHOLD)
121+
tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT);
122+
else if (isunboxed && elsz >= 4)
123+
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT);
121124
size_t doffs = tsz;
122125
tsz += tot;
123-
tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object
126+
// jl_array_t is large enough that objects will always be aligned 16
124127
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
128+
assert(((size_t)a & 15) == 0);
125129
// No allocation or safepoint allowed after this
126130
a->flags.how = 0;
127131
data = (char*)a + doffs;
128132
}
129133
else {
130-
tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT); // align whole object
131134
data = jl_gc_managed_malloc(tot);
132135
// Allocate the Array **after** allocating the data
133136
// to make sure the array is still young
@@ -223,7 +226,7 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
223226
assert(jl_types_equal(jl_tparam0(jl_typeof(data)), jl_tparam0(atype)));
224227

225228
int ndimwords = jl_array_ndimwords(ndims);
226-
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT);
229+
int tsz = sizeof(jl_array_t) + ndimwords * sizeof(size_t) + sizeof(void*);
227230
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
228231
// No allocation or safepoint allowed after this
229232
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
@@ -304,7 +307,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
304307
jl_array_t *a;
305308

306309
int ndimwords = jl_array_ndimwords(1);
307-
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT);
310+
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*);
308311
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, jl_array_uint8_type);
309312
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
310313
a->flags.ndims = 1;
@@ -351,7 +354,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
351354
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);
352355

353356
int ndimwords = jl_array_ndimwords(1);
354-
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
357+
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
355358
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
356359
// No allocation or safepoint allowed after this
357360
a->flags.pooled = tsz <= GC_MAX_SZCLASS;
@@ -418,7 +421,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
418421
"unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align);
419422

420423
int ndimwords = jl_array_ndimwords(ndims);
421-
int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
424+
int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
422425
a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype);
423426
// No allocation or safepoint allowed after this
424427
a->flags.pooled = tsz <= GC_MAX_SZCLASS;

‎src/julia_internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,7 @@ STATIC_INLINE uint8_t JL_CONST_FUNC jl_gc_szclass_align8(unsigned sz)
341341
// JL_HEAP_ALIGNMENT is the maximum alignment that the GC can provide
342342
#define JL_HEAP_ALIGNMENT JL_SMALL_BYTE_ALIGNMENT
343343
#define GC_MAX_SZCLASS (2032-sizeof(void*))
344+
static_assert(ARRAY_CACHE_ALIGN_THRESHOLD > GC_MAX_SZCLASS, "");
344345

345346
STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)
346347
{

‎src/options.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,14 @@
1212

1313
// object layout options ------------------------------------------------------
1414

15-
// how much space we're willing to waste if an array outgrows its
16-
// original object
15+
// The data for an array this size or below will be allocated within the
16+
// Array object. If the array outgrows that space, it will be wasted.
1717
#define ARRAY_INLINE_NBYTES (2048*sizeof(void*))
1818

19+
// Arrays at least this size will get larger alignment (JL_CACHE_BYTE_ALIGNMENT).
20+
// Must be bigger than GC_MAX_SZCLASS.
21+
#define ARRAY_CACHE_ALIGN_THRESHOLD 2048
22+
1923
// codegen options ------------------------------------------------------------
2024

2125
// (Experimental) Use MCJIT ELF, even where it's not the native format

‎src/staticdata.c

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -782,14 +782,23 @@ static void jl_write_values(jl_serializer_state *s)
782782
#define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes)
783783
jl_array_t *ar = (jl_array_t*)v;
784784
jl_value_t *et = jl_tparam0(jl_typeof(v));
785+
size_t alen = jl_array_len(ar);
786+
size_t datasize = alen * ar->elsize;
787+
size_t tot = datasize;
788+
int isbitsunion = jl_array_isbitsunion(ar);
789+
if (isbitsunion)
790+
tot += alen;
791+
else if (ar->elsize == 1)
792+
tot += 1;
785793
int ndimwords = jl_array_ndimwords(ar->flags.ndims);
786-
size_t tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
794+
size_t headersize = sizeof(jl_array_t) + ndimwords*sizeof(size_t);
787795
// copy header
788-
ios_write(s->s, (char*)v, tsz);
796+
ios_write(s->s, (char*)v, headersize);
797+
size_t alignment_amt = JL_SMALL_BYTE_ALIGNMENT;
798+
if (tot >= ARRAY_CACHE_ALIGN_THRESHOLD)
799+
alignment_amt = JL_CACHE_BYTE_ALIGNMENT;
789800
// make some header modifications in-place
790801
jl_array_t *newa = (jl_array_t*)&s->s->buf[reloc_offset];
791-
size_t alen = jl_array_len(ar);
792-
size_t tot = alen * ar->elsize;
793802
if (newa->flags.ndims == 1)
794803
newa->maxsize = alen;
795804
newa->offset = 0;
@@ -799,8 +808,7 @@ static void jl_write_values(jl_serializer_state *s)
799808

800809
// write data
801810
if (!ar->flags.ptrarray && !ar->flags.hasptr) {
802-
uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 16);
803-
// realign stream to max(data-align(array), sizeof(void*))
811+
uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), alignment_amt);
804812
write_padding(s->const_data, data - ios_pos(s->const_data));
805813
// write data and relocations
806814
newa->data = NULL; // relocation offset
@@ -815,22 +823,27 @@ static void jl_write_values(jl_serializer_state *s)
815823
write_pointer(s->const_data);
816824
}
817825
else {
818-
int isbitsunion = jl_array_isbitsunion(ar);
819-
if (ar->elsize == 1 && !isbitsunion)
820-
tot += 1;
821-
ios_write(s->const_data, (char*)jl_array_data(ar), tot);
822-
if (isbitsunion)
826+
if (isbitsunion) {
827+
ios_write(s->const_data, (char*)jl_array_data(ar), datasize);
823828
ios_write(s->const_data, jl_array_typetagdata(ar), alen);
829+
}
830+
else {
831+
ios_write(s->const_data, (char*)jl_array_data(ar), tot);
832+
}
824833
}
825834
}
826835
else {
827-
newa->data = (void*)tsz; // relocation offset
836+
size_t data = LLT_ALIGN(ios_pos(s->s), alignment_amt);
837+
size_t padding_amt = data - ios_pos(s->s);
838+
write_padding(s->s, padding_amt);
839+
headersize += padding_amt;
840+
newa->data = (void*)headersize; // relocation offset
828841
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location
829842
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target
830843
if (ar->flags.hasptr) {
831844
// copy all of the data first
832845
const char *data = (const char*)jl_array_data(ar);
833-
ios_write(s->s, data, tot);
846+
ios_write(s->s, data, datasize);
834847
// the rewrite all of the embedded pointers to null+relocation
835848
uint16_t elsz = ar->elsize;
836849
size_t j, np = ((jl_datatype_t*)et)->layout->npointers;
@@ -840,12 +853,12 @@ static void jl_write_values(jl_serializer_state *s)
840853
size_t offset = i * elsz + jl_ptr_offset(((jl_datatype_t*)et), j) * sizeof(jl_value_t*);
841854
jl_value_t *fld = *(jl_value_t**)&data[offset];
842855
if (fld != NULL) {
843-
arraylist_push(&s->relocs_list, (void*)(uintptr_t)(reloc_offset + tsz + offset)); // relocation location
856+
arraylist_push(&s->relocs_list, (void*)(uintptr_t)(reloc_offset + headersize + offset)); // relocation location
844857
arraylist_push(&s->relocs_list, (void*)backref_id(s, fld)); // relocation target
845-
memset(&s->s->buf[reloc_offset + tsz + offset], 0, sizeof(fld)); // relocation offset (none)
858+
memset(&s->s->buf[reloc_offset + headersize + offset], 0, sizeof(fld)); // relocation offset (none)
846859
}
847860
else {
848-
assert(*(jl_value_t**)&s->s->buf[reloc_offset + tsz + offset] == NULL);
861+
assert(*(jl_value_t**)&s->s->buf[reloc_offset + headersize + offset] == NULL);
849862
}
850863
}
851864
}

‎test/arrayops.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2660,7 +2660,7 @@ let TT = Union{UInt8, Int8}
26602660
resize!(b, 1)
26612661
@assert pointer(a) == pa
26622662
@assert pointer(b) == pb
2663-
unsafe_store!(pa, 0x1, 2) # reset a[2] to 1
2663+
unsafe_store!(Ptr{UInt8}(pa), 0x1, 2) # reset a[2] to 1
26642664
@test length(a) == length(b) == 1
26652665
@test a[1] == b[1] == 0x0
26662666
@test a == b

‎test/cmdlineargs.jl

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
324324
rm(memfile)
325325
@test popfirst!(got) == " 0 g(x) = x + 123456"
326326
@test popfirst!(got) == " - function f(x)"
327-
@test popfirst!(got) == " 80 []"
327+
if Sys.WORD_SIZE == 64
328+
@test popfirst!(got) == " 48 []"
329+
else
330+
@test popfirst!(got) == " 32 []"
331+
end
328332
if Sys.WORD_SIZE == 64
329333
# P64 pools with 64 bit tags
330334
@test popfirst!(got) == " 16 Base.invokelatest(g, 0)"
@@ -337,7 +341,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
337341
@test popfirst!(got) == " 8 Base.invokelatest(g, 0)"
338342
@test popfirst!(got) == " 32 Base.invokelatest(g, x)"
339343
end
340-
@test popfirst!(got) == " 80 []"
344+
if Sys.WORD_SIZE == 64
345+
@test popfirst!(got) == " 48 []"
346+
else
347+
@test popfirst!(got) == " 32 []"
348+
end
341349
@test popfirst!(got) == " - end"
342350
@test popfirst!(got) == " - f(1.23)"
343351
@test isempty(got) || got

0 commit comments

Comments
 (0)