Skip to content

Commit 465e71b

Browse files
anonriganonrig
committed
src: keep embedded ICU data file-backed
udata_setCommonData() stores the package address and later lookups return pointers into that storage, so the decompressed buffer cannot be freed. Inflating the full ICU data file at startup turned every page into private dirty memory. Map the original data from the executable again so unused pages stay demand-paged, clean, and shareable across processes. Builtin sources and the startup snapshot remain compressed. Signed-off-by: Yagiz Nizipli <yagiz@nizipli.com> Assisted-by: Grok
1 parent 8afd2ef commit 465e71b

5 files changed

Lines changed: 18 additions & 253 deletions

File tree

‎deps/zstd/zstd.gyp‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,5 @@
103103
],
104104
'toolsets': ['host', 'target'],
105105
},
106-
{
107-
'target_name': 'zstd_compress',
108-
'type': 'executable',
109-
'toolsets': ['host'],
110-
'dependencies': ['zstd#host'],
111-
'include_dirs': ['lib'],
112-
'sources': ['../../tools/zstd_compress.cc'],
113-
},
114106
]
115107
}

‎node.gypi‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,6 @@
135135
'<(icu_gyp_path):icuuc',
136136
],
137137
'conditions': [
138-
[ 'icu_system!="true"', {
139-
'defines': [ 'NODE_HAVE_EMBEDDED_ICU_ZSTD=1' ],
140-
}],
141138
[ 'icu_small=="true"', {
142139
'defines': [ 'NODE_HAVE_SMALL_ICU=1' ],
143140
'conditions': [
@@ -147,8 +144,7 @@
147144
],
148145
}],
149146
],
150-
}],
151-
],
147+
}]],
152148
}],
153149
[ 'node_use_bundled_v8=="true" and \
154150
node_enable_v8_vtunejit=="true" and (target_arch=="x64" or \

‎src/node_i18n.cc‎

Lines changed: 2 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,6 @@
4343
#include "node_i18n.h"
4444
#include "node_external_reference.h"
4545
#include "simdutf.h"
46-
#include "zstd_blob.h"
47-
48-
#include <cstdint>
49-
#include <cstring>
5046

5147
#if defined(NODE_HAVE_I18N_SUPPORT)
5248

@@ -72,7 +68,7 @@
7268
#include <unicode/uversion.h>
7369
#include "nbytes.h"
7470

75-
#if defined(NODE_HAVE_SMALL_ICU) || defined(NODE_HAVE_EMBEDDED_ICU_ZSTD)
71+
#ifdef NODE_HAVE_SMALL_ICU
7672
#include <unicode/udata.h>
7773

7874
/* if this is defined, we have a 'secondary' entry point.
@@ -89,38 +85,6 @@
8985
extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[];
9086
#endif
9187

92-
#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD
93-
extern "C" const uint8_t node_icu_zstd_dat[];
94-
95-
static uint64_t ReadU64LE(const uint8_t* bytes) {
96-
uint64_t value = 0;
97-
for (int i = 0; i < 8; i++) {
98-
value |= static_cast<uint64_t>(bytes[i]) << (8 * i);
99-
}
100-
return value;
101-
}
102-
103-
// Decompress the embedded ICU data file. The returned pointer is aligned and
104-
// lives for the process lifetime. nullptr on failure, with `error` set.
105-
static uint8_t* DecompressEmbeddedICU(std::string* error) {
106-
const uint8_t* bytes = node_icu_zstd_dat;
107-
if (memcmp(bytes, "ICUZ", 4) != 0) {
108-
*error = "embedded ICU data header is invalid";
109-
return nullptr;
110-
}
111-
uint64_t raw_size = ReadU64LE(bytes + 4);
112-
uint64_t compressed_size = ReadU64LE(bytes + 12);
113-
size_t got = 0;
114-
uint8_t* data = node::ZstdDecompressAligned(
115-
bytes + 20, static_cast<size_t>(compressed_size), &got);
116-
if (data == nullptr || got != raw_size) {
117-
*error = "failed to decompress embedded ICU data";
118-
return nullptr;
119-
}
120-
return data;
121-
}
122-
#endif // NODE_HAVE_EMBEDDED_ICU_ZSTD
123-
12488
namespace node {
12589

12690
using v8::Context;
@@ -591,13 +555,7 @@ ConverterObject::ConverterObject(
591555
bool InitializeICUDirectory(const std::string& path, std::string* error) {
592556
UErrorCode status = U_ZERO_ERROR;
593557
if (path.empty()) {
594-
#ifdef NODE_HAVE_EMBEDDED_ICU_ZSTD
595-
static uint8_t* icu_data = DecompressEmbeddedICU(error);
596-
if (icu_data == nullptr) {
597-
return false;
598-
}
599-
udata_setCommonData(icu_data, &status);
600-
#elif defined(NODE_HAVE_SMALL_ICU)
558+
#ifdef NODE_HAVE_SMALL_ICU
601559
// install the 'small' data.
602560
udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status);
603561
#else // !NODE_HAVE_SMALL_ICU

‎tools/icu/icu-generic.gyp‎

Lines changed: 15 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -138,32 +138,14 @@
138138
[ 'icu_small == "false"', { # and OS=win
139139
# full data - just build the full data file, then we are done.
140140
'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
141-
'dependencies': [
142-
'genccode#host',
143-
'icustubdata',
144-
'../../deps/zstd/zstd.gyp:zstd_compress#host',
145-
],
146-
'export_dependent_settings': [ 'icustubdata' ],
141+
'dependencies': [ 'genccode#host' ],
147142
'conditions': [
148143
[ 'clang==1', {
149144
'actions': [
150-
{
151-
'action_name': 'icu_zstd',
152-
'msvs_quote_cmd': 0,
153-
'inputs': [
154-
'<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
155-
'<(icu_data_in)',
156-
],
157-
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
158-
'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
159-
'--icu',
160-
'<(icu_data_in)',
161-
'<@(_outputs)' ],
162-
},
163145
{
164146
'action_name': 'icudata',
165147
'msvs_quote_cmd': 0,
166-
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
148+
'inputs': [ '<(icu_data_in)' ],
167149
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
168150
# on Windows, we can go directly to .obj file (-o) option.
169151
# for Clang use "-c <(target_arch)" option
@@ -172,39 +154,24 @@
172154
'-c', '<(target_arch)',
173155
'-d', '<(SHARED_INTERMEDIATE_DIR)',
174156
'-n', 'icudata',
175-
'-e', 'node_icu_zstd',
176-
'-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat',
157+
'-e', 'icudt<(icu_ver_major)',
177158
'<@(_inputs)' ],
178159
},
179160
],
180161
}, {
181162
'actions': [
182-
{
183-
'action_name': 'icu_zstd',
184-
'msvs_quote_cmd': 0,
185-
'inputs': [
186-
'<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
187-
'<(icu_data_in)',
188-
],
189-
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
190-
'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
191-
'--icu',
192-
'<(icu_data_in)',
193-
'<@(_outputs)' ],
194-
},
195163
{
196164
'action_name': 'icudata',
197165
'msvs_quote_cmd': 0,
198-
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
166+
'inputs': [ '<(icu_data_in)' ],
199167
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
200168
# on Windows, we can go directly to .obj file (-o) option.
201169
# for MSVC do not use "-c <(target_arch)" option
202170
'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)',
203171
'<@(icu_asm_opts)', # -o
204172
'-d', '<(SHARED_INTERMEDIATE_DIR)',
205173
'-n', 'icudata',
206-
'-e', 'node_icu_zstd',
207-
'-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat',
174+
'-e', 'icudt<(icu_ver_major)',
208175
'<@(_inputs)' ],
209176
},
210177
],
@@ -213,8 +180,7 @@
213180
}, { # icu_small == TRUE and OS == win
214181
# link against stub data primarily
215182
# then, use icupkg and genccode to rebuild data
216-
'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host',
217-
'../../deps/zstd/zstd.gyp:zstd_compress#host' ],
183+
'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host' ],
218184
'export_dependent_settings': [ 'icustubdata' ],
219185
'actions': [
220186
{
@@ -234,32 +200,18 @@
234200
'-v',
235201
'-L', '<(icu_locales)'],
236202
},
237-
{
238-
'action_name': 'icu_zstd',
239-
'msvs_quote_cmd': 0,
240-
'inputs': [
241-
'<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
242-
'<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat',
243-
],
244-
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
245-
'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
246-
'--icu',
247-
'<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat',
248-
'<@(_outputs)' ],
249-
},
250203
{
251204
# build final .dat -> .obj
252205
'action_name': 'genccode',
253206
'msvs_quote_cmd': 0,
254-
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
207+
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ],
255208
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ],
256209
'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)',
257210
'<@(icu_asm_opts)', # -o
258211
'-c', '<(target_arch)',
259212
'-d', '<(SHARED_INTERMEDIATE_DIR)/',
260213
'-n', 'icudata',
261-
'-e', 'node_icu_zstd',
262-
'-f', 'icudt<(icu_ver_major)<(icu_endianness)_dat',
214+
'-e', 'icusmdt<(icu_ver_major)',
263215
'<@(_inputs)' ],
264216
},
265217
],
@@ -269,19 +221,9 @@
269221
}, { # OS != win
270222
'conditions': [
271223
[ 'icu_small == "false"', {
272-
# full data - no trim needed. The bytes embedded below are
273-
# zstd-compressed; node decompresses them before udata_setCommonData.
274-
# icustubdata satisfies ICU's icudtXX_dat link reference.
224+
# full data - no trim needed
275225
'sources': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
276-
'dependencies': [
277-
'genccode#host',
278-
'icupkg#host',
279-
'icu_implementation#host',
280-
'icu_uconfig',
281-
'icustubdata',
282-
'../../deps/zstd/zstd.gyp:zstd_compress#host',
283-
],
284-
'export_dependent_settings': [ 'icustubdata' ],
226+
'dependencies': [ 'genccode#host', 'icupkg#host', 'icu_implementation#host', 'icu_uconfig' ],
285227
'include_dirs': [
286228
'<(icu_path)/source/common',
287229
],
@@ -308,26 +250,12 @@
308250
],
309251
},
310252
{
311-
'action_name': 'icu_zstd',
312-
'inputs': [
313-
'<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
314-
'<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat',
315-
],
316-
'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
317-
'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
318-
'--icu',
319-
'<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat',
320-
'<@(_outputs)' ],
321-
},
322-
{
323-
# convert compressed ICU data to .c, or .S, etc.
324-
# -e names the symbol node_icu_zstd_dat so ICU does not
325-
# treat the compressed bytes as its data entry point.
253+
# convert full ICU data file to .c, or .S, etc.
326254
'action_name': 'icudata',
327-
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
255+
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ],
328256
'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
329257
'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)',
330-
'-e', 'node_icu_zstd',
258+
'-e', 'icudt<(icu_ver_major)',
331259
'-d', '<(SHARED_INTERMEDIATE_DIR)',
332260
'<@(icu_asm_opts)',
333261
'-f', 'icudt<(icu_ver_major)_dat',
@@ -338,8 +266,7 @@
338266
# link against stub data (as primary data)
339267
# then, use icupkg and genccode to rebuild small data
340268
'dependencies': [ 'icustubdata', 'genccode#host', 'icupkg#host', 'genrb#host', 'iculslocs#host',
341-
'icu_implementation', 'icu_uconfig',
342-
'../../deps/zstd/zstd.gyp:zstd_compress#host' ],
269+
'icu_implementation', 'icu_uconfig' ],
343270
'export_dependent_settings': [ 'icustubdata' ],
344271
'actions': [
345272
{
@@ -367,26 +294,13 @@
367294
'<@(_inputs)',
368295
'<@(_outputs)',
369296
],
370-
}, {
371-
'action_name': 'icu_zstd',
372-
'inputs': [
373-
'<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
374-
'<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat',
375-
],
376-
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
377-
'action': [ '<(PRODUCT_DIR)/zstd_compress<(EXECUTABLE_SUFFIX)',
378-
'--icu',
379-
'<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat',
380-
'<@(_outputs)' ],
381297
}, {
382298
# For icu-small, always use .c, don't try to use .S, etc.
383299
'action_name': 'genccode',
384-
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/node_icu_zstd.dat' ],
300+
'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ],
385301
'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ],
386302
'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)',
387303
'<@(icu_asm_opts)',
388-
'-e', 'node_icu_zstd',
389-
'-f', 'icusmdt<(icu_ver_major)_dat',
390304
'-d', '<(SHARED_INTERMEDIATE_DIR)',
391305
'<@(_inputs)' ],
392306
},

0 commit comments

Comments
 (0)