mirror of
https://gitlab.torproject.org/tpo/core/tor.git
synced 2024-09-20 20:23:03 +02:00
Merge remote-tracking branch 'public/feature11791'
This commit is contained in:
commit
cf2ac8e255
4
changes/bug11791
Normal file
4
changes/bug11791
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
o Minor features (directory, memory usage):
|
||||||
|
- When we have recently been under memory pressure (over 3/4 of
|
||||||
|
MaxMemInQueues is allocated), then allocate smaller zlib objects for
|
||||||
|
small requests. Closes ticket 11791.
|
@ -92,10 +92,27 @@ tor_zlib_get_header_version_str(void)
|
|||||||
|
|
||||||
/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
|
/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
|
||||||
static INLINE int
|
static INLINE int
|
||||||
method_bits(compress_method_t method)
|
method_bits(compress_method_t method, zlib_compression_level_t level)
|
||||||
{
|
{
|
||||||
/* Bits+16 means "use gzip" in zlib >= 1.2 */
|
/* Bits+16 means "use gzip" in zlib >= 1.2 */
|
||||||
return method == GZIP_METHOD ? 15+16 : 15;
|
const int flag = method == GZIP_METHOD ? 16 : 0;
|
||||||
|
switch (level) {
|
||||||
|
default:
|
||||||
|
case HIGH_COMPRESSION: return flag + 15;
|
||||||
|
case MEDIUM_COMPRESSION: return flag + 13;
|
||||||
|
case LOW_COMPRESSION: return flag + 11;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static INLINE int
|
||||||
|
get_memlevel(zlib_compression_level_t level)
|
||||||
|
{
|
||||||
|
switch (level) {
|
||||||
|
default:
|
||||||
|
case HIGH_COMPRESSION: return 8;
|
||||||
|
case MEDIUM_COMPRESSION: return 7;
|
||||||
|
case LOW_COMPRESSION: return 6;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @{ */
|
/** @{ */
|
||||||
@ -162,8 +179,9 @@ tor_gzip_compress(char **out, size_t *out_len,
|
|||||||
stream->avail_in = (unsigned int)in_len;
|
stream->avail_in = (unsigned int)in_len;
|
||||||
|
|
||||||
if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
|
if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
|
||||||
method_bits(method),
|
method_bits(method, HIGH_COMPRESSION),
|
||||||
8, Z_DEFAULT_STRATEGY) != Z_OK) {
|
get_memlevel(HIGH_COMPRESSION),
|
||||||
|
Z_DEFAULT_STRATEGY) != Z_OK) {
|
||||||
log_warn(LD_GENERAL, "Error from deflateInit2: %s",
|
log_warn(LD_GENERAL, "Error from deflateInit2: %s",
|
||||||
stream->msg?stream->msg:"<no message>");
|
stream->msg?stream->msg:"<no message>");
|
||||||
goto err;
|
goto err;
|
||||||
@ -289,7 +307,7 @@ tor_gzip_uncompress(char **out, size_t *out_len,
|
|||||||
stream->avail_in = (unsigned int)in_len;
|
stream->avail_in = (unsigned int)in_len;
|
||||||
|
|
||||||
if (inflateInit2(stream,
|
if (inflateInit2(stream,
|
||||||
method_bits(method)) != Z_OK) {
|
method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
|
||||||
log_warn(LD_GENERAL, "Error from inflateInit2: %s",
|
log_warn(LD_GENERAL, "Error from inflateInit2: %s",
|
||||||
stream->msg?stream->msg:"<no message>");
|
stream->msg?stream->msg:"<no message>");
|
||||||
goto err;
|
goto err;
|
||||||
@ -315,7 +333,8 @@ tor_gzip_uncompress(char **out, size_t *out_len,
|
|||||||
log_warn(LD_BUG, "Error freeing gzip structures");
|
log_warn(LD_BUG, "Error freeing gzip structures");
|
||||||
goto err;
|
goto err;
|
||||||
}
|
}
|
||||||
if (inflateInit2(stream, method_bits(method)) != Z_OK) {
|
if (inflateInit2(stream,
|
||||||
|
method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
|
||||||
log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
|
log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
|
||||||
stream->msg?stream->msg:"<no message>");
|
stream->msg?stream->msg:"<no message>");
|
||||||
goto err;
|
goto err;
|
||||||
@ -426,10 +445,11 @@ struct tor_zlib_state_t {
|
|||||||
* <b>compress</b>, it's for compression; otherwise it's for
|
* <b>compress</b>, it's for compression; otherwise it's for
|
||||||
* decompression. */
|
* decompression. */
|
||||||
tor_zlib_state_t *
|
tor_zlib_state_t *
|
||||||
tor_zlib_new(int compress, compress_method_t method)
|
tor_zlib_new(int compress, compress_method_t method,
|
||||||
|
zlib_compression_level_t compression_level)
|
||||||
{
|
{
|
||||||
tor_zlib_state_t *out;
|
tor_zlib_state_t *out;
|
||||||
int bits;
|
int bits, memlevel;
|
||||||
|
|
||||||
if (method == GZIP_METHOD && !is_gzip_supported()) {
|
if (method == GZIP_METHOD && !is_gzip_supported()) {
|
||||||
/* Old zlib version don't support gzip in inflateInit2 */
|
/* Old zlib version don't support gzip in inflateInit2 */
|
||||||
@ -437,21 +457,29 @@ tor_zlib_new(int compress, compress_method_t method)
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (! compress) {
|
||||||
|
/* use this setting for decompression, since we might have the
|
||||||
|
* max number of window bits */
|
||||||
|
compression_level = HIGH_COMPRESSION;
|
||||||
|
}
|
||||||
|
|
||||||
out = tor_malloc_zero(sizeof(tor_zlib_state_t));
|
out = tor_malloc_zero(sizeof(tor_zlib_state_t));
|
||||||
out->stream.zalloc = Z_NULL;
|
out->stream.zalloc = Z_NULL;
|
||||||
out->stream.zfree = Z_NULL;
|
out->stream.zfree = Z_NULL;
|
||||||
out->stream.opaque = NULL;
|
out->stream.opaque = NULL;
|
||||||
out->compress = compress;
|
out->compress = compress;
|
||||||
bits = method_bits(method);
|
bits = method_bits(method, compression_level);
|
||||||
|
memlevel = get_memlevel(compression_level);
|
||||||
if (compress) {
|
if (compress) {
|
||||||
if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
|
if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
|
||||||
bits, 8, Z_DEFAULT_STRATEGY) != Z_OK)
|
bits, memlevel,
|
||||||
|
Z_DEFAULT_STRATEGY) != Z_OK)
|
||||||
goto err;
|
goto err;
|
||||||
} else {
|
} else {
|
||||||
if (inflateInit2(&out->stream, bits) != Z_OK)
|
if (inflateInit2(&out->stream, bits) != Z_OK)
|
||||||
goto err;
|
goto err;
|
||||||
}
|
}
|
||||||
out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8);
|
out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel);
|
||||||
|
|
||||||
total_zlib_allocation += out->allocation;
|
total_zlib_allocation += out->allocation;
|
||||||
|
|
||||||
|
@ -19,6 +19,15 @@ typedef enum {
|
|||||||
NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3
|
NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3
|
||||||
} compress_method_t;
|
} compress_method_t;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Enumeration to define tradeoffs between memory usage and compression level.
|
||||||
|
* HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
|
||||||
|
* memory.
|
||||||
|
**/
|
||||||
|
typedef enum {
|
||||||
|
HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
|
||||||
|
} zlib_compression_level_t;
|
||||||
|
|
||||||
int
|
int
|
||||||
tor_gzip_compress(char **out, size_t *out_len,
|
tor_gzip_compress(char **out, size_t *out_len,
|
||||||
const char *in, size_t in_len,
|
const char *in, size_t in_len,
|
||||||
@ -47,7 +56,8 @@ typedef enum {
|
|||||||
} tor_zlib_output_t;
|
} tor_zlib_output_t;
|
||||||
/** Internal state for an incremental zlib compression/decompression. */
|
/** Internal state for an incremental zlib compression/decompression. */
|
||||||
typedef struct tor_zlib_state_t tor_zlib_state_t;
|
typedef struct tor_zlib_state_t tor_zlib_state_t;
|
||||||
tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method);
|
tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method,
|
||||||
|
zlib_compression_level_t level);
|
||||||
|
|
||||||
tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
|
tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
|
||||||
char **out, size_t *out_len,
|
char **out, size_t *out_len,
|
||||||
|
@ -2911,6 +2911,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
|
|||||||
options->MaxMemInQueues =
|
options->MaxMemInQueues =
|
||||||
compute_real_max_mem_in_queues(options->MaxMemInQueues_raw,
|
compute_real_max_mem_in_queues(options->MaxMemInQueues_raw,
|
||||||
server_mode(options));
|
server_mode(options));
|
||||||
|
options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3;
|
||||||
|
|
||||||
options->AllowInvalid_ = 0;
|
options->AllowInvalid_ = 0;
|
||||||
|
|
||||||
|
@ -20,6 +20,7 @@
|
|||||||
#include "networkstatus.h"
|
#include "networkstatus.h"
|
||||||
#include "nodelist.h"
|
#include "nodelist.h"
|
||||||
#include "policies.h"
|
#include "policies.h"
|
||||||
|
#include "relay.h"
|
||||||
#include "rendclient.h"
|
#include "rendclient.h"
|
||||||
#include "rendcommon.h"
|
#include "rendcommon.h"
|
||||||
#include "rephist.h"
|
#include "rephist.h"
|
||||||
@ -2539,6 +2540,24 @@ client_likes_consensus(networkstatus_t *v, const char *want_url)
|
|||||||
return (have >= need_at_least);
|
return (have >= need_at_least);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Return the compression level we should use for sending a compressed
|
||||||
|
* response of size <b>n_bytes</b>. */
|
||||||
|
static zlib_compression_level_t
|
||||||
|
choose_compression_level(ssize_t n_bytes)
|
||||||
|
{
|
||||||
|
if (! have_been_under_memory_pressure()) {
|
||||||
|
return HIGH_COMPRESSION; /* we have plenty of RAM. */
|
||||||
|
} else if (n_bytes < 0) {
|
||||||
|
return HIGH_COMPRESSION; /* unknown; might be big. */
|
||||||
|
} else if (n_bytes < 1024) {
|
||||||
|
return LOW_COMPRESSION;
|
||||||
|
} else if (n_bytes < 2048) {
|
||||||
|
return MEDIUM_COMPRESSION;
|
||||||
|
} else {
|
||||||
|
return HIGH_COMPRESSION;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Helper function: called when a dirserver gets a complete HTTP GET
|
/** Helper function: called when a dirserver gets a complete HTTP GET
|
||||||
* request. Look for a request for a directory or for a rendezvous
|
* request. Look for a request for a directory or for a rendezvous
|
||||||
* service descriptor. On finding one, write a response into
|
* service descriptor. On finding one, write a response into
|
||||||
@ -2724,7 +2743,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
|
|||||||
smartlist_len(dir_fps) == 1 ? lifetime : 0);
|
smartlist_len(dir_fps) == 1 ? lifetime : 0);
|
||||||
conn->fingerprint_stack = dir_fps;
|
conn->fingerprint_stack = dir_fps;
|
||||||
if (! compressed)
|
if (! compressed)
|
||||||
conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);
|
conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
|
||||||
|
|
||||||
/* Prime the connection with some data. */
|
/* Prime the connection with some data. */
|
||||||
conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
|
conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
|
||||||
@ -2812,7 +2831,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
|
|||||||
|
|
||||||
if (smartlist_len(items)) {
|
if (smartlist_len(items)) {
|
||||||
if (compressed) {
|
if (compressed) {
|
||||||
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
|
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
|
||||||
|
choose_compression_level(estimated_len));
|
||||||
SMARTLIST_FOREACH(items, const char *, c,
|
SMARTLIST_FOREACH(items, const char *, c,
|
||||||
connection_write_to_buf_zlib(c, strlen(c), conn, 0));
|
connection_write_to_buf_zlib(c, strlen(c), conn, 0));
|
||||||
connection_write_to_buf_zlib("", 0, conn, 1);
|
connection_write_to_buf_zlib("", 0, conn, 1);
|
||||||
@ -2861,7 +2881,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
|
|||||||
conn->fingerprint_stack = fps;
|
conn->fingerprint_stack = fps;
|
||||||
|
|
||||||
if (compressed)
|
if (compressed)
|
||||||
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
|
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
|
||||||
|
choose_compression_level(dlen));
|
||||||
|
|
||||||
connection_dirserv_flushed_some(conn);
|
connection_dirserv_flushed_some(conn);
|
||||||
goto done;
|
goto done;
|
||||||
@ -2929,7 +2950,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
|
|||||||
}
|
}
|
||||||
write_http_response_header(conn, -1, compressed, cache_lifetime);
|
write_http_response_header(conn, -1, compressed, cache_lifetime);
|
||||||
if (compressed)
|
if (compressed)
|
||||||
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
|
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
|
||||||
|
choose_compression_level(dlen));
|
||||||
/* Prime the connection with some data. */
|
/* Prime the connection with some data. */
|
||||||
connection_dirserv_flushed_some(conn);
|
connection_dirserv_flushed_some(conn);
|
||||||
}
|
}
|
||||||
@ -3004,7 +3026,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
|
|||||||
|
|
||||||
write_http_response_header(conn, compressed?-1:len, compressed, 60*60);
|
write_http_response_header(conn, compressed?-1:len, compressed, 60*60);
|
||||||
if (compressed) {
|
if (compressed) {
|
||||||
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
|
conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
|
||||||
|
choose_compression_level(len));
|
||||||
SMARTLIST_FOREACH(certs, authority_cert_t *, c,
|
SMARTLIST_FOREACH(certs, authority_cert_t *, c,
|
||||||
connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,
|
connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,
|
||||||
c->cache_info.signed_descriptor_len,
|
c->cache_info.signed_descriptor_len,
|
||||||
|
@ -3196,7 +3196,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn)
|
|||||||
if (uncompressing && ! conn->zlib_state &&
|
if (uncompressing && ! conn->zlib_state &&
|
||||||
conn->fingerprint_stack &&
|
conn->fingerprint_stack &&
|
||||||
smartlist_len(conn->fingerprint_stack)) {
|
smartlist_len(conn->fingerprint_stack)) {
|
||||||
conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);
|
conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (r) return r;
|
if (r) return r;
|
||||||
|
@ -3537,6 +3537,8 @@ typedef struct {
|
|||||||
uint64_t MaxMemInQueues_raw;
|
uint64_t MaxMemInQueues_raw;
|
||||||
uint64_t MaxMemInQueues;/**< If we have more memory than this allocated
|
uint64_t MaxMemInQueues;/**< If we have more memory than this allocated
|
||||||
* for queues and buffers, run the OOM handler */
|
* for queues and buffers, run the OOM handler */
|
||||||
|
/** Above this value, consider ourselves low on RAM. */
|
||||||
|
uint64_t MaxMemInQueues_low_threshold;
|
||||||
|
|
||||||
/** @name port booleans
|
/** @name port booleans
|
||||||
*
|
*
|
||||||
|
@ -2433,6 +2433,12 @@ cell_queues_get_total_allocation(void)
|
|||||||
return total_cells_allocated * packed_cell_mem_cost();
|
return total_cells_allocated * packed_cell_mem_cost();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** How long after we've been low on memory should we try to conserve it? */
|
||||||
|
#define MEMORY_PRESSURE_INTERVAL (30*60)
|
||||||
|
|
||||||
|
/** The time at which we were last low on memory. */
|
||||||
|
static time_t last_time_under_memory_pressure = 0;
|
||||||
|
|
||||||
/** Check whether we've got too much space used for cells. If so,
|
/** Check whether we've got too much space used for cells. If so,
|
||||||
* call the OOM handler and return 1. Otherwise, return 0. */
|
* call the OOM handler and return 1. Otherwise, return 0. */
|
||||||
STATIC int
|
STATIC int
|
||||||
@ -2441,13 +2447,25 @@ cell_queues_check_size(void)
|
|||||||
size_t alloc = cell_queues_get_total_allocation();
|
size_t alloc = cell_queues_get_total_allocation();
|
||||||
alloc += buf_get_total_allocation();
|
alloc += buf_get_total_allocation();
|
||||||
alloc += tor_zlib_get_total_allocation();
|
alloc += tor_zlib_get_total_allocation();
|
||||||
if (alloc >= get_options()->MaxMemInQueues) {
|
if (alloc >= get_options()->MaxMemInQueues_low_threshold) {
|
||||||
circuits_handle_oom(alloc);
|
last_time_under_memory_pressure = approx_time();
|
||||||
return 1;
|
if (alloc >= get_options()->MaxMemInQueues) {
|
||||||
|
circuits_handle_oom(alloc);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Return true if we've been under memory pressure in the last
|
||||||
|
* MEMORY_PRESSURE_INTERVAL seconds. */
|
||||||
|
int
|
||||||
|
have_been_under_memory_pressure(void)
|
||||||
|
{
|
||||||
|
return last_time_under_memory_pressure + MEMORY_PRESSURE_INTERVAL
|
||||||
|
< approx_time();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update the number of cells available on the circuit's n_chan or p_chan's
|
* Update the number of cells available on the circuit's n_chan or p_chan's
|
||||||
* circuit mux.
|
* circuit mux.
|
||||||
|
@ -50,6 +50,8 @@ void clean_cell_pool(void);
|
|||||||
void dump_cell_pool_usage(int severity);
|
void dump_cell_pool_usage(int severity);
|
||||||
size_t packed_cell_mem_cost(void);
|
size_t packed_cell_mem_cost(void);
|
||||||
|
|
||||||
|
int have_been_under_memory_pressure(void);
|
||||||
|
|
||||||
/* For channeltls.c */
|
/* For channeltls.c */
|
||||||
void packed_cell_free(packed_cell_t *cell);
|
void packed_cell_free(packed_cell_t *cell);
|
||||||
|
|
||||||
|
@ -611,7 +611,7 @@ test_buffers_zlib_impl(int finalize_with_nil)
|
|||||||
int done;
|
int done;
|
||||||
|
|
||||||
buf = buf_new_with_capacity(128); /* will round up */
|
buf = buf_new_with_capacity(128); /* will round up */
|
||||||
zlib_state = tor_zlib_new(1, ZLIB_METHOD);
|
zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
|
||||||
|
|
||||||
msg = tor_malloc(512);
|
msg = tor_malloc(512);
|
||||||
crypto_rand(msg, 512);
|
crypto_rand(msg, 512);
|
||||||
@ -688,7 +688,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg)
|
|||||||
tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
|
tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
|
||||||
tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
|
tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
|
||||||
/* Write an empty string, with finalization on. */
|
/* Write an empty string, with finalization on. */
|
||||||
zlib_state = tor_zlib_new(1, ZLIB_METHOD);
|
zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
|
||||||
tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
|
tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
|
||||||
|
|
||||||
in_len = buf_datalen(buf);
|
in_len = buf_datalen(buf);
|
||||||
|
@ -1820,7 +1820,7 @@ test_util_gzip(void *arg)
|
|||||||
tor_free(buf1);
|
tor_free(buf1);
|
||||||
tor_free(buf2);
|
tor_free(buf2);
|
||||||
tor_free(buf3);
|
tor_free(buf3);
|
||||||
state = tor_zlib_new(1, ZLIB_METHOD);
|
state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
|
||||||
tt_assert(state);
|
tt_assert(state);
|
||||||
cp1 = buf1 = tor_malloc(1024);
|
cp1 = buf1 = tor_malloc(1024);
|
||||||
len1 = 1024;
|
len1 = 1024;
|
||||||
|
Loading…
Reference in New Issue
Block a user