diff -u -r -N squid-5.0.2/acinclude/os-deps.m4 squid-5.0.3/acinclude/os-deps.m4 --- squid-5.0.2/acinclude/os-deps.m4 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/acinclude/os-deps.m4 2020-06-08 03:33:42.000000000 +1200 @@ -925,11 +925,13 @@ ## Solaris 10+ backported IPv6 NAT to their IPFilter v4.1 instead of using v5 AC_CHECK_MEMBERS([ struct natlookup.nl_inipaddr.in6, - struct natlookup.nl_realipaddr.in6 - ],,,[ + struct natlookup.nl_realipaddr.in6],,,[ #if USE_SOLARIS_IPFILTER_MINOR_T_HACK #define minor_t fubar #endif +#if HAVE_SYS_PARAM_H +#include +#endif #if HAVE_SYS_TYPES_H #include #endif @@ -955,7 +957,11 @@ #elif HAVE_NETINET_IP_FIL_H #include #endif +#if HAVE_IP_NAT_H #include +#elif HAVE_NETINET_IP_NAT_H +#include +#endif ]) ]) diff -u -r -N squid-5.0.2/ChangeLog squid-5.0.3/ChangeLog --- squid-5.0.2/ChangeLog 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/ChangeLog 2020-06-08 03:33:42.000000000 +1200 @@ -1,3 +1,13 @@ +Changes in squid-5.0.3 (05 Jun 2020): + + - Bug 5046: FreeBSD lacks open(2) O_DSYNC flag + - Happy Eyeballs: Do not discard viable reforwarding destinations + - Reduced startup time with large rock cache_dirs + - Fix the ABA problem with Ipc::Mem::PageStack::pop() in v5.0.1 + - Fix sending of unknown validation errors to certificate validator + - ... and several debug improvements + - ... and all fixes from 4.12 + Changes in squid-5.0.2 (18 Apr 2020): - Bug 5030: Negative responses are never cached @@ -9,6 +19,7 @@ - Fix PURGE error responses - ... and several documentation changes - ... and some compile fixes + - ... and all fixes from 4.11 Changes in squid-5.0.1 (14 Jan 2020): @@ -56,6 +67,21 @@ - Centralized PagePool/PageStack ID generation - ... and many documentation changes - ... and much code cleanup and polishing + - ... and all fixes from 4.10 + +Changes to squid-4.12 (05 Jun 2020): + + - Regression Fix: Revert to slow search for new SMP shm pages + - Bug 5045: ext_edirectory_userip_acl is missing include files + - Bug 5041: Missing Debug::Extra breaks build on hosts with systemd + - Bug 5030: Negative responses are never cached + - HTTP: validate Content-Length value prefix + - HTTP: add flexible RFC 3986 URI encoder + - SslBump: disable OpenSSL TLSv1.3 support for older TLS traffic + - Tests: Support passing a custom config.cache to test builds + - Fix IPFilter IPv6 detection, especially on NetBSD + - Fix stall if transaction overwrites a recently active cache entry + - ... and some compile fixes Changes to squid-4.11 (18 Apr 2020): diff -u -r -N squid-5.0.2/configure squid-5.0.3/configure --- squid-5.0.2/configure 2020-04-20 00:08:55.000000000 +1200 +++ squid-5.0.3/configure 2020-06-09 18:58:26.000000000 +1200 @@ -1,7 +1,7 @@ #! /bin/sh # From configure.ac Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Squid Web Proxy 5.0.2. +# Generated by GNU Autoconf 2.69 for Squid Web Proxy 5.0.3. # # Report bugs to . # @@ -595,8 +595,8 @@ # Identity of this package. PACKAGE_NAME='Squid Web Proxy' PACKAGE_TARNAME='squid' -PACKAGE_VERSION='5.0.2' -PACKAGE_STRING='Squid Web Proxy 5.0.2' +PACKAGE_VERSION='5.0.3' +PACKAGE_STRING='Squid Web Proxy 5.0.3' PACKAGE_BUGREPORT='http://bugs.squid-cache.org/' PACKAGE_URL='' @@ -1662,7 +1662,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Squid Web Proxy 5.0.2 to adapt to many kinds of systems. +\`configure' configures Squid Web Proxy 5.0.3 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1733,7 +1733,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Squid Web Proxy 5.0.2:";; + short | recursive ) echo "Configuration of Squid Web Proxy 5.0.3:";; esac cat <<\_ACEOF @@ -2166,7 +2166,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Squid Web Proxy configure 5.0.2 +Squid Web Proxy configure 5.0.3 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -3270,7 +3270,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Squid Web Proxy $as_me 5.0.2, which was +It was created by Squid Web Proxy $as_me 5.0.3, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4137,7 +4137,7 @@ # Define the identity of the package. PACKAGE='squid' - VERSION='5.0.2' + VERSION='5.0.3' cat >>confdefs.h <<_ACEOF @@ -21097,8 +21097,9 @@ $as_echo "$as_me: With dl" >&6;} fi + + ## check for atomics library before anything that might need it -# AC_SEARCH_LIBS pollutes LIBS # save state, key is LIBATOMIC LIBATOMIC_CFLAGS="${CFLAGS}" @@ -21115,63 +21116,76 @@ eval "${squid_util_var_tosave2}=\"${squid_util_var_tosave}\"" done -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing __atomic_load_8" >&5 -$as_echo_n "checking for library containing __atomic_load_8... " >&6; } -if ${ac_cv_search___atomic_load_8+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether linking without -latomic works" >&5 +$as_echo_n "checking whether linking without -latomic works... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char __atomic_load_8 (); -int -main () -{ -return __atomic_load_8 (); - ; - return 0; -} + +#include +#include + int + main(int, char **) { + return std::atomic{}.is_lock_free() ? 0 : 1; + } + _ACEOF -for ac_lib in '' atomic; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_cxx_try_link "$LINENO"; then : - ac_cv_search___atomic_load_8=$ac_res +if ac_fn_cxx_try_link "$LINENO"; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + libatomic_checker_result="yes" +else + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + libatomic_checker_result="no" + fi rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search___atomic_load_8+:} false; then : - break -fi -done -if ${ac_cv_search___atomic_load_8+:} false; then : + conftest$ac_exeext conftest.$ac_ext +if test "x$libatomic_checker_result" = "xno"; then : + + LIBS="$LIBS -latomic" + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether linking with -latomic works" >&5 +$as_echo_n "checking whether linking with -latomic works... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +#include +#include + int + main(int, char **) { + return std::atomic{}.is_lock_free() ? 0 : 1; + } +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + libatomic_checker_result="yes" else - ac_cv_search___atomic_load_8=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search___atomic_load_8" >&5 -$as_echo "$ac_cv_search___atomic_load_8" >&6; } -ac_res=$ac_cv_search___atomic_load_8 -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - test "$ac_res" = "none required" || ATOMICLIB=$ac_res + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + libatomic_checker_result="no" + fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + if test "x$libatomic_checker_result" = "xyes"; then : + + ATOMICLIB="-latomic" +else + + as_fn_error $? "Required library libatomic not found." "$LINENO" 5 +fi +fi # rollback state, key is LIBATOMIC CFLAGS="${LIBATOMIC_CFLAGS}" @@ -42692,6 +42706,9 @@ #if USE_SOLARIS_IPFILTER_MINOR_T_HACK #define minor_t fubar #endif +#if HAVE_SYS_PARAM_H +#include +#endif #if HAVE_SYS_TYPES_H #include #endif @@ -42717,7 +42734,11 @@ #elif HAVE_NETINET_IP_FIL_H #include #endif +#if HAVE_IP_NAT_H #include +#elif HAVE_NETINET_IP_NAT_H +#include +#endif " if test "x$ac_cv_member_struct_natlookup_nl_inipaddr_in6" = xyes; then : @@ -42728,11 +42749,13 @@ fi -ac_fn_cxx_check_member "$LINENO" "struct natlookup" "nl_realipaddr.in6" - "ac_cv_member_struct_natlookup_nl_realipaddr_in6___" " +ac_fn_cxx_check_member "$LINENO" "struct natlookup" "nl_realipaddr.in6" "ac_cv_member_struct_natlookup_nl_realipaddr_in6" " #if USE_SOLARIS_IPFILTER_MINOR_T_HACK #define minor_t fubar #endif +#if HAVE_SYS_PARAM_H +#include +#endif #if HAVE_SYS_TYPES_H #include #endif @@ -42758,13 +42781,17 @@ #elif HAVE_NETINET_IP_FIL_H #include #endif +#if HAVE_IP_NAT_H #include +#elif HAVE_NETINET_IP_NAT_H +#include +#endif " -if test "x$ac_cv_member_struct_natlookup_nl_realipaddr_in6___" = xyes; then : +if test "x$ac_cv_member_struct_natlookup_nl_realipaddr_in6" = xyes; then : cat >>confdefs.h <<_ACEOF -#define HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6___ 1 +#define HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6 1 _ACEOF @@ -44931,7 +44958,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Squid Web Proxy $as_me 5.0.2, which was +This file was extended by Squid Web Proxy $as_me 5.0.3, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -44997,7 +45024,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Squid Web Proxy config.status 5.0.2 +Squid Web Proxy config.status 5.0.3 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -u -r -N squid-5.0.2/configure.ac squid-5.0.3/configure.ac --- squid-5.0.2/configure.ac 2020-04-20 00:08:55.000000000 +1200 +++ squid-5.0.3/configure.ac 2020-06-09 18:58:26.000000000 +1200 @@ -5,7 +5,7 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## -AC_INIT([Squid Web Proxy],[5.0.2],[http://bugs.squid-cache.org/],[squid]) +AC_INIT([Squid Web Proxy],[5.0.3],[http://bugs.squid-cache.org/],[squid]) AC_PREREQ(2.61) AC_CONFIG_HEADERS([include/autoconf.h]) AC_CONFIG_AUX_DIR(cfgaux) @@ -440,11 +440,33 @@ AC_MSG_NOTICE([With dl]) fi +AC_DEFUN([LIBATOMIC_CHECKER],[ + AC_MSG_CHECKING(whether linking $1 -latomic works) + AC_LINK_IFELSE([ + AC_LANG_SOURCE([[ +#include +#include + int + main(int, char **) { + return std::atomic{}.is_lock_free() ? 0 : 1; + } + ]])],[ + AC_MSG_RESULT(yes) + libatomic_checker_result="yes"],[ + AC_MSG_RESULT(no) + libatomic_checker_result="no" +])]) + ## check for atomics library before anything that might need it -# AC_SEARCH_LIBS pollutes LIBS SQUID_STATE_SAVE(LIBATOMIC) -AC_SEARCH_LIBS([__atomic_load_8],[atomic],[ - test "$ac_res" = "none required" || ATOMICLIB=$ac_res],[]) +LIBATOMIC_CHECKER(without) +AS_IF([test "x$libatomic_checker_result" = "xno"],[ + LIBS="$LIBS -latomic" + LIBATOMIC_CHECKER(with) + AS_IF([test "x$libatomic_checker_result" = "xyes"],[ + ATOMICLIB="-latomic"],[ + AC_MSG_ERROR([Required library libatomic not found.]) +])]) SQUID_STATE_ROLLBACK(LIBATOMIC) AC_SUBST(ATOMICLIB) diff -u -r -N squid-5.0.2/CONTRIBUTORS squid-5.0.3/CONTRIBUTORS --- squid-5.0.2/CONTRIBUTORS 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/CONTRIBUTORS 2020-06-08 03:33:42.000000000 +1200 @@ -25,6 +25,7 @@ Alex Wu Alin Nastac Alter + Amit Klein Amos Jeffries Amos Jeffries Amos Jeffries diff -u -r -N squid-5.0.2/doc/release-notes/release-5.html squid-5.0.3/doc/release-notes/release-5.html --- squid-5.0.2/doc/release-notes/release-5.html 2020-04-20 00:20:13.000000000 +1200 +++ squid-5.0.3/doc/release-notes/release-5.html 2020-06-09 19:09:17.000000000 +1200 @@ -1,12 +1,12 @@ - + - Squid 5.0.2 release notes + Squid 5.0.3 release notes -

Squid 5.0.2 release notes

+

Squid 5.0.3 release notes

Squid Developers

@@ -61,7 +61,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-5.0.2 for testing.

+

The Squid Team are pleased to announce the release of Squid-5.0.3 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v5/ or the mirrors.

@@ -326,6 +326,10 @@

New code %A to display Squid listening IP address the client TCP connection was connected to.

+
http_port
+

New worker-queues option to have TCP stack maintain dedicated +listening queue for each worker in SMP.

+
logformat

New ssl::<cert macro code to display received server X.509 certificate in PEM format.

@@ -402,6 +406,10 @@
--disable-optimizations

No longer implies --disable-inline option (which is removed).

+
--enable-external-acl-helpers
+

New helper type kerberos_sid_group to match group= +annotations AD Domain group SID.

+

4.3 Removed options diff -u -r -N squid-5.0.2/include/autoconf.h.in squid-5.0.3/include/autoconf.h.in --- squid-5.0.2/include/autoconf.h.in 2020-04-20 00:08:47.000000000 +1200 +++ squid-5.0.3/include/autoconf.h.in 2020-06-09 18:58:17.000000000 +1200 @@ -1041,8 +1041,8 @@ /* Define to 1 if `nl_inipaddr.in6' is a member of `struct natlookup'. */ #undef HAVE_STRUCT_NATLOOKUP_NL_INIPADDR_IN6 -/* Define to 1 if `nl_realipaddr.in6' is a member of `struct natlookup '. */ -#undef HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6___ +/* Define to 1 if `nl_realipaddr.in6' is a member of `struct natlookup'. */ +#undef HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6 /* The system provides struct rusage */ #undef HAVE_STRUCT_RUSAGE diff -u -r -N squid-5.0.2/include/version.h squid-5.0.3/include/version.h --- squid-5.0.2/include/version.h 2020-04-20 00:08:55.000000000 +1200 +++ squid-5.0.3/include/version.h 2020-06-09 18:58:26.000000000 +1200 @@ -7,7 +7,7 @@ */ #ifndef SQUID_RELEASE_TIME -#define SQUID_RELEASE_TIME 1587298122 +#define SQUID_RELEASE_TIME 1591685891 #endif /* diff -u -r -N squid-5.0.2/RELEASENOTES.html squid-5.0.3/RELEASENOTES.html --- squid-5.0.2/RELEASENOTES.html 2020-04-20 00:20:13.000000000 +1200 +++ squid-5.0.3/RELEASENOTES.html 2020-06-09 19:09:17.000000000 +1200 @@ -1,12 +1,12 @@ - + - Squid 5.0.2 release notes + Squid 5.0.3 release notes -

Squid 5.0.2 release notes

+

Squid 5.0.3 release notes

Squid Developers

@@ -61,7 +61,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-5.0.2 for testing.

+

The Squid Team are pleased to announce the release of Squid-5.0.3 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v5/ or the mirrors.

@@ -326,6 +326,10 @@

New code %A to display Squid listening IP address the client TCP connection was connected to.

+
http_port
+

New worker-queues option to have TCP stack maintain dedicated +listening queue for each worker in SMP.

+
logformat

New ssl::<cert macro code to display received server X.509 certificate in PEM format.

@@ -402,6 +406,10 @@
--disable-optimizations

No longer implies --disable-inline option (which is removed).

+
--enable-external-acl-helpers
+

New helper type kerberos_sid_group to match group= +annotations AD Domain group SID.

+

4.3 Removed options diff -u -r -N squid-5.0.2/src/acl/external/delayer/ext_delayer_acl.8 squid-5.0.3/src/acl/external/delayer/ext_delayer_acl.8 --- squid-5.0.2/src/acl/external/delayer/ext_delayer_acl.8 2020-04-20 00:20:15.000000000 +1200 +++ squid-5.0.3/src/acl/external/delayer/ext_delayer_acl.8 2020-06-09 19:09:20.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_DELAYER_ACL 8" -.TH EXT_DELAYER_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_DELAYER_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc squid-5.0.3/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc --- squid-5.0.2/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc 2020-06-08 03:33:42.000000000 +1200 @@ -69,6 +69,12 @@ #ifdef HAVE_NETDB_H #include #endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif #ifdef HELPER_INPUT_BUFFER #define EDUI_MAXLEN HELPER_INPUT_BUFFER diff -u -r -N squid-5.0.2/src/acl/external/kerberos_ldap_group/support_krb5.cc squid-5.0.3/src/acl/external/kerberos_ldap_group/support_krb5.cc --- squid-5.0.2/src/acl/external/kerberos_ldap_group/support_krb5.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/acl/external/kerberos_ldap_group/support_krb5.cc 2020-06-08 03:33:42.000000000 +1200 @@ -467,10 +467,15 @@ } // overwrite limitation of enctypes +#if USE_HEIMDAL_KRB5 + creds->session.keytype = 0; + if (creds->session.keyvalue.length > 0) + krb5_free_keyblock_contents(kparam.context, &creds->session); +#else creds->keyblock.enctype = 0; if (creds->keyblock.contents) krb5_free_keyblock_contents(kparam.context, &creds->keyblock); - +#endif code = krb5_get_credentials(kparam.context, 0, kparam.cc[ccindex], creds, &tgt_creds); if (code) { k5_error("Error while getting tgt", code); diff -u -r -N squid-5.0.2/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 squid-5.0.3/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 --- squid-5.0.2/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 2020-04-20 00:20:15.000000000 +1200 +++ squid-5.0.3/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 2020-06-09 19:09:20.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_KERBEROS_SID_GROUP_ACL 8" -.TH EXT_KERBEROS_SID_GROUP_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_KERBEROS_SID_GROUP_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/acl/external/session/ext_session_acl.cc squid-5.0.3/src/acl/external/session/ext_session_acl.cc --- squid-5.0.2/src/acl/external/session/ext_session_acl.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/acl/external/session/ext_session_acl.cc 2020-06-08 03:33:42.000000000 +1200 @@ -137,6 +137,10 @@ } } #elif USE_TRIVIALDB +#if _SQUID_FREEBSD_ && !defined(O_DSYNC) + // FreeBSD lacks O_DSYNC, O_SYNC is closest to correct behaviour +#define O_DSYNC O_SYNC +#endif db = tdb_open(db_path, 0, TDB_CLEAR_IF_FIRST, O_CREAT|O_DSYNC, 0666); #endif if (!db) { diff -u -r -N squid-5.0.2/src/acl/external/SQL_session/ext_sql_session_acl.8 squid-5.0.3/src/acl/external/SQL_session/ext_sql_session_acl.8 --- squid-5.0.2/src/acl/external/SQL_session/ext_sql_session_acl.8 2020-04-20 00:20:16.000000000 +1200 +++ squid-5.0.3/src/acl/external/SQL_session/ext_sql_session_acl.8 2020-06-09 19:09:20.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_SQL_SESSION_ACL 8" -.TH EXT_SQL_SESSION_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_SQL_SESSION_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 squid-5.0.3/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 --- squid-5.0.2/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2020-04-20 00:20:16.000000000 +1200 +++ squid-5.0.3/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2020-06-09 19:09:20.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_WBINFO_GROUP_ACL 8" -.TH EXT_WBINFO_GROUP_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_WBINFO_GROUP_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/anyp/ProtocolVersion.h squid-5.0.3/src/anyp/ProtocolVersion.h --- squid-5.0.2/src/anyp/ProtocolVersion.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/anyp/ProtocolVersion.h 2020-06-08 03:33:42.000000000 +1200 @@ -40,6 +40,9 @@ unsigned int major; ///< major version number unsigned int minor; ///< minor version number + /// whether the version is "known" (e.g., has been parsed or explicitly set) + explicit operator bool() const { return protocol != PROTO_NONE; } + bool operator==(const ProtocolVersion& that) const { if (this->protocol != that.protocol) return false; diff -u -r -N squid-5.0.2/src/anyp/Uri.cc squid-5.0.3/src/anyp/Uri.cc --- squid-5.0.2/src/anyp/Uri.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/anyp/Uri.cc 2020-06-08 03:33:42.000000000 +1200 @@ -30,6 +30,56 @@ "[:]" ; +/// Characters which are valid within a URI userinfo section +static const CharacterSet & +UserInfoChars() +{ + /* + * RFC 3986 section 3.2.1 + * + * userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + * unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + * pct-encoded = "%" HEXDIG HEXDIG + * sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" + */ + static const auto userInfoValid = CharacterSet("userinfo", ":-._~%!$&'()*+,;=") + + CharacterSet::ALPHA + + CharacterSet::DIGIT; + return userInfoValid; +} + +/** + * Governed by RFC 3986 section 2.1 + */ +SBuf +AnyP::Uri::Encode(const SBuf &buf, const CharacterSet &ignore) +{ + if (buf.isEmpty()) + return buf; + + Parser::Tokenizer tk(buf); + SBuf goodSection; + // optimization for the arguably common "no encoding necessary" case + if (tk.prefix(goodSection, ignore) && tk.atEnd()) + return buf; + + SBuf output; + output.reserveSpace(buf.length() * 3); // worst case: encode all chars + output.append(goodSection); // may be empty + + while (!tk.atEnd()) { + // TODO: Add Tokenizer::parseOne(void). + const auto ch = tk.remaining()[0]; + output.appendf("%%%02X", static_cast(ch)); // TODO: Optimize using a table + (void)tk.skip(ch); + + if (tk.prefix(goodSection, ignore)) + output.append(goodSection); + } + + return output; +} + const SBuf & AnyP::Uri::Asterisk() { @@ -557,7 +607,10 @@ getScheme() == AnyP::PROTO_UNKNOWN; if (allowUserInfo && !userInfo().isEmpty()) { - absolute_.append(userInfo()); + static const CharacterSet uiChars = CharacterSet(UserInfoChars()) + .remove('%') + .rename("userinfo-reserved"); + absolute_.append(Encode(userInfo(), uiChars)); absolute_.append("@", 1); } absolute_.append(authority()); @@ -565,7 +618,7 @@ absolute_.append(host()); absolute_.append(":", 1); } - absolute_.append(path()); + absolute_.append(path()); // TODO: Encode each URI subcomponent in path_ as needed. } return absolute_; @@ -619,102 +672,76 @@ return request->canonicalCleanUrl(); } -/* - * Test if a URL is relative. +/** + * Test if a URL is a relative reference. + * + * Governed by RFC 3986 section 4.2 + * + * relative-ref = relative-part [ "?" query ] [ "#" fragment ] * - * RFC 2396, Section 5 (Page 17) implies that in a relative URL, a '/' will - * appear before a ':'. + * relative-part = "//" authority path-abempty + * / path-absolute + * / path-noscheme + * / path-empty */ bool urlIsRelative(const char *url) { - const char *p; + if (!url) + return false; // no URL - if (url == NULL) { - return (false); - } - if (*url == '\0') { - return (false); - } + /* + * RFC 3986 section 5.2.3 + * + * path = path-abempty ; begins with "/" or is empty + * / path-absolute ; begins with "/" but not "//" + * / path-noscheme ; begins with a non-colon segment + * / path-rootless ; begins with a segment + * / path-empty ; zero characters + */ - for (p = url; *p != '\0' && *p != ':' && *p != '/'; ++p); + if (*url == '\0') + return true; // path-empty - if (*p == ':') { - return (false); + if (*url == '/') { + // RFC 3986 section 5.2.3 + // path-absolute ; begins with "/" but not "//" + if (url[1] == '/') + return true; // network-path reference, aka. 'scheme-relative URI' + else + return true; // path-absolute, aka 'absolute-path reference' } - return (true); -} - -/* - * Convert a relative URL to an absolute URL using the context of a given - * request. - * - * It is assumed that you have already ensured that the URL is relative. - * - * If NULL is returned it is an indication that the method in use in the - * request does not distinguish between relative and absolute and you should - * use the url unchanged. - * - * If non-NULL is returned, it is up to the caller to free the resulting - * memory using safe_free(). - */ -char * -urlMakeAbsolute(const HttpRequest * req, const char *relUrl) -{ - if (req->method.id() == Http::METHOD_CONNECT) { - return (NULL); + for (const auto *p = url; *p != '\0' && *p != '/' && *p != '?' && *p != '#'; ++p) { + if (*p == ':') + return false; // colon is forbidden in first segment } - char *urlbuf = (char *)xmalloc(MAX_URL * sizeof(char)); + return true; // path-noscheme, path-abempty, path-rootless +} - if (req->url.getScheme() == AnyP::PROTO_URN) { - // XXX: this is what the original code did, but it seems to break the - // intended behaviour of this function. It returns the stored URN path, - // not converting the given one into a URN... - snprintf(urlbuf, MAX_URL, SQUIDSBUFPH, SQUIDSBUFPRINT(req->url.absolute())); - return (urlbuf); - } +void +AnyP::Uri::addRelativePath(const char *relUrl) +{ + // URN cannot be merged + if (getScheme() == AnyP::PROTO_URN) + return; - SBuf authorityForm = req->url.authority(); // host[:port] - const SBuf &scheme = req->url.getScheme().image(); - size_t urllen = snprintf(urlbuf, MAX_URL, SQUIDSBUFPH "://" SQUIDSBUFPH "%s" SQUIDSBUFPH, - SQUIDSBUFPRINT(scheme), - SQUIDSBUFPRINT(req->url.userInfo()), - !req->url.userInfo().isEmpty() ? "@" : "", - SQUIDSBUFPRINT(authorityForm)); + // TODO: Handle . and .. segment normalization - // if the first char is '/' assume its a relative path - // XXX: this breaks on scheme-relative URLs, - // but we should not see those outside ESI, and rarely there. - // XXX: also breaks on any URL containing a '/' in the query-string portion - if (relUrl[0] == '/') { - xstrncpy(&urlbuf[urllen], relUrl, MAX_URL - urllen - 1); + const auto lastSlashPos = path_.rfind('/'); + // TODO: To optimize and simplify, add and use SBuf::replace(). + const auto relUrlLength = strlen(relUrl); + if (lastSlashPos == SBuf::npos) { + // start replacing the whole path + path_.reserveCapacity(1 + relUrlLength); + path_.assign("/", 1); } else { - SBuf path = req->url.path(); - SBuf::size_type lastSlashPos = path.rfind('/'); - - if (lastSlashPos == SBuf::npos) { - // replace the whole path with the given bit(s) - urlbuf[urllen] = '/'; - ++urllen; - xstrncpy(&urlbuf[urllen], relUrl, MAX_URL - urllen - 1); - } else { - // replace only the last (file?) segment with the given bit(s) - ++lastSlashPos; - if (lastSlashPos > MAX_URL - urllen - 1) { - // XXX: crops bits in the middle of the combined URL. - lastSlashPos = MAX_URL - urllen - 1; - } - SBufToCstring(&urlbuf[urllen], path.substr(0,lastSlashPos)); - urllen += lastSlashPos; - if (urllen + 1 < MAX_URL) { - xstrncpy(&urlbuf[urllen], relUrl, MAX_URL - urllen - 1); - } - } + // start replacing just the last segment + path_.reserveCapacity(lastSlashPos + 1 + relUrlLength); + path_.chop(0, lastSlashPos+1); } - - return (urlbuf); + path_.append(relUrl, relUrlLength); } int @@ -890,105 +917,6 @@ return rc; } -/* - * Quick-n-dirty host extraction from a URL. Steps: - * Look for a colon - * Skip any '/' after the colon - * Copy the next SQUID_MAXHOSTNAMELEN bytes to host[] - * Look for an ending '/' or ':' and terminate - * Look for login info preceeded by '@' - */ - -class URLHostName -{ - -public: - char * extract(char const *url); - -private: - static char Host [SQUIDHOSTNAMELEN]; - void init(char const *); - void findHostStart(); - void trimTrailingChars(); - void trimAuth(); - char const *hostStart; - char const *url; -}; - -char * -urlHostname(const char *url) -{ - return URLHostName().extract(url); -} - -char URLHostName::Host[SQUIDHOSTNAMELEN]; - -void -URLHostName::init(char const *aUrl) -{ - Host[0] = '\0'; - url = aUrl; -} - -void -URLHostName::findHostStart() -{ - if (NULL == (hostStart = strchr(url, ':'))) - return; - - ++hostStart; - - while (*hostStart != '\0' && *hostStart == '/') - ++hostStart; - - if (*hostStart == ']') - ++hostStart; -} - -void -URLHostName::trimTrailingChars() -{ - char *t; - - if ((t = strchr(Host, '/'))) - *t = '\0'; - - if ((t = strrchr(Host, ':'))) - *t = '\0'; - - if ((t = strchr(Host, ']'))) - *t = '\0'; -} - -void -URLHostName::trimAuth() -{ - char *t; - - if ((t = strrchr(Host, '@'))) { - ++t; - memmove(Host, t, strlen(t) + 1); - } -} - -char * -URLHostName::extract(char const *aUrl) -{ - init(aUrl); - findHostStart(); - - if (hostStart == NULL) - return NULL; - - xstrncpy(Host, hostStart, SQUIDHOSTNAMELEN); - - trimTrailingChars(); - - trimAuth(); - - return Host; -} - AnyP::Uri::Uri(AnyP::UriScheme const &aScheme) : scheme_(aScheme), hostIsNumeric_(false), diff -u -r -N squid-5.0.2/src/anyp/Uri.h squid-5.0.3/src/anyp/Uri.h --- squid-5.0.2/src/anyp/Uri.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/anyp/Uri.h 2020-06-08 03:33:42.000000000 +1200 @@ -77,6 +77,8 @@ } void userInfo(const SBuf &s) {userInfo_=s; touch();} + /// \returns raw userinfo subcomponent (or an empty string) + /// the caller is responsible for caller-specific encoding const SBuf &userInfo() const {return userInfo_;} void host(const char *src); @@ -98,12 +100,27 @@ void path(const SBuf &p) {path_=p; touch();} const SBuf &path() const; + /** + * Merge a relative-path URL into the existing URI details. + * Implements RFC 3986 section 5.2.3 + * + * The caller must ensure relUrl is a valid relative-path. + * + * NP: absolute-path are also accepted, but path() method + * should be used instead when possible. + */ + void addRelativePath(const char *relUrl); + /// the static '/' default URL-path static const SBuf &SlashPath(); /// the static '*' pseudo-URI static const SBuf &Asterisk(); + /// %-encode characters in a buffer which do not conform to + /// the provided set of expected characters. + static SBuf Encode(const SBuf &, const CharacterSet &expected); + /** * The authority-form URI for currently stored values. * @@ -199,7 +216,6 @@ char *urlCanonicalCleanWithoutRequest(const SBuf &url, const HttpRequestMethod &, const AnyP::UriScheme &); const char *urlCanonicalFakeHttps(const HttpRequest * request); bool urlIsRelative(const char *); -char *urlMakeAbsolute(const HttpRequest *, const char *); char *urlRInternal(const char *host, unsigned short port, const char *dir, const char *name); char *urlInternal(const char *dir, const char *name); bool urlAppendDomain(char *host); ///< apply append_domain config to the given hostname @@ -245,7 +261,6 @@ */ int matchDomainName(const char *host, const char *domain, MatchDomainNameFlags flags = mdnNone); int urlCheckRequest(const HttpRequest *); -char *urlHostname(const char *url); void urlExtMethodConfigure(void); #endif /* SQUID_SRC_ANYP_URI_H */ diff -u -r -N squid-5.0.2/src/auth/basic/DB/basic_db_auth.8 squid-5.0.3/src/auth/basic/DB/basic_db_auth.8 --- squid-5.0.2/src/auth/basic/DB/basic_db_auth.8 2020-04-20 00:20:16.000000000 +1200 +++ squid-5.0.3/src/auth/basic/DB/basic_db_auth.8 2020-06-09 19:09:21.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_DB_AUTH 8" -.TH BASIC_DB_AUTH 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH BASIC_DB_AUTH 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/auth/basic/POP3/basic_pop3_auth.8 squid-5.0.3/src/auth/basic/POP3/basic_pop3_auth.8 --- squid-5.0.2/src/auth/basic/POP3/basic_pop3_auth.8 2020-04-20 00:20:17.000000000 +1200 +++ squid-5.0.3/src/auth/basic/POP3/basic_pop3_auth.8 2020-06-09 19:09:21.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_POP3_AUTH 8" -.TH BASIC_POP3_AUTH 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH BASIC_POP3_AUTH 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/base/CharacterSet.cc squid-5.0.3/src/base/CharacterSet.cc --- squid-5.0.2/src/base/CharacterSet.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/base/CharacterSet.cc 2020-06-08 03:33:42.000000000 +1200 @@ -51,6 +51,13 @@ } CharacterSet & +CharacterSet::remove(const unsigned char c) +{ + chars_[static_cast(c)] = 0; + return *this; +} + +CharacterSet & CharacterSet::addRange(unsigned char low, unsigned char high) { //manual loop splitting is needed to cover case where high is 255 diff -u -r -N squid-5.0.2/src/base/CharacterSet.h squid-5.0.3/src/base/CharacterSet.h --- squid-5.0.2/src/base/CharacterSet.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/base/CharacterSet.h 2020-06-08 03:33:42.000000000 +1200 @@ -41,6 +41,9 @@ /// add a given character to the character set CharacterSet & add(const unsigned char c); + /// remove a given character from the character set + CharacterSet & remove(const unsigned char c); + /// add a list of character ranges, expressed as pairs [low,high], including both ends CharacterSet & addRange(unsigned char low, unsigned char high); diff -u -r -N squid-5.0.2/src/ClientInfo.h squid-5.0.3/src/ClientInfo.h --- squid-5.0.2/src/ClientInfo.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ClientInfo.h 2020-06-08 03:33:42.000000000 +1200 @@ -78,6 +78,8 @@ unsigned int quotaPeekReserv() const; ///< returns the next reserv. to pop void quotaDequeue(); ///< pops queue head from queue void kickQuotaQueue(); ///< schedule commHandleWriteHelper call + /// either selects the head descriptor for writing or calls quotaDequeue() + void writeOrDequeue(); /* BandwidthBucket API */ virtual int quota() override; ///< allocate quota for a just dequeued client diff -u -r -N squid-5.0.2/src/clients/Client.cc squid-5.0.3/src/clients/Client.cc --- squid-5.0.2/src/clients/Client.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/clients/Client.cc 2020-06-08 03:33:42.000000000 +1200 @@ -426,7 +426,7 @@ static bool sameUrlHosts(const char *url1, const char *url2) { - // XXX: Want urlHostname() here, but it uses static storage and copying + // XXX: Want AnyP::Uri::parse() here, but it uses static storage and copying const char *host1 = strchr(url1, ':'); const char *host2 = strchr(url2, ':'); @@ -455,33 +455,40 @@ static void purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr) { - const char *hdrUrl, *absUrl; - - absUrl = NULL; - hdrUrl = rep->header.getStr(hdr); - if (hdrUrl == NULL) { + const auto hdrUrl = rep->header.getStr(hdr); + if (!hdrUrl) return; - } /* * If the URL is relative, make it absolute so we can find it. * If it's absolute, make sure the host parts match to avoid DOS attacks * as per RFC 2616 13.10. */ + SBuf absUrlMaker; + const char *absUrl = nullptr; if (urlIsRelative(hdrUrl)) { - absUrl = urlMakeAbsolute(req, hdrUrl); - if (absUrl != NULL) { - hdrUrl = absUrl; + if (req->method.id() == Http::METHOD_CONNECT) + absUrl = hdrUrl; // TODO: merge authority-uri and hdrUrl + else if (req->url.getScheme() == AnyP::PROTO_URN) + absUrl = req->url.absolute().c_str(); + else { + AnyP::Uri tmpUrl = req->url; + if (*hdrUrl == '/') { + // RFC 3986 section 4.2: absolute-path reference + // for this logic replace the entire request-target URI path + tmpUrl.path(hdrUrl); + } else { + tmpUrl.addRelativePath(reqUrl); + } + absUrlMaker = tmpUrl.absolute(); + absUrl = absUrlMaker.c_str(); } } else if (!sameUrlHosts(reqUrl, hdrUrl)) { return; - } + } else + absUrl = hdrUrl; - purgeEntriesByUrl(req, hdrUrl); - - if (absUrl != NULL) { - safe_free(absUrl); - } + purgeEntriesByUrl(req, absUrl); } // some HTTP methods should purge matching cache entries diff -u -r -N squid-5.0.2/src/clients/HttpTunnelerAnswer.cc squid-5.0.3/src/clients/HttpTunnelerAnswer.cc --- squid-5.0.2/src/clients/HttpTunnelerAnswer.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/clients/HttpTunnelerAnswer.cc 2020-06-08 03:33:42.000000000 +1200 @@ -32,6 +32,9 @@ if (answer.peerResponseStatus != Http::scNone) os << ' ' << answer.peerResponseStatus; + if (answer.conn) + os << ' ' << answer.conn; + os << ']'; return os; } diff -u -r -N squid-5.0.2/src/clients/HttpTunnelerAnswer.h squid-5.0.3/src/clients/HttpTunnelerAnswer.h --- squid-5.0.2/src/clients/HttpTunnelerAnswer.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/clients/HttpTunnelerAnswer.h 2020-06-08 03:33:42.000000000 +1200 @@ -43,6 +43,8 @@ /// the status code of the successfully parsed CONNECT response (or scNone) StatusCode peerResponseStatus = scNone; + + Comm::ConnectionPointer conn; }; std::ostream &operator <<(std::ostream &, const Http::TunnelerAnswer &); diff -u -r -N squid-5.0.2/src/clients/HttpTunneler.cc squid-5.0.3/src/clients/HttpTunneler.cc --- squid-5.0.2/src/clients/HttpTunneler.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/clients/HttpTunneler.cc 2020-06-08 03:33:42.000000000 +1200 @@ -18,6 +18,8 @@ #include "http/one/ResponseParser.h" #include "http/StateFlags.h" #include "HttpRequest.h" +#include "neighbors.h" +#include "pconn.h" #include "SquidConfig.h" #include "StatCounters.h" @@ -25,6 +27,7 @@ Http::Tunneler::Tunneler(const Comm::ConnectionPointer &conn, const HttpRequest::Pointer &req, AsyncCall::Pointer &aCallback, time_t timeout, const AccessLogEntryPointer &alp): AsyncJob("Http::Tunneler"), + noteFwdPconnUse(false), connection(conn), request(req), callback(aCallback), @@ -41,6 +44,7 @@ assert(callback); assert(dynamic_cast(callback->getDialer())); url = request->url.authority(); + watchForClosures(); } Http::Tunneler::~Tunneler() @@ -73,11 +77,22 @@ Must(url.length()); Must(lifetimeLimit >= 0); + // we own this Comm::Connection object and its fd exclusively, but must bail + // if others started closing the socket while we were waiting to start() + assert(Comm::IsConnOpen(connection)); + if (fd_table[connection->fd].closing()) { + bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw(), al)); + return; + } + const auto peer = connection->getPeer(); - Must(peer); // bail if our peer was reconfigured away + // bail if our peer was reconfigured away + if (!peer) { + bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scInternalServerError, request.getRaw(), al)); + return; + } request->prepForPeering(*peer); - watchForClosures(); writeRequest(); startReadingResponse(); } @@ -85,8 +100,8 @@ void Http::Tunneler::handleConnectionClosure(const CommCloseCbParams ¶ms) { - mustStop("server connection gone"); - callback = nullptr; // the caller must monitor closures + closer = nullptr; + bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw(), al)); } /// make sure we quit if/when the connection is gone @@ -104,12 +119,11 @@ comm_add_close_handler(connection->fd, closer); } +/// The connection read timeout callback handler. void -Http::Tunneler::handleException(const std::exception& e) +Http::Tunneler::handleTimeout(const CommTimeoutCbParams &) { - debugs(83, 2, e.what() << status()); - connection->close(); - bailWith(new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw(), al)); + bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scGatewayTimeout, request.getRaw(), al)); } void @@ -255,8 +269,11 @@ Comm::Read(connection, reader); AsyncCall::Pointer nil; + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(93, 5, + TimeoutDialer, this, Http::Tunneler::handleTimeout); const auto timeout = Comm::MortalReadTimeout(startTime, lifetimeLimit); - commSetConnTimeout(connection, timeout, nil); + commSetConnTimeout(connection, timeout, timeoutCall); } /// Parses [possibly incomplete] CONNECT response and reacts to it. @@ -342,13 +359,51 @@ { Must(error); answer().squidError = error; + + if (const auto p = connection->getPeer()) + peerConnectFailed(p); + callBack(); + disconnect(); + + if (noteFwdPconnUse) + fwdPconnPool->noteUses(fd_table[connection->fd].pconn.uses); + // TODO: Reuse to-peer connections after a CONNECT error response. + connection->close(); + connection = nullptr; +} + +void +Http::Tunneler::sendSuccess() +{ + assert(answer().positive()); + callBack(); + disconnect(); +} + +void +Http::Tunneler::disconnect() +{ + if (closer) { + comm_remove_close_handler(connection->fd, closer); + closer = nullptr; + } + + if (reader) { + Comm::ReadCancel(connection->fd, reader); + reader = nullptr; + } + + // remove connection timeout handler + commUnsetConnTimeout(connection); } void Http::Tunneler::callBack() { debugs(83, 5, connection << status()); + if (answer().positive()) + answer().conn = connection; auto cb = callback; callback = nullptr; ScheduleCallHere(cb); @@ -361,8 +416,7 @@ if (callback) { if (requestWritten && tunnelEstablished) { - assert(answer().positive()); - callBack(); // success + sendSuccess(); } else { // we should have bailed when we discovered the job-killing problem debugs(83, DBG_IMPORTANT, "BUG: Unexpected state while establishing a CONNECT tunnel " << connection << status()); @@ -370,16 +424,6 @@ } assert(!callback); } - - if (closer) { - comm_remove_close_handler(connection->fd, closer); - closer = nullptr; - } - - if (reader) { - Comm::ReadCancel(connection->fd, reader); - reader = nullptr; - } } const char * diff -u -r -N squid-5.0.2/src/clients/HttpTunneler.h squid-5.0.3/src/clients/HttpTunneler.h --- squid-5.0.2/src/clients/HttpTunneler.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/clients/HttpTunneler.h 2020-06-08 03:33:42.000000000 +1200 @@ -26,15 +26,9 @@ namespace Http { -/// Establishes an HTTP CONNECT tunnel through a forward proxy. -/// -/// The caller receives a call back with Http::TunnelerAnswer. -/// -/// The caller must monitor the connection for closure because this job will not -/// inform the caller about such events. -/// -/// This job never closes the connection, even on errors. If a 3rd-party closes -/// the connection, this job simply quits without informing the caller. +/// Negotiates an HTTP CONNECT tunnel through a forward proxy using a given +/// (open and, if needed, encrypted) TCP connection to that proxy. Owns the +/// connection during these negotiations. The caller receives TunnelerAnswer. class Tunneler: virtual public AsyncJob { CBDATA_CLASS(Tunneler); @@ -71,6 +65,9 @@ void setDelayId(DelayId delay_id) {delayId = delay_id;} #endif + /// hack: whether the connection requires fwdPconnPool->noteUses() + bool noteFwdPconnUse; + protected: /* AsyncJob API */ virtual ~Tunneler(); @@ -81,7 +78,7 @@ void handleConnectionClosure(const CommCloseCbParams&); void watchForClosures(); - void handleException(const std::exception&); + void handleTimeout(const CommTimeoutCbParams &); void startReadingResponse(); void writeRequest(); void handleWrittenRequest(const CommIoCbParams&); @@ -89,9 +86,19 @@ void readMore(); void handleResponse(const bool eof); void bailOnResponseError(const char *error, HttpReply *); + + /// sends the given error to the initiator void bailWith(ErrorState*); + + /// sends the ready-to-use tunnel to the initiator + void sendSuccess(); + + /// a bailWith(), sendSuccess() helper: sends results to the initiator void callBack(); + /// a bailWith(), sendSuccess() helper: stops monitoring the connection + void disconnect(); + TunnelerAnswer &answer(); private: diff -u -r -N squid-5.0.2/src/client_side.cc squid-5.0.3/src/client_side.cc --- squid-5.0.2/src/client_side.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/client_side.cc 2020-06-08 03:33:42.000000000 +1200 @@ -2187,6 +2187,7 @@ bodyParser(nullptr), #if USE_OPENSSL sslBumpMode(Ssl::bumpEnd), + tlsParser(Security::HandshakeParser::fromClient), #endif needProxyProtocolHeader_(false), #if USE_OPENSSL diff -u -r -N squid-5.0.2/src/client_side_reply.cc squid-5.0.3/src/client_side_reply.cc --- squid-5.0.2/src/client_side_reply.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/client_side_reply.cc 2020-06-08 03:33:42.000000000 +1200 @@ -913,7 +913,7 @@ const cache_key *key = storeKeyPublic(url, m); debugs(88, 5, m << ' ' << url << ' ' << storeKeyText(key)); #if USE_HTCP - neighborsHtcpClear(nullptr, url, req, m, HTCP_CLR_INVALIDATION); + neighborsHtcpClear(nullptr, req, m, HTCP_CLR_INVALIDATION); #endif Store::Root().evictIfFound(key); } @@ -1051,7 +1051,7 @@ /* Release the cached URI */ debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" ); #if USE_HTCP - neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE); + neighborsHtcpClear(newEntry, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE); #endif newEntry->release(true); purgeStatus = Http::scOkay; @@ -1067,7 +1067,7 @@ if (newEntry) { debugs(88, 4, "HEAD " << newEntry->url()); #if USE_HTCP - neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE); + neighborsHtcpClear(newEntry, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE); #endif newEntry->release(true); purgeStatus = Http::scOkay; @@ -1083,7 +1083,7 @@ if (entry) { debugs(88, 4, "Vary GET " << entry->url()); #if USE_HTCP - neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE); + neighborsHtcpClear(entry, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE); #endif entry->release(true); purgeStatus = Http::scOkay; @@ -1094,7 +1094,7 @@ if (entry) { debugs(88, 4, "Vary HEAD " << entry->url()); #if USE_HTCP - neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE); + neighborsHtcpClear(entry, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE); #endif entry->release(true); purgeStatus = Http::scOkay; diff -u -r -N squid-5.0.2/src/comm/forward.h squid-5.0.3/src/comm/forward.h --- squid-5.0.2/src/comm/forward.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/comm/forward.h 2020-06-08 03:33:42.000000000 +1200 @@ -26,8 +26,6 @@ typedef RefCount ConnectionPointer; -typedef std::vector ConnectionList; - bool IsConnOpen(const Comm::ConnectionPointer &conn); // callback handler to process an FD which is available for writing. diff -u -r -N squid-5.0.2/src/comm.cc squid-5.0.3/src/comm.cc --- squid-5.0.2/src/comm.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/comm.cc 2020-06-08 03:33:42.000000000 +1200 @@ -1275,30 +1275,38 @@ assert(clientInfo); assert(clientInfo->hasQueue()); assert(clientInfo->hasQueue(queue)); - assert(!clientInfo->selectWaiting); assert(clientInfo->eventWaiting); clientInfo->eventWaiting = false; do { - // check that the head descriptor is still relevant - const int head = clientInfo->quotaPeekFd(); - Comm::IoCallback *ccb = COMMIO_FD_WRITECB(head); + clientInfo->writeOrDequeue(); + if (clientInfo->selectWaiting) + return; + } while (clientInfo->hasQueue()); - if (fd_table[head].clientInfo == clientInfo && - clientInfo->quotaPeekReserv() == ccb->quotaQueueReserv && - !fd_table[head].closing()) { + debugs(77, 3, "emptied queue"); +} + +void +ClientInfo::writeOrDequeue() +{ + assert(!selectWaiting); + const auto head = quotaPeekFd(); + const auto &headFde = fd_table[head]; + CallBack(headFde.codeContext, [&] { + const auto ccb = COMMIO_FD_WRITECB(head); + // check that the head descriptor is still relevant + if (headFde.clientInfo == this && + quotaPeekReserv() == ccb->quotaQueueReserv && + !headFde.closing()) { // wait for the head descriptor to become ready for writing Comm::SetSelect(head, COMM_SELECT_WRITE, Comm::HandleWrite, ccb, 0); - clientInfo->selectWaiting = true; - return; + selectWaiting = true; + } else { + quotaDequeue(); // remove the no longer relevant descriptor } - - clientInfo->quotaDequeue(); // remove the no longer relevant descriptor - // and continue looking for a relevant one - } while (clientInfo->hasQueue()); - - debugs(77,3, HERE << "emptied queue"); + }); } bool @@ -1474,6 +1482,7 @@ debugs(77,5, "clt" << (const char*)clientInfo->key << ": FD " << fd << " with qqid" << (ins+1) << ' ' << fds.size()); fds.push_back(fd); + fd_table[fd].codeContext = CodeContext::Current(); return ++ins; } @@ -1639,6 +1648,7 @@ debugs(5, 5, HERE << "adding FD " << fd << " to " << *TheHalfClosed); assert(isOpen(fd) && !commHasHalfClosedMonitor(fd)); (void)TheHalfClosed->add(fd); // could also assert the result + fd_table[fd].codeContext = CodeContext::Current(); commPlanHalfClosedCheck(); // may schedule check if we added the first FD } @@ -1666,10 +1676,12 @@ Comm::ConnectionPointer c = new Comm::Connection; // XXX: temporary. make HalfClosed a list of these. c->fd = *i; if (!fd_table[c->fd].halfClosedReader) { // not reading already - AsyncCall::Pointer call = commCbCall(5,4, "commHalfClosedReader", - CommIoCbPtrFun(&commHalfClosedReader, NULL)); - Comm::Read(c, call); - fd_table[c->fd].halfClosedReader = call; + CallBack(fd_table[c->fd].codeContext, [&c] { + AsyncCall::Pointer call = commCbCall(5,4, "commHalfClosedReader", + CommIoCbPtrFun(&commHalfClosedReader, nullptr)); + Comm::Read(c, call); + fd_table[c->fd].halfClosedReader = call; + }); } else c->fd = -1; // XXX: temporary. prevent c replacement erase closing listed FD } diff -u -r -N squid-5.0.2/src/format/Format.cc squid-5.0.3/src/format/Format.cc --- squid-5.0.2/src/format/Format.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/format/Format.cc 2020-06-08 03:33:42.000000000 +1200 @@ -341,15 +341,6 @@ *p = '\0'; } -#if USE_OPENSSL -static char * -sslErrorName(Security::ErrorCode err, char *buf, size_t size) -{ - snprintf(buf, size, "SSL_ERR=%d", err); - return buf; -} -#endif - /// XXX: Misnamed. TODO: Split request && al->request->errType == ERR_SECURE_CONNECT_FAIL) { - out = Ssl::GetErrorName(al->request->errDetail); - if (!out) - out = sslErrorName(al->request->errDetail, tmp, sizeof(tmp)); + out = Ssl::GetErrorName(al->request->errDetail, true); } else #endif if (al->request && al->request->errDetail != ERR_DETAIL_NONE) { @@ -1312,10 +1301,7 @@ for (const Security::CertErrors *sslError = srvBump->sslErrors(); sslError; sslError = sslError->next) { if (!sb.isEmpty()) sb.append(separator); - if (const char *errorName = Ssl::GetErrorName(sslError->element.code)) - sb.append(errorName); - else - sb.append(sslErrorName(sslError->element.code, tmp, sizeof(tmp))); + sb.append(Ssl::GetErrorName(sslError->element.code, true)); if (sslError->element.depth >= 0) sb.appendf("@depth=%d", sslError->element.depth); } diff -u -r -N squid-5.0.2/src/fs/rock/RockSwapDir.cc squid-5.0.3/src/fs/rock/RockSwapDir.cc --- squid-5.0.2/src/fs/rock/RockSwapDir.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/fs/rock/RockSwapDir.cc 2020-06-08 03:33:42.000000000 +1200 @@ -1136,19 +1136,14 @@ mapOwners.push_back(mapOwner); // TODO: somehow remove pool id and counters from PageStack? + Ipc::Mem::PageStack::Config config; + config.poolId = Ipc::Mem::PageStack::IdForSwapDirSpace(i); + config.pageSize = 0; // this is an index of slots on _disk_ + config.capacity = capacity; + config.createFull = false; // Rebuild finds and pushes free slots Ipc::Mem::Owner *const freeSlotsOwner = - shm_new(Ipc::Mem::PageStack)(sd->freeSlotsPath(), - Ipc::Mem::PageStack::IdForSwapDirSpace(i), - capacity, - 0); + shm_new(Ipc::Mem::PageStack)(sd->freeSlotsPath(), config); freeSlotsOwners.push_back(freeSlotsOwner); - - // TODO: add method to initialize PageStack with no free pages - while (true) { - Ipc::Mem::PageId pageId; - if (!freeSlotsOwner->object()->pop(pageId)) - break; - } } } } diff -u -r -N squid-5.0.2/src/FwdState.cc squid-5.0.3/src/FwdState.cc --- squid-5.0.2/src/FwdState.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/FwdState.cc 2020-06-08 03:33:42.000000000 +1200 @@ -119,6 +119,18 @@ } void +FwdState::closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason) +{ + debugs(17, 3, "because " << reason << "; " << conn); + assert(!serverConn); + assert(!closeHandler); + if (IsConnOpen(conn)) { + fwdPconnPool->noteUses(fd_table[conn->fd].pconn.uses); + conn->close(); + } +} + +void FwdState::closeServerConnection(const char *reason) { debugs(17, 3, "because " << reason << "; " << serverConn); @@ -753,6 +765,24 @@ retryOrBail(); } +/// starts a preparation step for an established connection; retries on failures +template +void +FwdState::advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep) +{ + // TODO: Extract destination-specific handling from FwdState so that all the + // awkward, limited-scope advanceDestination() calls can be replaced with a + // single simple try/catch,retry block. + try { + startStep(); + // now wait for the step callback + } catch (...) { + debugs (17, 2, "exception while trying to " << stepDescription << ": " << CurrentException); + closePendingConnection(conn, "connection preparation exception"); + retryOrBail(); + } +} + /// called when a to-peer connection has been successfully obtained or /// when all candidate destinations have been tried and all have failed void @@ -764,22 +794,31 @@ Must(n_tries <= answer.n_tries); // n_tries cannot decrease n_tries = answer.n_tries; - if (const auto error = answer.error.get()) { + ErrorState *error = nullptr; + if ((error = answer.error.get())) { flags.dont_retry = true; // or HappyConnOpener would not have given up syncHierNote(answer.conn, request->url.host()); - fail(error); + Must(!Comm::IsConnOpen(answer.conn)); answer.error.clear(); // preserve error for errorSendComplete() + } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) { + syncHierNote(answer.conn, request->url.host()); + closePendingConnection(answer.conn, "conn was closed while waiting for noteConnection"); + error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al); + } + + if (error) { + fail(error); retryOrBail(); // will notice flags.dont_retry and bail return; } - syncWithServerConn(answer.conn, request->url.host(), answer.reused); - - if (answer.reused) + if (answer.reused) { + syncWithServerConn(answer.conn, request->url.host(), answer.reused); return dispatch(); + } // Check if we need to TLS before use - if (const CachePeer *peer = serverConnection()->getPeer()) { + if (const auto *peer = answer.conn->getPeer()) { // Assume that it is only possible for the client-first from the // bumping modes to try connect to a remote server. The bumped // requests with other modes are using pinned connections or fails. @@ -793,20 +832,27 @@ if (originWantsEncryptedTraffic && // the "encrypted traffic" part !peer->options.originserver && // the "through a proxy" part !peer->secure.encryptTransport) // the "exclude HTTPS proxies" part - return establishTunnelThruProxy(); + return advanceDestination("establish tunnel through proxy", answer.conn, [this,&answer] { + establishTunnelThruProxy(answer.conn); + }); } - secureConnectionToPeerIfNeeded(); + secureConnectionToPeerIfNeeded(answer.conn); } void -FwdState::establishTunnelThruProxy() +FwdState::establishTunnelThruProxy(const Comm::ConnectionPointer &conn) { AsyncCall::Pointer callback = asyncCall(17,4, "FwdState::tunnelEstablishmentDone", Http::Tunneler::CbDialer(&FwdState::tunnelEstablishmentDone, this)); HttpRequest::Pointer requestPointer = request; - const auto tunneler = new Http::Tunneler(serverConnection(), requestPointer, callback, connectingTimeout(serverConnection()), al); + const auto tunneler = new Http::Tunneler(conn, requestPointer, callback, connectingTimeout(serverConnection()), al); + + // TODO: Replace this hack with proper Comm::Connection-Pool association + // that is not tied to fwdPconnPool and can handle disappearing pools. + tunneler->noteFwdPconnUse = true; + #if USE_DELAY_POOLS Must(serverConnection()->getPeer()); if (!serverConnection()->getPeer()->options.no_delay) @@ -820,11 +866,18 @@ void FwdState::tunnelEstablishmentDone(Http::TunnelerAnswer &answer) { - if (answer.positive()) { - if (answer.leftovers.isEmpty()) { - secureConnectionToPeerIfNeeded(); - return; - } + ErrorState *error = nullptr; + if (!answer.positive()) { + Must(!Comm::IsConnOpen(answer.conn)); + error = answer.squidError.get(); + Must(error); + answer.squidError.clear(); // preserve error for fail() + } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) { + // The socket could get closed while our callback was queued. + // We close Connection here to sync Connection::fd. + closePendingConnection(answer.conn, "conn was closed while waiting for tunnelEstablishmentDone"); + error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al); + } else if (!answer.leftovers.isEmpty()) { // This should not happen because TLS servers do not speak first. If we // have to handle this, then pass answer.leftovers via a PeerConnector // to ServerBio. See ClientBio::setReadBufData(). @@ -832,33 +885,26 @@ const auto level = (occurrences++ < 100) ? DBG_IMPORTANT : 2; debugs(17, level, "ERROR: Early data after CONNECT response. " << "Found " << answer.leftovers.length() << " bytes. " << - "Closing " << serverConnection()); - fail(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request, al)); - closeServerConnection("found early data after CONNECT response"); + "Closing " << answer.conn); + error = new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request, al); + closePendingConnection(answer.conn, "server spoke before tunnelEstablishmentDone"); + } + if (error) { + fail(error); retryOrBail(); return; } - // TODO: Reuse to-peer connections after a CONNECT error response. - - if (const auto peer = serverConnection()->getPeer()) - peerConnectFailed(peer); - - const auto error = answer.squidError.get(); - Must(error); - answer.squidError.clear(); // preserve error for fail() - fail(error); - closeServerConnection("Squid-generated CONNECT error"); - retryOrBail(); + secureConnectionToPeerIfNeeded(answer.conn); } /// handles an established TCP connection to peer (including origin servers) void -FwdState::secureConnectionToPeerIfNeeded() +FwdState::secureConnectionToPeerIfNeeded(const Comm::ConnectionPointer &conn) { assert(!request->flags.pinned); - const CachePeer *p = serverConnection()->getPeer(); + const auto p = conn->getPeer(); const bool peerWantsTls = p && p->secure.encryptTransport; // userWillTlsToPeerForUs assumes CONNECT == HTTPS const bool userWillTlsToPeerForUs = p && p->options.originserver && @@ -872,56 +918,70 @@ const bool needTlsToOrigin = !p && request->url.getScheme() == AnyP::PROTO_HTTPS && !clientFirstBump; if (needTlsToPeer || needTlsToOrigin || needsBump) { - HttpRequest::Pointer requestPointer = request; - AsyncCall::Pointer callback = asyncCall(17,4, - "FwdState::ConnectedToPeer", - FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this)); - const auto sslNegotiationTimeout = connectingTimeout(serverConnection()); - Security::PeerConnector *connector = nullptr; -#if USE_OPENSSL - if (request->flags.sslPeek) - connector = new Ssl::PeekingPeerConnector(requestPointer, serverConnection(), clientConn, callback, al, sslNegotiationTimeout); - else -#endif - connector = new Security::BlindPeerConnector(requestPointer, serverConnection(), callback, al, sslNegotiationTimeout); - AsyncJob::Start(connector); // will call our callback - return; + return advanceDestination("secure connection to peer", conn, [this,&conn] { + secureConnectionToPeer(conn); + }); } // if not encrypting just run the post-connect actions - successfullyConnectedToPeer(); + successfullyConnectedToPeer(conn); +} + +/// encrypts an established TCP connection to peer (including origin servers) +void +FwdState::secureConnectionToPeer(const Comm::ConnectionPointer &conn) +{ + HttpRequest::Pointer requestPointer = request; + AsyncCall::Pointer callback = asyncCall(17,4, + "FwdState::ConnectedToPeer", + FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this)); + const auto sslNegotiationTimeout = connectingTimeout(conn); + Security::PeerConnector *connector = nullptr; +#if USE_OPENSSL + if (request->flags.sslPeek) + connector = new Ssl::PeekingPeerConnector(requestPointer, conn, clientConn, callback, al, sslNegotiationTimeout); + else +#endif + connector = new Security::BlindPeerConnector(requestPointer, conn, callback, al, sslNegotiationTimeout); + connector->noteFwdPconnUse = true; + AsyncJob::Start(connector); // will call our callback } /// called when all negotiations with the TLS-speaking peer have been completed void FwdState::connectedToPeer(Security::EncryptorAnswer &answer) { - if (ErrorState *error = answer.error.get()) { - fail(error); + ErrorState *error = nullptr; + if ((error = answer.error.get())) { + Must(!Comm::IsConnOpen(answer.conn)); answer.error.clear(); // preserve error for errorSendComplete() - if (CachePeer *p = serverConnection()->getPeer()) - peerConnectFailed(p); - serverConnection()->close(); - return; - } - - if (answer.tunneled) { + } else if (answer.tunneled) { // TODO: When ConnStateData establishes tunnels, its state changes // [in ways that may affect logging?]. Consider informing // ConnStateData about our tunnel or otherwise unifying tunnel // establishment [side effects]. - unregister(serverConn); // async call owns it now complete(); // destroys us return; + } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) { + closePendingConnection(answer.conn, "conn was closed while waiting for connectedToPeer"); + error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al); } - successfullyConnectedToPeer(); + if (error) { + fail(error); + retryOrBail(); + return; + } + + successfullyConnectedToPeer(answer.conn); } /// called when all negotiations with the peer have been completed void -FwdState::successfullyConnectedToPeer() +FwdState::successfullyConnectedToPeer(const Comm::ConnectionPointer &conn) { + syncWithServerConn(conn, request->url.host(), false); + // should reach ConnStateData before the dispatched Client job starts CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData, ConnStateData::notePeerConnection, serverConnection()); @@ -1146,7 +1206,7 @@ // Set the dont_retry flag because this is not a transient (network) error. flags.dont_retry = true; if (Comm::IsConnOpen(serverConn)) { - serverConn->close(); + serverConn->close(); // trigger cleanup } break; } @@ -1353,7 +1413,7 @@ } void -getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn) +getOutgoingAddress(HttpRequest * request, const Comm::ConnectionPointer &conn) { // skip if an outgoing address is already set. if (!conn->local.isAnyAddr()) return; diff -u -r -N squid-5.0.2/src/FwdState.h squid-5.0.3/src/FwdState.h --- squid-5.0.2/src/FwdState.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/FwdState.h 2020-06-08 03:33:42.000000000 +1200 @@ -102,6 +102,9 @@ void dontRetry(bool val) { flags.dont_retry = val; } + /// get rid of a to-server connection that failed to become serverConn + void closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason); + /** return a ConnectionPointer to the current server connection (may or may not be open) */ Comm::ConnectionPointer const & serverConnection() const { return serverConn; }; @@ -131,14 +134,18 @@ /// (in order to retry or reforward a failed request) bool pinnedCanRetry() const; + template + void advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep); + ErrorState *makeConnectingError(const err_type type) const; void connectedToPeer(Security::EncryptorAnswer &answer); static void RegisterWithCacheManager(void); - void establishTunnelThruProxy(); + void establishTunnelThruProxy(const Comm::ConnectionPointer &); void tunnelEstablishmentDone(Http::TunnelerAnswer &answer); - void secureConnectionToPeerIfNeeded(); - void successfullyConnectedToPeer(); + void secureConnectionToPeerIfNeeded(const Comm::ConnectionPointer &); + void secureConnectionToPeer(const Comm::ConnectionPointer &); + void successfullyConnectedToPeer(const Comm::ConnectionPointer &); /// stops monitoring server connection for closure and updates pconn stats void closeServerConnection(const char *reason); @@ -197,7 +204,7 @@ PconnRace pconnRace; ///< current pconn race state }; -void getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn); +void getOutgoingAddress(HttpRequest * request, const Comm::ConnectionPointer &conn); /// a collection of previously used persistent Squid-to-peer HTTP(S) connections extern PconnPool *fwdPconnPool; diff -u -r -N squid-5.0.2/src/HappyConnOpener.cc squid-5.0.3/src/HappyConnOpener.cc --- squid-5.0.2/src/HappyConnOpener.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/HappyConnOpener.cc 2020-06-08 03:33:42.000000000 +1200 @@ -397,15 +397,11 @@ // TODO: Find an automated, faster way to kill no-longer-needed jobs. if (prime) { - if (prime.connector) - prime.connector->cancel("HappyConnOpener object destructed"); - prime.clear(); + cancelAttempt(prime, "job finished during a prime attempt"); } if (spare) { - if (spare.connector) - spare.connector->cancel("HappyConnOpener object destructed"); - spare.clear(); + cancelAttempt(spare, "job finished during a spare attempt"); if (gotSpareAllowance) { TheSpareAllowanceGiver.jobDroppedAllowance(); gotSpareAllowance = false; @@ -484,6 +480,15 @@ callback_ = nullptr; } +/// cancels the in-progress attempt, making its path a future candidate +void +HappyConnOpener::cancelAttempt(Attempt &attempt, const char *reason) +{ + Must(attempt); + destinations->retryPath(attempt.path); // before attempt.cancel() clears path + attempt.cancel(reason); +} + /// inform the initiator about our failure to connect (if needed) void HappyConnOpener::sendFailure() @@ -563,6 +568,7 @@ attempt.path = dest; attempt.connector = callConnect; + attempt.opener = cs; AsyncJob::Start(cs); } @@ -578,9 +584,9 @@ Must(itWasPrime != itWasSpare); if (itWasPrime) { - prime.clear(); + prime.finish(); } else { - spare.clear(); + spare.finish(); if (gotSpareAllowance) { TheSpareAllowanceGiver.jobUsedAllowance(); gotSpareAllowance = false; @@ -869,3 +875,13 @@ return false; } +void +HappyConnOpener::Attempt::cancel(const char *reason) +{ + if (connector) { + connector->cancel(reason); + CallJobHere(17, 3, opener, Comm::ConnOpener, noteAbort); + } + clear(); +} + diff -u -r -N squid-5.0.2/src/HappyConnOpener.h squid-5.0.3/src/HappyConnOpener.h --- squid-5.0.2/src/HappyConnOpener.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/HappyConnOpener.h 2020-06-08 03:33:42.000000000 +1200 @@ -157,10 +157,20 @@ class Attempt { public: explicit operator bool() const { return static_cast(path); } - void clear() { path = nullptr; connector = nullptr; } + + /// reacts to a natural attempt completion (successful or otherwise) + void finish() { clear(); } + + /// aborts an in-progress attempt + void cancel(const char *reason); Comm::ConnectionPointer path; ///< the destination we are connecting to - AsyncCall::Pointer connector; ///< our Comm::ConnOpener callback + AsyncCall::Pointer connector; ///< our opener callback + Comm::ConnOpener::Pointer opener; ///< connects to path and calls us + + private: + /// cleans up after the attempt ends (successfully or otherwise) + void clear() { path = nullptr; connector = nullptr; opener = nullptr; } }; /* AsyncJob API */ @@ -194,6 +204,7 @@ Answer *futureAnswer(const Comm::ConnectionPointer &); void sendSuccess(const Comm::ConnectionPointer &conn, bool reused, const char *connKind); void sendFailure(); + void cancelAttempt(Attempt &, const char *reason); const time_t fwdStart; ///< requestor start time diff -u -r -N squid-5.0.2/src/htcp.cc squid-5.0.3/src/htcp.cc --- squid-5.0.2/src/htcp.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/htcp.cc 2020-06-08 03:33:42.000000000 +1200 @@ -145,7 +145,7 @@ public: const char *method = nullptr; - char *uri = nullptr; + const char *uri = nullptr; char *version = nullptr; char *req_hdrs = nullptr; size_t reqHdrsSz = 0; ///< size of the req_hdrs content @@ -816,6 +816,7 @@ if (spec) { stuff.S.method = spec->method; + stuff.S.request = spec->request; stuff.S.uri = spec->uri; stuff.S.version = spec->version; stuff.S.req_hdrs = spec->req_hdrs; @@ -850,15 +851,15 @@ hdr.clean(); #if USE_ICMP - if (char *host = urlHostname(spec->uri)) { + if (const char *host = spec->request->url.host()) { int rtt = 0; int hops = 0; int samp = 0; netdbHostData(host, &samp, &rtt, &hops); if (rtt || hops) { - char cto_buf[128]; - snprintf(cto_buf, 128, "%s %d %f %d", + char cto_buf[SQUIDHOSTNAMELEN+128]; + snprintf(cto_buf, sizeof(cto_buf), "%s %d %f %d", host, samp, 0.001 * rtt, hops); hdr.putExt("Cache-to-Origin", cto_buf); } @@ -1563,7 +1564,7 @@ * Send an HTCP CLR message for a specified item to a given CachePeer. */ void -htcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &, CachePeer * p, htcp_clr_reason reason) +htcpClear(StoreEntry * e, HttpRequest * req, const HttpRequestMethod &, CachePeer * p, htcp_clr_reason reason) { static char pkt[8192]; ssize_t pktlen; @@ -1585,14 +1586,9 @@ SBuf sb = req->method.image(); stuff.S.method = sb.c_str(); - if (e == NULL || e->mem_obj == NULL) { - if (uri == NULL) { - return; - } - stuff.S.uri = xstrdup(uri); - } else { - stuff.S.uri = (char *) e->url(); - } + stuff.S.request = req; + SBuf uri = req->effectiveRequestUri(); + stuff.S.uri = uri.c_str(); stuff.S.version = vbuf; if (reason != HTCP_CLR_INVALIDATION) { HttpStateData::httpBuildRequestHeader(req, e, NULL, &hdr, flags); @@ -1607,9 +1603,6 @@ if (reason != HTCP_CLR_INVALIDATION) { mb.clean(); } - if (e == NULL) { - xfree(stuff.S.uri); - } if (!pktlen) { debugs(31, 3, "htcpClear: htcpBuildPacket() failed"); return; diff -u -r -N squid-5.0.2/src/htcp.h squid-5.0.3/src/htcp.h --- squid-5.0.2/src/htcp.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/htcp.h 2020-06-08 03:33:42.000000000 +1200 @@ -61,7 +61,7 @@ int htcpQuery(StoreEntry * e, HttpRequest * req, CachePeer * p); /// \ingroup ServerProtocolHTCP -void htcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &method, CachePeer * p, htcp_clr_reason reason); +void htcpClear(StoreEntry * e, HttpRequest * req, const HttpRequestMethod &method, CachePeer * p, htcp_clr_reason reason); /// \ingroup ServerProtocolHTCP void htcpSocketShutdown(void); diff -u -r -N squid-5.0.2/src/http/ContentLengthInterpreter.cc squid-5.0.3/src/http/ContentLengthInterpreter.cc --- squid-5.0.2/src/http/ContentLengthInterpreter.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/http/ContentLengthInterpreter.cc 2020-06-08 03:33:42.000000000 +1200 @@ -29,6 +29,24 @@ { } +/// checks whether all characters before the Content-Length number are allowed +/// \returns the start of the digit sequence (or nil on errors) +const char * +Http::ContentLengthInterpreter::findDigits(const char *prefix, const char * const valueEnd) const +{ + // skip leading OWS in RFC 7230's `OWS field-value OWS` + const CharacterSet &whitespace = Http::One::Parser::WhitespaceCharacters(); + while (prefix < valueEnd) { + const auto ch = *prefix; + if (CharacterSet::DIGIT[ch]) + return prefix; // common case: a pre-trimmed field value + if (!whitespace[ch]) + return nullptr; // (trimmed) length does not start with a digit + ++prefix; + } + return nullptr; // empty or whitespace-only value +} + /// checks whether all characters after the Content-Length are allowed bool Http::ContentLengthInterpreter::goodSuffix(const char *suffix, const char * const end) const @@ -53,10 +71,19 @@ { Must(!sawBad); + const auto valueEnd = rawValue + valueSize; + + const auto digits = findDigits(rawValue, valueEnd); + if (!digits) { + debugs(55, debugLevel, "WARNING: Leading garbage or empty value in" << Raw("Content-Length", rawValue, valueSize)); + sawBad = true; + return false; + } + int64_t latestValue = -1; char *suffix = nullptr; - // TODO: Handle malformed values with leading signs (e.g., "-0" or "+1"). - if (!httpHeaderParseOffset(rawValue, &latestValue, &suffix)) { + + if (!httpHeaderParseOffset(digits, &latestValue, &suffix)) { debugs(55, DBG_IMPORTANT, "WARNING: Malformed" << Raw("Content-Length", rawValue, valueSize)); sawBad = true; return false; @@ -69,7 +96,7 @@ } // check for garbage after the number - if (!goodSuffix(suffix, rawValue + valueSize)) { + if (!goodSuffix(suffix, valueEnd)) { debugs(55, debugLevel, "WARNING: Trailing garbage in" << Raw("Content-Length", rawValue, valueSize)); sawBad = true; return false; diff -u -r -N squid-5.0.2/src/http/ContentLengthInterpreter.h squid-5.0.3/src/http/ContentLengthInterpreter.h --- squid-5.0.2/src/http/ContentLengthInterpreter.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/http/ContentLengthInterpreter.h 2020-06-08 03:33:42.000000000 +1200 @@ -67,6 +67,7 @@ bool sawGood; protected: + const char *findDigits(const char *prefix, const char *valueEnd) const; bool goodSuffix(const char *suffix, const char * const end) const; bool checkValue(const char *start, const int size); bool checkList(const String &list); diff -u -r -N squid-5.0.2/src/http/url_rewriters/LFS/url_lfs_rewrite.8 squid-5.0.3/src/http/url_rewriters/LFS/url_lfs_rewrite.8 --- squid-5.0.2/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2020-04-20 00:20:17.000000000 +1200 +++ squid-5.0.3/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2020-06-09 19:09:22.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "URL_LFS_REWRITE 8" -.TH URL_LFS_REWRITE 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH URL_LFS_REWRITE 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/http.cc squid-5.0.3/src/http.cc --- squid-5.0.2/src/http.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/http.cc 2020-06-08 03:33:42.000000000 +1200 @@ -253,7 +253,7 @@ if (pe != NULL) { assert(e != pe); #if USE_HTCP - neighborsHtcpClear(e, nullptr, e->mem_obj->request.getRaw(), e->mem_obj->method, HTCP_CLR_INVALIDATION); + neighborsHtcpClear(e, e->mem_obj->request.getRaw(), e->mem_obj->method, HTCP_CLR_INVALIDATION); #endif pe->release(true); } @@ -270,7 +270,7 @@ if (pe != NULL) { assert(e != pe); #if USE_HTCP - neighborsHtcpClear(e, nullptr, e->mem_obj->request.getRaw(), HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_INVALIDATION); + neighborsHtcpClear(e, e->mem_obj->request.getRaw(), HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_INVALIDATION); #endif pe->release(true); } diff -u -r -N squid-5.0.2/src/ip/Intercept.cc squid-5.0.3/src/ip/Intercept.cc --- squid-5.0.2/src/ip/Intercept.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ip/Intercept.cc 2020-06-08 03:33:42.000000000 +1200 @@ -204,7 +204,7 @@ memset(&natLookup, 0, sizeof(natLookup)); // for NAT lookup set local and remote IP:port's if (newConn->remote.isIPv6()) { -#if HAVE_NATLOOKUP_NL_INIPADDR_IN6 +#if HAVE_STRUCT_NATLOOKUP_NL_INIPADDR_IN6 natLookup.nl_v = 6; newConn->local.getInAddr(natLookup.nl_inipaddr.in6); newConn->remote.getInAddr(natLookup.nl_outipaddr.in6); @@ -292,7 +292,7 @@ debugs(89, 9, HERE << "address: " << newConn); return false; } else { -#if HAVE_NATLOOKUP_NL_REALIPADDR_IN6 +#if HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6 if (newConn->remote.isIPv6()) newConn->local = natLookup.nl_realipaddr.in6; else diff -u -r -N squid-5.0.2/src/ipc/Makefile.am squid-5.0.3/src/ipc/Makefile.am --- squid-5.0.2/src/ipc/Makefile.am 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ipc/Makefile.am 2020-06-08 03:33:42.000000000 +1200 @@ -71,3 +71,6 @@ install-data-local: $(mkinstalldirs) $(DESTDIR)$(localstatedir)/run/squid; + +libipc_la_LIBADD = $(ATOMICLIB) + diff -u -r -N squid-5.0.2/src/ipc/Makefile.in squid-5.0.3/src/ipc/Makefile.in --- squid-5.0.2/src/ipc/Makefile.in 2020-04-20 00:08:53.000000000 +1200 +++ squid-5.0.3/src/ipc/Makefile.in 2020-06-09 18:58:23.000000000 +1200 @@ -163,7 +163,8 @@ CONFIG_CLEAN_FILES = CONFIG_CLEAN_VPATH_FILES = LTLIBRARIES = $(noinst_LTLIBRARIES) -libipc_la_LIBADD = +am__DEPENDENCIES_1 = +libipc_la_DEPENDENCIES = $(am__DEPENDENCIES_1) am__dirstamp = $(am__leading_dot)dirstamp am_libipc_la_OBJECTS = FdNotes.lo Kid.lo Kids.lo MemMap.lo Queue.lo \ ReadWriteLock.lo StartListening.lo StoreMap.lo StrandCoord.lo \ @@ -801,6 +802,7 @@ mem/Segment.h \ mem/forward.h +libipc_la_LIBADD = $(ATOMICLIB) all: all-am .SUFFIXES: diff -u -r -N squid-5.0.2/src/ipc/mem/PagePool.cc squid-5.0.3/src/ipc/mem/PagePool.cc --- squid-5.0.2/src/ipc/mem/PagePool.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ipc/mem/PagePool.cc 2020-06-08 03:33:42.000000000 +1200 @@ -18,7 +18,12 @@ Ipc::Mem::PagePool::Owner * Ipc::Mem::PagePool::Init(const char *const shmId, const Ipc::Mem::PoolId stackId, const unsigned int capacity, const size_t pageSize) { - return shm_new(PageStack)(shmId, stackId, capacity, pageSize); + PageStack::Config config; + config.poolId = stackId; + config.pageSize = pageSize; // the pages are stored in Ipc::Mem::Pages + config.capacity = capacity; + config.createFull = true; // all pages are initially available + return shm_new(PageStack)(shmId, config); } Ipc::Mem::PagePool::PagePool(const char *const id): diff -u -r -N squid-5.0.2/src/ipc/mem/PageStack.cc squid-5.0.3/src/ipc/mem/PageStack.cc --- squid-5.0.2/src/ipc/mem/PageStack.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ipc/mem/PageStack.cc 2020-06-08 03:33:42.000000000 +1200 @@ -10,54 +10,430 @@ #include "squid.h" -#include "base/TextException.h" #include "Debug.h" #include "ipc/mem/Page.h" #include "ipc/mem/PageStack.h" -/* Ipc::Mem::PageStackStorageSlot */ +#include +#include -static_assert(sizeof(Ipc::Mem::PageStackStorageSlot::Pointer) == - sizeof(decltype(Ipc::Mem::PageId::number)), "page indexing types are consistent"); +/* + +Ipc::Mem::IdSet and related code maintains a perfect full binary tree structure: + + (l,r) + /\ + (ll,lr) (rl,rr) + /\ /\ + L1 L2 L3 L4 + +where + + * (l,r) is an always-present root node; + * inner nodes, including the root one, count the total number of available + IDs in the leaf nodes of the left and right subtrees (e.g., r = rl + rr); + * leaf nodes are bitsets of available IDs (e.g., rl = number of 1s in L3); + all leaf nodes are always present. + +The above sample tree would be stored as seven 64-bit atomic integers: + (l,r), (ll,lr), (rl,rr), L1, L2, L3, L4 + +*/ + +namespace Ipc +{ + +namespace Mem +{ + +/// the maximum number of pages that a leaf node can store +static const IdSet::size_type BitsPerLeaf = 64; + +class IdSetPosition +{ +public: + using size_type = IdSet::size_type; + + IdSetPosition() = default; ///< root node position + IdSetPosition(size_type aLevel, size_type anOffset); + + /// whether we are at the top of the tree + bool atRoot() const { return !level && !offset; } + + /// which direction is this position from our parent node + IdSetNavigationDirection ascendDirection() const; + + /// the number of levels above us (e.g., zero for the root node) + IdSet::size_type level = 0; + /// the number of nodes (at our level) to the left of us + IdSet::size_type offset = 0; +}; + +/// a helper class to perform inner node manipulation for IdSet +class IdSetInnerNode +{ +public: + using size_type = IdSet::size_type; + typedef uint64_t Packed; ///< (atomically) stored serialized value + + /// de-serializes a given value + static IdSetInnerNode Unpack(Packed packed); + + IdSetInnerNode() = default; + IdSetInnerNode(size_type left, size_type right); + + /// returns a serializes value suitable for shared memory storage + Packed pack() const { return (static_cast(left) << 32) | right; } + + size_type left = 0; ///< the number of available IDs in the left subtree + size_type right = 0; ///< the number of available IDs in the right subtree +}; + +} // namespace Mem + +} // namespace Ipc + +/* Ipc::Mem::IdSetPosition */ + +Ipc::Mem::IdSetPosition::IdSetPosition(size_type aLevel, size_type anOffset): + level(aLevel), + offset(anOffset) +{ +} + +Ipc::Mem::IdSetNavigationDirection +Ipc::Mem::IdSetPosition::ascendDirection() const +{ + return (offset % 2 == 0) ? dirLeft : dirRight; +} + +/* Ipc::Mem::IdSetMeasurements */ + +Ipc::Mem::IdSetMeasurements::IdSetMeasurements(const size_type aCapacity) +{ + capacity = aCapacity; + + // For simplicity, we want a perfect full binary tree with root and leaves. + // We could compute all this with log2() calls, but rounding and honoring + // root+leaves minimums make that approach more complex than this fast loop. + requestedLeafNodeCount = (capacity + (BitsPerLeaf-1))/BitsPerLeaf; + treeHeight = 1+1; // the root level plus the leaf nodes level + leafNodeCount = 2; // the root node can have only two leaf nodes + while (leafNodeCount < requestedLeafNodeCount) { + leafNodeCount *= 2; + ++treeHeight; + } + innerLevelCount = treeHeight - 1; + + debugs(54, 5, "rounded capacity up from " << capacity << " to " << (leafNodeCount*BitsPerLeaf)); + + // we do (1 << level) when computing 32-bit IdSetInnerNode::left + assert(treeHeight < 32); +} + +/* Ipc::Mem::IdSetInnerNode */ + +Ipc::Mem::IdSetInnerNode::IdSetInnerNode(size_type aLeft, size_type aRight): + left(aLeft), + right(aRight) +{ +} + +Ipc::Mem::IdSetInnerNode +Ipc::Mem::IdSetInnerNode::Unpack(Packed packed) +{ + // truncation during the cast is intentional here + return IdSetInnerNode(packed >> 32, static_cast(packed)); +} + +/* Ipc::Mem::IdSet */ + +Ipc::Mem::IdSet::IdSet(const size_type capacity): + measurements(capacity), + nodes_(capacity) +{ + // For valueAddress() to be able to return a raw uint64_t pointer, the + // atomic wrappers in nodes_ must be zero-size. Check the best we can. Once. + static_assert(sizeof(StoredNode) == sizeof(Node), "atomic locks use no storage"); + assert(StoredNode().is_lock_free()); +} void -Ipc::Mem::PageStackStorageSlot::take() +Ipc::Mem::IdSet::makeFullBeforeSharing() { - const auto nxt = nextOrMarker.exchange(TakenPage); - assert(nxt != TakenPage); + // initially, all IDs are marked as available + fillAllNodes(); + + // ... but IDs beyond the requested capacity should not be available + if (measurements.capacity != measurements.leafNodeCount*BitsPerLeaf) + truncateExtras(); } +/// populates the entire allocated tree with available IDs +/// may exceed the requested capacity; \see truncateExtras() void -Ipc::Mem::PageStackStorageSlot::put(const PointerOrMarker expected, const Pointer nxt) +Ipc::Mem::IdSet::fillAllNodes() { - assert(nxt != TakenPage); - const auto old = nextOrMarker.exchange(nxt); - assert(old == expected); + // leaf nodes + auto pos = Position(measurements.treeHeight-1, 0); + const auto allOnes = ~uint64_t(0); + std::fill_n(valueAddress(pos), measurements.leafNodeCount, allOnes); + + // inner nodes, starting from the bottom of the tree + auto nodesAtLevel = measurements.leafNodeCount/2; + auto pagesBelow = BitsPerLeaf; + do { + pos = ascend(pos); + const auto value = IdSetInnerNode(pagesBelow, pagesBelow).pack(); + std::fill_n(valueAddress(pos), nodesAtLevel, value); + nodesAtLevel /= 2; + pagesBelow *= 2; + } while (!pos.atRoot()); } -/* Ipc::Mem::PageStack */ +/// effectively removes IDs that exceed the requested capacity after makeFull() +void +Ipc::Mem::IdSet::truncateExtras() +{ + // leaf nodes + // start with the left-most leaf that should have some 0s; it may even have + // no 1s at all (i.e. be completely unused) + auto pos = Position(measurements.treeHeight-1, measurements.capacity/BitsPerLeaf); + leafTruncate(pos, measurements.capacity % BitsPerLeaf); + const auto rightLeaves = measurements.leafNodeCount - measurements.requestedLeafNodeCount; + // this zeroing of the leaf nodes to the right from pos is only necessary to + // trigger asserts if the code dealing with the inner node counters is buggy + if (rightLeaves > 1) + std::fill_n(valueAddress(pos) + 1, rightLeaves-1, 0); + + // inner nodes, starting from the bottom of the tree; optimization: only + // adjusting nodes on the way up from the first leaf-with-0s position + auto toSubtract = BitsPerLeaf - (measurements.capacity % BitsPerLeaf); + do { + const auto direction = pos.ascendDirection(); + pos = ascend(pos); + toSubtract = innerTruncate(pos, direction, toSubtract); + } while (!pos.atRoot()); +} -Ipc::Mem::PageStack::PageStack(const PoolId aPoolId, const PageCount aCapacity, const size_t aPageSize): - thePoolId(aPoolId), capacity_(aCapacity), thePageSize(aPageSize), - size_(0), - head_(Slot::NilPtr), - slots_(aCapacity) +/// fill the leaf node at a given position with 0s, leaving only idsToKeep IDs +void +Ipc::Mem::IdSet::leafTruncate(const Position pos, const size_type idsToKeep) +{ + Node &node = *valueAddress(pos); // no auto to simplify the asserts() below + assert(node == std::numeric_limits::max()); // all 1s + static_assert(std::is_unsigned::value, "right shift prepends 0s"); + node >>= BitsPerLeaf - idsToKeep; + // node can be anything here, including all 0s and all 1s +} + +/// accounts for toSubtract IDs removal from a subtree in the given direction of +/// the given position +/// \returns the number of IDs to subtract from the parent node +Ipc::Mem::IdSet::size_type +Ipc::Mem::IdSet::innerTruncate(const Position pos, const NavigationDirection dir, const size_type toSubtract) +{ + auto *valuePtr = valueAddress(pos); + auto value = IdSetInnerNode::Unpack(*valuePtr); + size_type toSubtractNext = 0; + if (dir == dirLeft) { + toSubtractNext = toSubtract + value.right; + assert(value.left >= toSubtract); + value.left -= toSubtract; + value.right = 0; + } else { + assert(dir == dirRight); + toSubtractNext = toSubtract; + assert(value.right >= toSubtract); + // value.left is unchanged; we have only adjusted the right branch + value.right -= toSubtract; + } + *valuePtr = value.pack(); + return toSubtractNext; +} + +/// accounts for an ID added to subtree in the given dir from the given position +void +Ipc::Mem::IdSet::innerPush(const Position pos, const NavigationDirection dir) +{ + // either left or right component will be true/1; the other will be false/0 + const auto increment = IdSetInnerNode(dir == dirLeft, dir == dirRight).pack(); + const auto previousValue = nodeAt(pos).fetch_add(increment); + // no overflows + assert(previousValue <= std::numeric_limits::max() - increment); +} + +/// accounts for future ID removal from a subtree of the given position +/// \returns the direction of the subtree chosen to relinquish the ID +Ipc::Mem::IdSet::NavigationDirection +Ipc::Mem::IdSet::innerPop(const Position pos) +{ + NavigationDirection direction = dirNone; + + auto &node = nodeAt(pos); + auto oldValue = node.load(); + IdSetInnerNode newValue; + do { + newValue = IdSetInnerNode::Unpack(oldValue); + if (newValue.left) { + --newValue.left; + direction = dirLeft; + } else if (newValue.right) { + --newValue.right; + direction = dirRight; + } else { + return dirEnd; + } + } while (!node.compare_exchange_weak(oldValue, newValue.pack())); + + assert(direction == dirLeft || direction == dirRight); + return direction; +} + +/// adds the given ID to the leaf node at the given position +void +Ipc::Mem::IdSet::leafPush(const Position pos, const size_type id) +{ + const auto mask = Node(1) << (id % BitsPerLeaf); + const auto oldValue = nodeAt(pos).fetch_or(mask); + // this was a new entry + assert((oldValue & mask) == 0); +} + +// TODO: After switching to C++20, use countr_zero() which may compile to a +// single TZCNT assembly instruction on modern CPUs. +/// a temporary C++20 countr_zero() replacement +static inline +int trailingZeros(uint64_t x) +{ + if (!x) + return 64; + int count = 0; + for (uint64_t mask = 1; !(x & mask); mask <<= 1) + ++count; + return count; +} + +/// extracts and returns an ID from the leaf node at the given position +Ipc::Mem::IdSet::size_type +Ipc::Mem::IdSet::leafPop(const Position pos) +{ + auto &node = nodeAt(pos); + auto oldValue = node.load(); + Node newValue; + do { + assert(oldValue > 0); + const auto mask = oldValue - 1; // flips the rightmost 1 and trailing 0s + newValue = oldValue & mask; // clears the rightmost 1 + } while (!node.compare_exchange_weak(oldValue, newValue)); + + return pos.offset*BitsPerLeaf + trailingZeros(oldValue); +} + +/// \returns the position of a parent node of the node at the given position +Ipc::Mem::IdSet::Position +Ipc::Mem::IdSet::ascend(Position pos) +{ + assert(pos.level > 0); + --pos.level; + pos.offset /= 2; + return pos; +} + +/// \returns the position of a child node in the given direction of the parent +/// node at the given position +Ipc::Mem::IdSet::Position +Ipc::Mem::IdSet::descend(Position pos, const NavigationDirection direction) { - assert(thePoolId); + assert(pos.level < measurements.treeHeight); + ++pos.level; - assert(capacity_ < Slot::TakenPage); - assert(capacity_ < Slot::NilPtr); + pos.offset *= 2; + if (direction == dirRight) + ++pos.offset; + else + assert(direction == dirLeft); - // initially, all pages are free - if (capacity_) { - const auto lastIndex = capacity_-1; - // FlexibleArray cannot construct its phantom elements so, technically, - // all slots (except the very first one) are uninitialized until now. - for (Slot::Pointer i = 0; i < lastIndex; ++i) - (void)new(&slots_[i])Slot(i+1); - (void)new(&slots_[lastIndex])Slot(Slot::NilPtr); - size_ = capacity_; - head_ = 0; + return pos; +} + +/// \returns the atomic node (either inner or leaf) at the given position +Ipc::Mem::IdSet::StoredNode & +Ipc::Mem::IdSet::nodeAt(const Position pos) +{ + assert(pos.level < measurements.treeHeight); + // n = 2^(h+1) - 1 with h = level-1 + const auto nodesAbove = (1U << pos.level) - 1; + + // the second clause is for the special case of a root node + assert(pos.offset < nodesAbove*2 || (pos.atRoot() && nodesAbove == 0)); + const auto nodesToTheLeft = pos.offset; + + const size_t nodesBefore = nodesAbove + nodesToTheLeft; + assert(nodesBefore < measurements.nodeCount()); + return nodes_[nodesBefore]; +} + +/// \returns the location of the raw (inner or leaf) node at the given position +Ipc::Mem::IdSet::Node * +Ipc::Mem::IdSet::valueAddress(const Position pos) +{ + // IdSet() constructor asserts that this frequent reinterpret_cast is safe + return &reinterpret_cast(nodeAt(pos)); +} + +bool +Ipc::Mem::IdSet::pop(size_type &id) +{ + Position rootPos; + const auto directionFromRoot = innerPop(rootPos); + if (directionFromRoot == dirEnd) + return false; // an empty tree + + auto pos = descend(rootPos, directionFromRoot); + for (size_t level = 1; level < measurements.innerLevelCount; ++level) { + const auto direction = innerPop(pos); + pos = descend(pos, direction); + } + + id = leafPop(pos); + return true; +} + +void +Ipc::Mem::IdSet::push(const size_type id) +{ + const auto offsetAtLeafLevel = id/BitsPerLeaf; + auto pos = Position(measurements.innerLevelCount, offsetAtLeafLevel); + leafPush(pos, id); + + do { + const auto direction = pos.ascendDirection(); + pos = ascend(pos); + innerPush(pos, direction); + } while (!pos.atRoot()); +} + +size_t +Ipc::Mem::IdSet::MemorySize(const size_type capacity) +{ + const IdSetMeasurements measurements(capacity); + // Adding sizeof(IdSet) double-counts the first node but it is better to + // overestimate (a little) than to underestimate our memory needs due to + // padding, new data members, etc. + return sizeof(IdSet) + measurements.nodeCount() * sizeof(StoredNode); +} + +/* Ipc::Mem::PageStack */ + +Ipc::Mem::PageStack::PageStack(const Config &config): + config_(config), + size_(0), + ids_(config_.capacity) +{ + if (config.createFull) { + ids_.makeFullBeforeSharing(); + size_ = config_.capacity; } } @@ -66,23 +442,21 @@ { assert(!page); - Slot::Pointer current = head_.load(); + if (!config_.capacity) + return false; - auto nextFree = Slot::NilPtr; - do { - if (current == Slot::NilPtr) - return false; - nextFree = slots_[current].next(); - } while (!head_.compare_exchange_weak(current, nextFree)); + IdSet::size_type pageIndex = 0; + if (!ids_.pop(pageIndex)) + return false; // must decrement after removing the page to avoid underflow const auto newSize = --size_; - assert(newSize < capacity_); + assert(newSize < config_.capacity); - slots_[current].take(); - page.number = current + 1; - page.pool = thePoolId; + page.number = pageIndex + 1; + page.pool = config_.poolId; debugs(54, 8, page << " size: " << newSize); + assert(pageIdIsValid(page)); return true; } @@ -93,19 +467,12 @@ assert(page); assert(pageIdIsValid(page)); - const auto pageIndex = page.number - 1; - auto &slot = slots_[pageIndex]; - // must increment before inserting the page to avoid underflow in pop() const auto newSize = ++size_; - assert(newSize <= capacity_); + assert(newSize <= config_.capacity); - auto current = head_.load(); - auto expected = Slot::TakenPage; - do { - slot.put(expected, current); - expected = current; - } while (!head_.compare_exchange_weak(current, pageIndex)); + const auto pageIndex = page.number - 1; + ids_.push(pageIndex); debugs(54, 8, page << " size: " << newSize); page = PageId(); @@ -114,34 +481,37 @@ bool Ipc::Mem::PageStack::pageIdIsValid(const PageId &page) const { - return page.pool == thePoolId && + return page.pool == config_.poolId && 0 < page.number && page.number <= capacity(); } size_t Ipc::Mem::PageStack::sharedMemorySize() const { - return SharedMemorySize(thePoolId, capacity_, thePageSize); + return SharedMemorySize(config_); } size_t -Ipc::Mem::PageStack::SharedMemorySize(const PoolId, const PageCount capacity, const size_t pageSize) +Ipc::Mem::PageStack::SharedMemorySize(const Config &cfg) { const auto levelsSize = PageId::maxPurpose * sizeof(Levels_t); - const size_t pagesDataSize = capacity * pageSize; - return StackSize(capacity) + LevelsPaddingSize(capacity) + levelsSize + pagesDataSize; + const size_t pagesDataSize = cfg.capacity * cfg.pageSize; + return StackSize(cfg.capacity) + pagesDataSize + levelsSize; } size_t Ipc::Mem::PageStack::StackSize(const PageCount capacity) { - return sizeof(PageStack) + capacity * sizeof(Slot); + // Adding sizeof(PageStack) double-counts the fixed portion of the ids_ data + // member but it is better to overestimate (a little) than to underestimate + // our memory needs due to padding, new data members, etc. + return sizeof(PageStack) + IdSet::MemorySize(capacity); } size_t Ipc::Mem::PageStack::stackSize() const { - return StackSize(capacity_); + return StackSize(config_.capacity); } size_t diff -u -r -N squid-5.0.2/src/ipc/mem/PageStack.h squid-5.0.3/src/ipc/mem/PageStack.h --- squid-5.0.2/src/ipc/mem/PageStack.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ipc/mem/PageStack.h 2020-06-08 03:33:42.000000000 +1200 @@ -23,39 +23,83 @@ class PageId; -/// reflects the dual nature of PageStack storage: -/// - for free pages, this is a pointer to the next free page -/// - for used pages, this is a "used page" marker -class PageStackStorageSlot +class IdSetPosition; +typedef enum { dirNone, dirLeft, dirRight, dirEnd } IdSetNavigationDirection; + +/// basic IdSet storage parameters, extracted here to keep them constant +class IdSetMeasurements +{ +public: + /// we need to fit two size_type counters into one 64-bit lockless atomic + typedef uint32_t size_type; + + explicit IdSetMeasurements(size_type capacity); + + /// the maximum number of pages our tree is allowed to store + size_type capacity = 0; + + /// the number of leaf nodes that satisfy capacity requirements + size_type requestedLeafNodeCount = 0; + + size_type treeHeight = 0; ///< total number of levels, including the leaf level + size_type leafNodeCount = 0; ///< the number of nodes at the leaf level + size_type innerLevelCount = 0; ///< all levels except the leaf level + + /// the total number of nodes at all levels + size_type nodeCount() const { return leafNodeCount ? leafNodeCount*2 -1 : 0; } +}; + +/// a shareable set of positive uint32_t IDs with O(1) insertion/removal ops +class IdSet { public: - // We are using uint32_t for Pointer because PageId::number is uint32_t. - // PageId::number should probably be uint64_t to accommodate caches with - // page numbers exceeding UINT32_MAX. - typedef uint32_t PointerOrMarker; - typedef PointerOrMarker Pointer; - typedef PointerOrMarker Marker; - - /// represents a nil next slot pointer - static const Pointer NilPtr = std::numeric_limits::max(); - /// marks a slot of a used (i.e. take()n) page - static const Marker TakenPage = std::numeric_limits::max() - 1; - static_assert(TakenPage != NilPtr, "magic PointerOrMarker values do not clash"); - - explicit PageStackStorageSlot(const Pointer nxt = NilPtr): nextOrMarker(nxt) {} - - /// returns a (possibly nil) pointer to the next free page - Pointer next() const { return nextOrMarker.load(); } - - /// marks our page as used - void take(); - - /// marks our page as free, to be used before the given `nxt` page; - /// also checks that the slot state matches the caller expectations - void put(const PointerOrMarker expected, const Pointer nxt); + using size_type = IdSetMeasurements::size_type; + using Position = IdSetPosition; + using NavigationDirection = IdSetNavigationDirection; + + /// memory size required to store a tree with the given capacity + static size_t MemorySize(size_type capacity); + + explicit IdSet(size_type capacity); + + /// populates the allocated tree with the requested capacity IDs + /// optimized to run without atomic protection + void makeFullBeforeSharing(); + + /// finds/extracts (into the given `id`) an ID value and returns true + /// \retval false no IDs are left + bool pop(size_type &id); + + /// makes `id` value available to future pop() callers + void push(size_type id); + + const IdSetMeasurements measurements; private: - std::atomic nextOrMarker; + typedef uint64_t Node; ///< either leaf or intermediate node + typedef std::atomic StoredNode; ///< a Node stored in shared memory + + /* optimization: these initialization methods bypass atomic protections */ + void fillAllNodes(); + void truncateExtras(); + Node *valueAddress(Position); + size_type innerTruncate(Position pos, NavigationDirection dir, size_type toSubtract); + void leafTruncate(Position pos, size_type idsToKeep); + + void innerPush(Position, NavigationDirection); + NavigationDirection innerPop(Position); + + void leafPush(Position, size_type id); + size_type leafPop(Position); + + Position ascend(Position); + Position descend(Position, NavigationDirection); + + StoredNode &nodeAt(Position); + + /// the entire binary tree flattened into an array + FlexibleArray nodes_; + // No more data members should follow! See FlexibleArray<> for details. }; /// Atomic container of "free" PageIds. Detects (some) double-free bugs. @@ -72,10 +116,24 @@ /// the number of (free and/or used) pages in a stack typedef unsigned int PageCount; - PageStack(const PoolId aPoolId, const PageCount aCapacity, const size_t aPageSize); + // XXX: poolId, pageSize look misplaced due to messy separation of PagePool + // (which should support multiple Segments but does not) and PageStack + // (which should not calculate the Segment size but does) duties. + /// PageStack construction and SharedMemorySize calculation parameters + class Config { + public: + uint32_t poolId = 0; ///< pool ID + size_t pageSize = 0; ///< page size, used to calculate shared memory size + PageCount capacity = 0; ///< the maximum number of pages + + /// whether a newly created PageStack should be prefilled with PageIds + bool createFull = false; + }; + + explicit PageStack(const Config &); - PageCount capacity() const { return capacity_; } - size_t pageSize() const { return thePageSize; } + PageCount capacity() const { return config_.capacity; } + size_t pageSize() const { return config_.pageSize; } /// an approximate number of free pages PageCount size() const { return size_.load(); } @@ -87,7 +145,7 @@ bool pageIdIsValid(const PageId &page) const; /// total shared memory size required to share - static size_t SharedMemorySize(const PoolId aPoolId, const PageCount capacity, const size_t pageSize); + static size_t SharedMemorySize(const Config &); size_t sharedMemorySize() const; /// shared memory size required only by PageStack, excluding @@ -97,7 +155,7 @@ /// \returns the number of padding bytes to align PagePool::theLevels array static size_t LevelsPaddingSize(const PageCount capacity); - size_t levelsPaddingSize() const { return LevelsPaddingSize(capacity_); } + size_t levelsPaddingSize() const { return LevelsPaddingSize(config_.capacity); } /** * The following functions return PageStack IDs for the corresponding @@ -113,23 +171,12 @@ static PoolId IdForSwapDirSpace(const int dirIdx) { return 900 + dirIdx + 1; } private: - using Slot = PageStackStorageSlot; - - // XXX: theFoo members look misplaced due to messy separation of PagePool - // (which should support multiple Segments but does not) and PageStack - // (which should not calculate the Segment size but does) duties. - const PoolId thePoolId; ///< pool ID - const PageCount capacity_; ///< the maximum number of pages - const size_t thePageSize; ///< page size, used to calculate shared memory size + const Config config_; /// a lower bound for the number of free pages (for debugging purposes) std::atomic size_; - /// the index of the first free stack element or nil - std::atomic head_; - - /// slots indexed using their page number - Ipc::Mem::FlexibleArray slots_; - // No more data members should follow! See Ipc::Mem::FlexibleArray<> for details. + IdSet ids_; ///< free pages (valid with positive capacity_) + // No more data members should follow! See IdSet for details. }; } // namespace Mem diff -u -r -N squid-5.0.2/src/log/DB/log_db_daemon.8 squid-5.0.3/src/log/DB/log_db_daemon.8 --- squid-5.0.2/src/log/DB/log_db_daemon.8 2020-04-20 00:20:18.000000000 +1200 +++ squid-5.0.3/src/log/DB/log_db_daemon.8 2020-06-09 19:09:22.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "LOG_DB_DAEMON 8" -.TH LOG_DB_DAEMON 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH LOG_DB_DAEMON 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/Makefile.am squid-5.0.3/src/Makefile.am --- squid-5.0.2/src/Makefile.am 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/Makefile.am 2020-06-08 03:33:42.000000000 +1200 @@ -576,7 +576,6 @@ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ $(top_builddir)/lib/libmiscutil.la \ - $(ATOMICLIB) \ $(SSLLIB) \ $(EPOLL_LIBS) \ $(MINGW_LIBS) \ diff -u -r -N squid-5.0.2/src/Makefile.in squid-5.0.3/src/Makefile.in --- squid-5.0.2/src/Makefile.in 2020-04-20 00:08:51.000000000 +1200 +++ squid-5.0.3/src/Makefile.in 2020-06-09 18:58:21.000000000 +1200 @@ -454,8 +454,8 @@ $(top_builddir)/lib/libmiscutil.la $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ - $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_3) \ - $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6) + $(am__DEPENDENCIES_3) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_6) squid_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(squid_LDFLAGS) $(LDFLAGS) -o $@ @@ -3179,9 +3179,9 @@ $(ESI_LIBS) $(SNMP_LIBS) mem/libmem.la store/libstore.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ - $(top_builddir)/lib/libmiscutil.la $(ATOMICLIB) $(SSLLIB) \ - $(EPOLL_LIBS) $(MINGW_LIBS) $(KRB5LIBS) $(SYSTEMD_LIBS) \ - $(COMPAT_LIB) $(XTRA_LIBS) $(am__append_9) + $(top_builddir)/lib/libmiscutil.la $(SSLLIB) $(EPOLL_LIBS) \ + $(MINGW_LIBS) $(KRB5LIBS) $(SYSTEMD_LIBS) $(COMPAT_LIB) \ + $(XTRA_LIBS) $(am__append_9) @ENABLE_LOADABLE_MODULES_TRUE@squid_LDFLAGS = -export-dynamic -dlopen force unlinkd_SOURCES = unlinkd_daemon.cc unlinkd_LDADD = \ diff -u -r -N squid-5.0.2/src/mem/PoolingAllocator.h squid-5.0.3/src/mem/PoolingAllocator.h --- squid-5.0.2/src/mem/PoolingAllocator.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/mem/PoolingAllocator.h 2020-06-08 03:33:42.000000000 +1200 @@ -22,6 +22,23 @@ template PoolingAllocator(const PoolingAllocator &) noexcept {} value_type *allocate(std::size_t n) { return static_cast(memAllocRigid(n*sizeof(value_type))); } void deallocate(value_type *vp, std::size_t n) noexcept { memFreeRigid(vp, n*sizeof(value_type)); } + + // The following declarations are only necessary for compilers that do not + // fully support C++11 Allocator-related APIs, such as GCC v4.8. + // TODO: Remove after dropping support for such compilers. + + using size_type = size_t; + using pointer = Value*; + using const_pointer = const Value*; + using reference = Value&; + using const_reference = const Value&; + + template + struct rebind { + typedef PoolingAllocator other; + }; + + template void destroy(OtherValue *p) { p->~OtherValue(); } }; template diff -u -r -N squid-5.0.2/src/MemStore.cc squid-5.0.3/src/MemStore.cc --- squid-5.0.2/src/MemStore.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/MemStore.cc 2020-06-08 03:33:42.000000000 +1200 @@ -1008,10 +1008,14 @@ const int64_t entryLimit = MemStore::EntryLimit(); assert(entryLimit > 0); + + Ipc::Mem::PageStack::Config spaceConfig; + spaceConfig.poolId = Ipc::Mem::PageStack::IdForMemStoreSpace(), + spaceConfig.pageSize = 0; // the pages are stored in Ipc::Mem::Pages + spaceConfig.capacity = entryLimit; + spaceConfig.createFull = true; // all pages are initially available Must(!spaceOwner); - spaceOwner = shm_new(Ipc::Mem::PageStack)(SpaceLabel, - Ipc::Mem::PageStack::IdForMemStoreSpace(), - entryLimit, 0); + spaceOwner = shm_new(Ipc::Mem::PageStack)(SpaceLabel, spaceConfig); Must(!mapOwner); mapOwner = MemStoreMap::Init(MapLabel, entryLimit); Must(!extrasOwner); diff -u -r -N squid-5.0.2/src/neighbors.cc squid-5.0.3/src/neighbors.cc --- squid-5.0.2/src/neighbors.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/neighbors.cc 2020-06-08 03:33:42.000000000 +1200 @@ -63,6 +63,8 @@ static void peerCountMcastPeersDone(void *data); static void peerCountMcastPeersStart(void *data); static void peerCountMcastPeersSchedule(CachePeer * p, time_t when); +static void peerCountMcastPeersAbort(PeerSelector *); +static void peerCountMcastPeersCreateAndSend(CachePeer *p); static IRCB peerCountHandleIcpReply; static void neighborIgnoreNonPeer(const Ip::Address &, icp_opcode); @@ -1393,9 +1395,19 @@ static void peerCountMcastPeersStart(void *data) { + const auto peer = static_cast(data); + CallContextCreator([peer] { + peerCountMcastPeersCreateAndSend(peer); + }); + peerCountMcastPeersSchedule(peer, MCAST_COUNT_RATE); +} + +/// initiates an ICP transaction to a multicast peer +static void +peerCountMcastPeersCreateAndSend(CachePeer * const p) +{ // XXX: Do not create lots of complex fake objects (while abusing their // APIs) to pass around a few basic data points like start_ping and ping! - CachePeer *p = (CachePeer *)data; MemObject *mem; int reqnum; // TODO: use class AnyP::Uri instead of constructing and re-parsing a string @@ -1408,6 +1420,9 @@ const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initPeerMcast); auto *req = HttpRequest::FromUrlXXX(url, mx); assert(req != nullptr); + const AccessLogEntry::Pointer ale = new AccessLogEntry; + ale->request = req; + CodeContext::Reset(ale); StoreEntry *fake = storeCreateEntry(url, url, RequestFlags(), Http::METHOD_GET); const auto psstate = new PeerSelector(nullptr); psstate->request = req; @@ -1415,6 +1430,7 @@ psstate->entry = fake; psstate->peerCountMcastPeerXXX = cbdataReference(p); psstate->ping.start = current_time; + psstate->al = ale; mem = fake->mem_obj; mem->request = psstate->request; mem->start_ping = current_time; @@ -1431,13 +1447,23 @@ psstate, Config.Timeout.mcast_icp_query / 1000.0, 1); p->mcast.flags.counting = true; - peerCountMcastPeersSchedule(p, MCAST_COUNT_RATE); } static void peerCountMcastPeersDone(void *data) { const auto psstate = static_cast(data); + CallBack(psstate->al, [psstate] { + peerCountMcastPeersAbort(psstate); + delete psstate; + }); +} + +/// ends counting of multicast ICP replies +/// to the ICP query initiated by peerCountMcastPeersCreateAndSend() +static void +peerCountMcastPeersAbort(PeerSelector * const psstate) +{ StoreEntry *fake = psstate->entry; if (cbdataReferenceValid(psstate->peerCountMcastPeerXXX)) { @@ -1455,7 +1481,6 @@ fake->abort(); // sets ENTRY_ABORTED and initiates releated cleanup fake->mem_obj->request = nullptr; fake->unlock("peerCountMcastPeersDone"); - delete psstate; } static void @@ -1786,7 +1811,7 @@ * Send HTCP CLR messages to all peers configured to receive them. */ void -neighborsHtcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &method, htcp_clr_reason reason) +neighborsHtcpClear(StoreEntry * e, HttpRequest * req, const HttpRequestMethod &method, htcp_clr_reason reason) { CachePeer *p; char buf[128]; @@ -1802,7 +1827,7 @@ continue; } debugs(15, 3, "neighborsHtcpClear: sending CLR to " << p->in_addr.toUrl(buf, 128)); - htcpClear(e, uri, req, method, p, reason); + htcpClear(e, req, method, p, reason); } } diff -u -r -N squid-5.0.2/src/neighbors.h squid-5.0.3/src/neighbors.h --- squid-5.0.2/src/neighbors.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/neighbors.h 2020-06-08 03:33:42.000000000 +1200 @@ -40,7 +40,7 @@ void neighborAdd(const char *, const char *, int, int, int, int, int); void neighbors_init(void); #if USE_HTCP -void neighborsHtcpClear(StoreEntry *, const char *, HttpRequest *, const HttpRequestMethod &, htcp_clr_reason); +void neighborsHtcpClear(StoreEntry *, HttpRequest *, const HttpRequestMethod &, htcp_clr_reason); #endif CachePeer *peerFindByName(const char *); CachePeer *peerFindByNameAndPort(const char *, unsigned short); diff -u -r -N squid-5.0.2/src/ResolvedPeers.cc squid-5.0.3/src/ResolvedPeers.cc --- squid-5.0.2/src/ResolvedPeers.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ResolvedPeers.cc 2020-06-08 03:33:42.000000000 +1200 @@ -24,59 +24,117 @@ { debugs(17, 4, path); assert(path); - paths_.emplace(paths_.begin(), path); + // Cannot use pathsToSkip for a faster (reverse) search because there + // may be unavailable paths past pathsToSkip. We could remember + // the last extraction index, but, to completely avoid a linear search, + // extract*() methods should return the path index. + const auto found = std::find_if(paths_.begin(), paths_.end(), + [path](const ResolvedPeerPath &candidate) { + return candidate.connection == path; // (refcounted) pointer comparison + }); + assert(found != paths_.end()); + assert(!found->available); + found->available = true; + increaseAvailability(); + + // if we restored availability of a path that we used to skip, update + const auto pathsToTheLeft = static_cast(found - paths_.begin()); + if (pathsToTheLeft < pathsToSkip) { + pathsToSkip = pathsToTheLeft; + } else { + // *found was unavailable so pathsToSkip could not end at it + Must(pathsToTheLeft != pathsToSkip); + } } void ResolvedPeers::addPath(const Comm::ConnectionPointer &path) { paths_.emplace_back(path); + Must(paths_.back().available); // no pathsToSkip updates are needed + increaseAvailability(); +} + +/// \returns the beginning iterator for any available-path search +ResolvedPeers::Paths::iterator +ResolvedPeers::start() +{ + Must(pathsToSkip <= paths_.size()); + return paths_.begin() + pathsToSkip; // may return end() +} + +/// finalizes the iterator part of the given preliminary find*() result +ResolvedPeers::Finding +ResolvedPeers::makeFinding(const Paths::iterator &path, bool foundOther) +{ + return std::make_pair((foundOther ? paths_.end() : path), foundOther); +} + +/// \returns the first available same-peer same-family Finding or +ResolvedPeers::Finding +ResolvedPeers::findPrime(const Comm::Connection ¤tPeer) +{ + const auto path = start(); + const auto foundNextOrSpare = path != paths_.end() && + (currentPeer.getPeer() != path->connection->getPeer() || // next peer + ConnectionFamily(currentPeer) != ConnectionFamily(*path->connection)); + return makeFinding(path, foundNextOrSpare); +} + +/// \returns the first available same-peer different-family Finding or +ResolvedPeers::Finding +ResolvedPeers::findSpare(const Comm::Connection ¤tPeer) +{ + const auto primeFamily = ConnectionFamily(currentPeer); + const auto primePeer = currentPeer.getPeer(); + const auto path = std::find_if(start(), paths_.end(), + [primeFamily, primePeer](const ResolvedPeerPath &candidate) { + if (!candidate.available) + return false; + if (primePeer != candidate.connection->getPeer()) + return true; // found next peer + if (primeFamily != ConnectionFamily(*candidate.connection)) + return true; // found spare + return false; + }); + const auto foundNext = path != paths_.end() && + primePeer != path->connection->getPeer(); + return makeFinding(path, foundNext); +} + +/// \returns the first available same-peer Finding or +ResolvedPeers::Finding +ResolvedPeers::findPeer(const Comm::Connection ¤tPeer) +{ + const auto path = start(); + const auto foundNext = path != paths_.end() && + currentPeer.getPeer() != path->connection->getPeer(); + return makeFinding(path, foundNext); } Comm::ConnectionPointer ResolvedPeers::extractFront() { Must(!empty()); - return extractFound("first: ", paths_.begin()); + return extractFound("first: ", start()); } Comm::ConnectionPointer ResolvedPeers::extractPrime(const Comm::Connection ¤tPeer) { - if (!empty()) { - const auto peerToMatch = currentPeer.getPeer(); - const auto familyToMatch = ConnectionFamily(currentPeer); - const auto &conn = paths_.front(); - if (conn->getPeer() == peerToMatch && familyToMatch == ConnectionFamily(*conn)) - return extractFound("same-peer same-family match: ", paths_.begin()); - } + const auto found = findPrime(currentPeer).first; + if (found != paths_.end()) + return extractFound("same-peer same-family match: ", found); debugs(17, 7, "no same-peer same-family paths"); return nullptr; } -/// If spare paths exist for currentPeer, returns the first spare path iterator. -/// Otherwise, if there are paths for other peers, returns one of those. -/// Otherwise, returns the end() iterator. -Comm::ConnectionList::iterator -ResolvedPeers::findSpareOrNextPeer(const Comm::Connection ¤tPeer) -{ - const auto peerToMatch = currentPeer.getPeer(); - const auto familyToAvoid = ConnectionFamily(currentPeer); - // Optimization: Also stop at the first mismatching peer because all - // same-peer paths are grouped together. - return std::find_if(paths_.begin(), paths_.end(), - [peerToMatch, familyToAvoid](const Comm::ConnectionPointer &conn) { - return peerToMatch != conn->getPeer() || - familyToAvoid != ConnectionFamily(*conn); - }); -} - Comm::ConnectionPointer ResolvedPeers::extractSpare(const Comm::Connection ¤tPeer) { - auto found = findSpareOrNextPeer(currentPeer); - if (found != paths_.end() && currentPeer.getPeer() == (*found)->getPeer()) + const auto found = findSpare(currentPeer).first; + if (found != paths_.end()) return extractFound("same-peer different-family match: ", found); debugs(17, 7, "no same-peer different-family paths"); @@ -85,48 +143,58 @@ /// convenience method to finish a successful extract*() call Comm::ConnectionPointer -ResolvedPeers::extractFound(const char *description, const Comm::ConnectionList::iterator &found) +ResolvedPeers::extractFound(const char *description, const Paths::iterator &found) { - const auto path = *found; - paths_.erase(found); - debugs(17, 7, description << path); - return path; + auto &path = *found; + debugs(17, 7, description << path.connection); + assert(path.available); + path.available = false; + decreaseAvailability(); + + // if we extracted the left-most available path, find the next one + if (static_cast(found - paths_.begin()) == pathsToSkip) { + while (++pathsToSkip < paths_.size() && !paths_[pathsToSkip].available) {} + } + + return path.connection; } bool ResolvedPeers::haveSpare(const Comm::Connection ¤tPeer) { - const auto found = findSpareOrNextPeer(currentPeer); - return found != paths_.end() && - currentPeer.getPeer() == (*found)->getPeer(); + return findSpare(currentPeer).first != paths_.end(); +} + +/// whether paths_ have no (and will have no) Xs for the current peer based on +/// the given findX(current peer) result +bool +ResolvedPeers::doneWith(const Finding &findings) const +{ + if (findings.first != paths_.end()) + return false; // not done because the caller found a viable path X + + // The caller did not find any path X. If the caller found any "other" + // paths, then we are done with paths X. If there are no "other" paths, + // then destinationsFinalized is the answer. + return findings.second ? true : destinationsFinalized; } bool ResolvedPeers::doneWithSpares(const Comm::Connection ¤tPeer) { - const auto found = findSpareOrNextPeer(currentPeer); - if (found == paths_.end()) - return destinationsFinalized; - return currentPeer.getPeer() != (*found)->getPeer(); + return doneWith(findSpare(currentPeer)); } bool -ResolvedPeers::doneWithPrimes(const Comm::Connection ¤tPeer) const +ResolvedPeers::doneWithPrimes(const Comm::Connection ¤tPeer) { - const auto first = paths_.begin(); - if (first == paths_.end()) - return destinationsFinalized; - return currentPeer.getPeer() != (*first)->getPeer() || - ConnectionFamily(currentPeer) != ConnectionFamily(**first); + return doneWith(findPrime(currentPeer)); } bool -ResolvedPeers::doneWithPeer(const Comm::Connection ¤tPeer) const +ResolvedPeers::doneWithPeer(const Comm::Connection ¤tPeer) { - const auto first = paths_.begin(); - if (first == paths_.end()) - return destinationsFinalized; - return currentPeer.getPeer() != (*first)->getPeer(); + return doneWith(findPeer(currentPeer)); } int @@ -135,6 +203,22 @@ return conn.remote.isIPv4() ? AF_INET : AF_INET6; } +/// increments the number of available paths +void +ResolvedPeers::increaseAvailability() +{ + ++availablePaths; + assert(availablePaths <= paths_.size()); +} + +/// decrements the number of available paths +void +ResolvedPeers::decreaseAvailability() +{ + assert(availablePaths > 0); + --availablePaths; +} + std::ostream & operator <<(std::ostream &os, const ResolvedPeers &peers) { diff -u -r -N squid-5.0.2/src/ResolvedPeers.h squid-5.0.3/src/ResolvedPeers.h --- squid-5.0.2/src/ResolvedPeers.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ResolvedPeers.h 2020-06-08 03:33:42.000000000 +1200 @@ -13,6 +13,16 @@ #include "comm/forward.h" #include +#include + +class ResolvedPeerPath +{ +public: + explicit ResolvedPeerPath(const Comm::ConnectionPointer &conn) : connection(conn), available(true) {} + + Comm::ConnectionPointer connection; ///< (the address of) a path + bool available; ///< whether this path may be used (i.e., has not been tried already) +}; /// cache_peer and origin server addresses (a.k.a. paths) /// selected and resolved by the peering code @@ -21,17 +31,20 @@ MEMPROXY_CLASS(ResolvedPeers); public: + // ResolvedPeerPaths in addPath() call order + typedef std::vector Paths; + using size_type = Paths::size_type; typedef RefCount Pointer; ResolvedPeers(); /// whether we lack any known candidate paths - bool empty() const { return paths_.empty(); } + bool empty() const { return !availablePaths; } /// add a candidate path to try after all the existing paths void addPath(const Comm::ConnectionPointer &); - /// add a candidate path to try before all the existing paths + /// re-inserts the previously extracted address into the same position void retryPath(const Comm::ConnectionPointer &); /// extracts and returns the first queued address @@ -49,16 +62,16 @@ bool haveSpare(const Comm::Connection ¤tPeer); /// whether extractPrime() returns and will continue to return nil - bool doneWithPrimes(const Comm::Connection ¤tPeer) const; + bool doneWithPrimes(const Comm::Connection ¤tPeer); /// whether extractSpare() returns and will continue to return nil bool doneWithSpares(const Comm::Connection ¤tPeer); /// whether doneWithPrimes() and doneWithSpares() are true for currentPeer - bool doneWithPeer(const Comm::Connection ¤tPeer) const; + bool doneWithPeer(const Comm::Connection ¤tPeer); /// the current number of candidate paths - Comm::ConnectionList::size_type size() const { return paths_.size(); } + size_type size() const { return availablePaths; } /// whether all of the available candidate paths received from DNS bool destinationsFinalized = false; @@ -67,13 +80,34 @@ bool notificationPending = false; private: + /// A find*() result: An iterator of the found path (or paths_.end()) and + /// whether the "other" path was found instead. + typedef std::pair Finding; + /// The protocol family of the given path, AF_INET or AF_INET6 static int ConnectionFamily(const Comm::Connection &conn); - Comm::ConnectionList::iterator findSpareOrNextPeer(const Comm::Connection ¤tPeer); - Comm::ConnectionPointer extractFound(const char *description, const Comm::ConnectionList::iterator &found); + Paths::iterator start(); + Finding findSpare(const Comm::Connection ¤tPeer); + Finding findPrime(const Comm::Connection ¤tPeer); + Finding findPeer(const Comm::Connection ¤tPeer); + Comm::ConnectionPointer extractFound(const char *description, const Paths::iterator &found); + Finding makeFinding(const Paths::iterator &found, bool foundOther); + + bool doneWith(const Finding &findings) const; + + void increaseAvailability(); + void decreaseAvailability(); + + Paths paths_; ///< resolved addresses in (peer, family) order + + /// the number of leading paths_ elements that are all currently unavailable + /// i.e. the size of the front paths_ segment comprised of unavailable items + /// i.e. the position of the first available path (or paths_.size()) + size_type pathsToSkip = 0; - Comm::ConnectionList paths_; + /// the total number of currently available elements in paths_ + size_type availablePaths = 0; }; /// summarized ResolvedPeers (for debugging) diff -u -r -N squid-5.0.2/src/security/BlindPeerConnector.cc squid-5.0.3/src/security/BlindPeerConnector.cc --- squid-5.0.2/src/security/BlindPeerConnector.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/security/BlindPeerConnector.cc 2020-06-08 03:33:42.000000000 +1200 @@ -7,6 +7,7 @@ */ #include "squid.h" +#include "AccessLogEntry.h" #include "CachePeer.h" #include "comm/Connection.h" #include "errorpage.h" diff -u -r -N squid-5.0.2/src/security/cert_validators/fake/security_fake_certverify.8 squid-5.0.3/src/security/cert_validators/fake/security_fake_certverify.8 --- squid-5.0.2/src/security/cert_validators/fake/security_fake_certverify.8 2020-04-20 00:20:18.000000000 +1200 +++ squid-5.0.3/src/security/cert_validators/fake/security_fake_certverify.8 2020-06-09 19:09:23.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "SECURITY_FAKE_CERTVERIFY 8" -.TH SECURITY_FAKE_CERTVERIFY 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH SECURITY_FAKE_CERTVERIFY 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/security/Handshake.cc squid-5.0.3/src/security/Handshake.cc --- squid-5.0.2/src/security/Handshake.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/security/Handshake.cc 2020-06-08 03:33:42.000000000 +1200 @@ -106,11 +106,11 @@ } // namespace Security -/// Convenience helper: We parse ProtocolVersion but store "int". +/// parse TLS ProtocolVersion (uint16) and convert it to AnyP::ProtocolVersion static AnyP::ProtocolVersion -ParseProtocolVersion(Parser::BinaryTokenizer &tk) +ParseProtocolVersion(Parser::BinaryTokenizer &tk, const char *contextLabel = ".version") { - Parser::BinaryTokenizerContext context(tk, ".version"); + Parser::BinaryTokenizerContext context(tk, contextLabel); uint8_t vMajor = tk.uint8(".major"); uint8_t vMinor = tk.uint8(".minor"); if (vMajor == 0 && vMinor == 2) @@ -187,10 +187,11 @@ /* Security::HandshakeParser */ -Security::HandshakeParser::HandshakeParser(): +Security::HandshakeParser::HandshakeParser(const MessageSource source): details(new TlsDetails), state(atHelloNone), resumingSession(false), + messageSource(source), currentContentType(0), done(nullptr), expectingModernRecords(false) @@ -285,12 +286,19 @@ { Must(currentContentType == ContentType::ctChangeCipherSpec); // We are currently ignoring Change Cipher Spec Protocol messages. - skipMessage("ChangeCipherCpec msg [fragment]"); + skipMessage("ChangeCipherSpec msg [fragment]"); + + // In TLS v1.2 and earlier, ChangeCipherSpec is sent after Hello (when + // tlsSupportedVersion is already known) and indicates session resumption. + // In later TLS versions, ChangeCipherSpec may be sent before and after + // Hello, but it is unused for session resumption and should be ignored. + if (!details->tlsSupportedVersion || Tls1p3orLater(details->tlsSupportedVersion)) + return; - // Everything after the ChangeCipherCpec message may be encrypted. - // Continuing parsing is pointless. Stop here. resumingSession = true; - done = "ChangeCipherCpec"; + + // Everything after the ChangeCipherSpec message may be encrypted. Stop. + done = "ChangeCipherSpec in v1.2-"; } void @@ -316,14 +324,19 @@ switch (message.msg_type) { case HandshakeType::hskClientHello: Must(state < atHelloReceived); + Must(messageSource == fromClient); Security::HandshakeParser::parseClientHelloHandshakeMessage(message.msg_body); state = atHelloReceived; done = "ClientHello"; return; case HandshakeType::hskServerHello: Must(state < atHelloReceived); + Must(messageSource == fromServer); parseServerHelloHandshakeMessage(message.msg_body); state = atHelloReceived; + // for TLSv1.3 and later, anything after the server Hello is encrypted + if (Tls1p3orLater(details->tlsSupportedVersion)) + done = "ServerHello in v1.3+"; return; case HandshakeType::hskCertificate: Must(state < atCertificatesReceived); @@ -424,6 +437,10 @@ case 35: // SessionTicket TLS Extension; RFC 5077 details->tlsTicketsExtension = true; details->hasTlsTicket = !extension.data.isEmpty(); + break; + case 43: // supported_versions extension; RFC 8446 + parseSupportedVersionsExtension(extension.data); + break; case 13172: // Next Protocol Negotiation Extension (expired draft?) default: break; @@ -504,6 +521,78 @@ return SBuf(); // SNI extension lacks host_name } +/// RFC 8446 Section 4.2.1: SupportedVersions extension +void +Security::HandshakeParser::parseSupportedVersionsExtension(const SBuf &extensionData) const +{ + // Upon detecting a quoted RFC MUST violation, this parser immediately + // returns, ignoring the entire extension and resulting in Squid relying on + // the legacy_version field value or another (valid) supported_versions + // extension. The alternative would be to reject the whole handshake as + // invalid. Deployment experience will show which alternative is the best. + + // Please note that several of these MUSTs also imply certain likely + // handling of a hypothetical next TLS version (e.g., v1.4). + + // RFC 8446 Section 4.1.2: + // In TLS 1.3, the client indicates its version preferences in the + // "supported_versions" extension (Section 4.2.1) and the legacy_version + // field MUST be set to 0x0303, which is the version number for TLS 1.2. + // + // RFC 8446 Section 4.2.1: + // A server which negotiates TLS 1.3 MUST respond by sending a + // "supported_versions" extension containing the selected version value + // (0x0304). It MUST set the ServerHello.legacy_version field to 0x0303 + // (TLS 1.2). + // + // Ignore supported_versions senders violating legacy_version MUSTs above: + if (details->tlsSupportedVersion != AnyP::ProtocolVersion(AnyP::PROTO_TLS, 1, 2)) + return; + + AnyP::ProtocolVersion supportedVersionMax; + if (messageSource == fromClient) { + Parser::BinaryTokenizer tkList(extensionData); + Parser::BinaryTokenizer tkVersions(tkList.pstring8("SupportedVersions")); + while (!tkVersions.atEnd()) { + const auto version = ParseProtocolVersion(tkVersions, "supported_version"); + if (!supportedVersionMax || TlsVersionEarlierThan(supportedVersionMax, version)) + supportedVersionMax = version; + } + + // ignore empty supported_versions + if (!supportedVersionMax) + return; + + // supportedVersionMax here may be "earlier" than tlsSupportedVersion: A + // TLS v1.3 client may try to negotiate a _legacy_ version X with a TLS + // v1.3 server by sending supported_versions containing just X. + } else { + assert(messageSource == fromServer); + Parser::BinaryTokenizer tkVersion(extensionData); + const auto version = ParseProtocolVersion(tkVersion, "selected_version"); + // RFC 8446 Section 4.2.1: + // A server which negotiates a version of TLS prior to TLS 1.3 [...] + // MUST NOT send the "supported_versions" extension. + if (Tls1p2orEarlier(version)) + return; + supportedVersionMax = version; + } + + // We overwrite Hello-derived legacy_version because the following MUSTs + // indicate that it is ignored in the presence of valid supported_versions + // as far as the negotiated version is concerned. For simplicity sake, we + // may also overwrite previous valid supported_versions extensions (if any). + // + // RFC 8446 Section 4.2.1: + // If this extension is present in the ClientHello, servers MUST NOT use the + // ClientHello.legacy_version value for version negotiation and MUST use + // only the "supported_versions" extension to determine client preferences. + // Servers MUST only select a version of TLS present in that extension + debugs(83, 7, "found " << supportedVersionMax); + assert(supportedVersionMax); + details->tlsSupportedVersion = supportedVersionMax; +} + void Security::HandshakeParser::skipMessage(const char *description) { @@ -647,6 +736,9 @@ #if defined(TLSEXT_TYPE_next_proto_neg) // 13172 extensions.insert(TLSEXT_TYPE_next_proto_neg); #endif +#if defined(TLSEXT_TYPE_supported_versions) // 43 + extensions.insert(TLSEXT_TYPE_supported_versions); +#endif /* * OpenSSL does not support these last extensions by default, but those diff -u -r -N squid-5.0.2/src/security/Handshake.h squid-5.0.3/src/security/Handshake.h --- squid-5.0.2/src/security/Handshake.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/security/Handshake.h 2020-06-08 03:33:42.000000000 +1200 @@ -29,7 +29,11 @@ std::ostream & print(std::ostream &os) const; AnyP::ProtocolVersion tlsVersion; ///< The TLS hello message version - AnyP::ProtocolVersion tlsSupportedVersion; ///< The requested/used TLS version + + /// For most compliant TLS v1.3+ agents, this is supported_versions maximum. + /// For others agents, this is the legacy_version field. + AnyP::ProtocolVersion tlsSupportedVersion; + bool compressionSupported; ///< The requested/used compressed method SBuf serverName; ///< The SNI hostname, if any bool doHeartBeats; @@ -59,7 +63,10 @@ /// The parsing states typedef enum {atHelloNone = 0, atHelloStarted, atHelloReceived, atCertificatesReceived, atHelloDoneReceived, atNstReceived, atCcsReceived, atFinishReceived} ParserState; - HandshakeParser(); + /// the originator of the TLS handshake being parsed + typedef enum { fromClient = 0, fromServer } MessageSource; + + explicit HandshakeParser(MessageSource); /// Parses the initial sequence of raw bytes sent by the TLS/SSL agent. /// Returns true upon successful completion (e.g., got HelloDone). @@ -67,7 +74,7 @@ /// Throws on errors. bool parseHello(const SBuf &data); - TlsDetails::Pointer details; ///< TLS handshake meta info or nil. + TlsDetails::Pointer details; ///< TLS handshake meta info. Never nil. Security::CertList serverCertificates; ///< parsed certificates chain @@ -75,6 +82,9 @@ bool resumingSession; ///< True if this is a resuming session + /// whether we are parsing Server or Client TLS handshake messages + MessageSource messageSource; + private: bool isSslv2Record(const SBuf &raw) const; void parseRecord(); @@ -96,6 +106,7 @@ bool parseCompressionMethods(const SBuf &raw); void parseExtensions(const SBuf &raw); SBuf parseSniExtension(const SBuf &extensionData) const; + void parseSupportedVersionsExtension(const SBuf &extensionData) const; void parseCiphers(const SBuf &raw); void parseV23Ciphers(const SBuf &raw); @@ -120,6 +131,40 @@ YesNoNone expectingModernRecords; }; +/// whether the given protocol belongs to the TLS/SSL group of protocols +inline bool +TlsFamilyProtocol(const AnyP::ProtocolVersion &version) +{ + return (version.protocol == AnyP::PROTO_TLS || version.protocol == AnyP::PROTO_SSL); +} + +/// whether TLS/SSL protocol `a` precedes TLS/SSL protocol `b` +inline bool +TlsVersionEarlierThan(const AnyP::ProtocolVersion &a, const AnyP::ProtocolVersion &b) +{ + Must(TlsFamilyProtocol(a)); + Must(TlsFamilyProtocol(b)); + + if (a.protocol == b.protocol) + return a < b; + + return a.protocol == AnyP::PROTO_SSL; // implies that b is TLS +} + +/// whether the given TLS/SSL protocol is TLS v1.2 or earlier, including SSL +inline bool +Tls1p2orEarlier(const AnyP::ProtocolVersion &p) +{ + return TlsVersionEarlierThan(p, AnyP::ProtocolVersion(AnyP::PROTO_TLS, 1, 3)); +} + +/// whether the given TLS/SSL protocol is TLS v1.3 or later +inline bool +Tls1p3orLater(const AnyP::ProtocolVersion &p) +{ + return !Tls1p2orEarlier(p); +} + } #endif // SQUID_SECURITY_HANDSHAKE_H diff -u -r -N squid-5.0.2/src/security/NegotiationHistory.cc squid-5.0.3/src/security/NegotiationHistory.cc --- squid-5.0.2/src/security/NegotiationHistory.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/security/NegotiationHistory.cc 2020-06-08 03:33:42.000000000 +1200 @@ -25,7 +25,7 @@ const char * Security::NegotiationHistory::printTlsVersion(AnyP::ProtocolVersion const &v) const { - if (v.protocol != AnyP::PROTO_SSL && v.protocol != AnyP::PROTO_TLS) + if (!TlsFamilyProtocol(v)) return nullptr; static char buf[512]; diff -u -r -N squid-5.0.2/src/security/PeerConnector.cc squid-5.0.3/src/security/PeerConnector.cc --- squid-5.0.2/src/security/PeerConnector.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/security/PeerConnector.cc 2020-06-08 03:33:42.000000000 +1200 @@ -15,8 +15,11 @@ #include "Downloader.h" #include "errorpage.h" #include "fde.h" +#include "FwdState.h" #include "http/Stream.h" #include "HttpRequest.h" +#include "neighbors.h" +#include "pconn.h" #include "security/NegotiationHistory.h" #include "security/PeerConnector.h" #include "SquidConfig.h" @@ -31,6 +34,7 @@ Security::PeerConnector::PeerConnector(const Comm::ConnectionPointer &aServerConn, AsyncCall::Pointer &aCallback, const AccessLogEntryPointer &alp, const time_t timeout) : AsyncJob("Security::PeerConnector"), + noteFwdPconnUse(false), serverConn(aServerConn), al(alp), callback(aCallback), @@ -39,14 +43,17 @@ useCertValidator_(true), certsDownloads(0) { - debugs(83, 5, "Security::PeerConnector constructed, this=" << (void*)this); + debugs(83, 5, serverConn); + // if this throws, the caller's cb dialer is not our CbDialer Must(dynamic_cast(callback->getDialer())); -} -Security::PeerConnector::~PeerConnector() -{ - debugs(83, 5, "Security::PeerConnector destructed, this=" << (void*)this); + // watch for external connection closures + Must(Comm::IsConnOpen(serverConn)); + Must(!fd_table[serverConn->fd].closing()); + typedef CommCbMemFunT Dialer; + closeHandler = JobCallback(9, 5, Dialer, this, Security::PeerConnector::commCloseHandler); + comm_add_close_handler(serverConn->fd, closeHandler); } bool Security::PeerConnector::doneAll() const @@ -61,8 +68,16 @@ AsyncJob::start(); debugs(83, 5, "this=" << (void*)this); + // we own this Comm::Connection object and its fd exclusively, but must bail + // if others started closing the socket while we were waiting to start() + assert(Comm::IsConnOpen(serverConn)); + if (fd_table[serverConn->fd].closing()) { + bail(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw(), al)); + return; + } + Security::SessionPointer tmp; - if (prepareSocket() && initialize(tmp)) + if (initialize(tmp)) negotiate(); else mustStop("Security::PeerConnector TLS socket initialize failed"); @@ -71,34 +86,25 @@ void Security::PeerConnector::commCloseHandler(const CommCloseCbParams ¶ms) { + closeHandler = nullptr; + debugs(83, 5, "FD " << params.fd << ", Security::PeerConnector=" << params.data); - connectionClosed("Security::PeerConnector::commCloseHandler"); + const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request.getRaw(), al); +#if USE_OPENSSL + err->detail = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, nullptr, nullptr); +#endif + bail(err); } void -Security::PeerConnector::connectionClosed(const char *reason) +Security::PeerConnector::commTimeoutHandler(const CommTimeoutCbParams &) { - debugs(83, 5, reason << " socket closed/closing. this=" << (void*)this); - mustStop(reason); - callback = NULL; -} - -bool -Security::PeerConnector::prepareSocket() -{ - debugs(83, 5, serverConnection() << ", this=" << (void*)this); - if (!Comm::IsConnOpen(serverConnection()) || fd_table[serverConnection()->fd].closing()) { - connectionClosed("Security::PeerConnector::prepareSocket"); - return false; - } - - debugs(83, 5, serverConnection()); - - // watch for external connection closures - typedef CommCbMemFunT Dialer; - closeHandler = JobCallback(9, 5, Dialer, this, Security::PeerConnector::commCloseHandler); - comm_add_close_handler(serverConnection()->fd, closeHandler); - return true; + debugs(83, 5, serverConnection() << " timedout. this=" << (void*)this); + const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scGatewayTimeout, request.getRaw(), al); +#if USE_OPENSSL + err->detail = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, nullptr, nullptr); +#endif + bail(err); } bool @@ -163,9 +169,6 @@ void Security::PeerConnector::negotiate() { - if (!Comm::IsConnOpen(serverConnection())) - return; - const int fd = serverConnection()->fd; if (fd_table[fd].closing()) return; @@ -204,7 +207,7 @@ if (!sslFinalized()) return; - callBack(); + sendSuccess(); } bool @@ -241,7 +244,6 @@ noteNegotiationDone(anErr); bail(anErr); - serverConn->close(); return true; } } @@ -259,9 +261,6 @@ Ssl::ErrorDetail *errDetails = NULL; bool validatorFailed = false; - if (!Comm::IsConnOpen(serverConnection())) { - return; - } if (Debug::Enabled(83, 5)) { Security::SessionPointer ssl(fd_table[serverConnection()->fd].ssl); @@ -281,7 +280,7 @@ if (!errDetails && !validatorFailed) { noteNegotiationDone(NULL); - callBack(); + sendSuccess(); return; } @@ -296,7 +295,6 @@ noteNegotiationDone(anErr); bail(anErr); - serverConn->close(); return; } #endif @@ -473,9 +471,11 @@ #endif // read timeout to avoid getting stuck while reading from a silent server - AsyncCall::Pointer nil; + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(83, 5, + TimeoutDialer, this, Security::PeerConnector::commTimeoutHandler); const auto timeout = Comm::MortalReadTimeout(startTime, negotiationTimeout); - commSetConnTimeout(serverConnection(), timeout, nil); + commSetConnTimeout(serverConnection(), timeout, timeoutCall); Comm::SetSelect(fd, COMM_SELECT_READ, &NegotiateSsl, new Pointer(this), 0); } @@ -545,13 +545,34 @@ Must(dialer); dialer->answer().error = error; + if (const auto p = serverConnection()->getPeer()) + peerConnectFailed(p); + + callBack(); + disconnect(); + + if (noteFwdPconnUse) + fwdPconnPool->noteUses(fd_table[serverConn->fd].pconn.uses); + serverConn->close(); + serverConn = nullptr; +} + +void +Security::PeerConnector::sendSuccess() +{ callBack(); - // Our job is done. The callabck recepient will probably close the failed - // peer connection and try another peer or go direct (if possible). We - // can close the connection ourselves (our error notification would reach - // the recepient before the fd-closure notification), but we would rather - // minimize the number of fd-closure notifications and let the recepient - // manage the TCP state of the connection. + disconnect(); +} + +void +Security::PeerConnector::disconnect() +{ + if (closeHandler) { + comm_remove_close_handler(serverConnection()->fd, closeHandler); + closeHandler = nullptr; + } + + commUnsetConnTimeout(serverConnection()); } void @@ -563,10 +584,6 @@ // Do this now so that if we throw below, swanSong() assert that we _tried_ // to call back holds. callback = NULL; // this should make done() true - - // remove close handler - comm_remove_close_handler(serverConnection()->fd, closeHandler); - CbDialer *dialer = dynamic_cast(cb->getDialer()); Must(dialer); dialer->answer().conn = serverConnection(); diff -u -r -N squid-5.0.2/src/security/PeerConnector.h squid-5.0.3/src/security/PeerConnector.h --- squid-5.0.2/src/security/PeerConnector.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/security/PeerConnector.h 2020-06-08 03:33:42.000000000 +1200 @@ -31,34 +31,12 @@ { /** - * Initiates encryption on a connection to peers or servers. - * Despite its name does not perform any connect(2) operations. + * Initiates encryption of a given open TCP connection to a peer or server. + * Despite its name does not perform any connect(2) operations. Owns the + * connection during TLS negotiations. The caller receives EncryptorAnswer. * * Contains common code and interfaces of various specialized PeerConnector's, * including peer certificate validation code. - \par - * The caller receives a call back with Security::EncryptorAnswer. If answer.error - * is not nil, then there was an error and the encryption to the peer or server - * was not fully established. The error object is suitable for error response - * generation. - \par - * The caller must monitor the connection for closure because this - * job will not inform the caller about such events. - \par - * PeerConnector class currently supports a form of TLS negotiation timeout, - * which is accounted only when sets the read timeout from encrypted peers/servers. - * For a complete solution, the caller must monitor the overall connection - * establishment timeout and close the connection on timeouts. This is probably - * better than having dedicated (or none at all!) timeouts for peer selection, - * DNS lookup, TCP handshake, SSL handshake, etc. Some steps may have their - * own timeout, but not all steps should be forced to have theirs. - * XXX: tunnel.cc and probably other subsystems do not have an "overall - * connection establishment" timeout. We need to change their code so that they - * start monitoring earlier and close on timeouts. This change may need to be - * discussed on squid-dev. - \par - * This job never closes the connection, even on errors. If a 3rd-party - * closes the connection, this job simply quits without informing the caller. */ class PeerConnector: virtual public AsyncJob { @@ -81,7 +59,10 @@ AsyncCall::Pointer &aCallback, const AccessLogEntryPointer &alp, const time_t timeout = 0); - virtual ~PeerConnector(); + virtual ~PeerConnector() = default; + + /// hack: whether the connection requires fwdPconnPool->noteUses() + bool noteFwdPconnUse; protected: // AsyncJob API @@ -90,17 +71,12 @@ virtual void swanSong(); virtual const char *status() const; + /// The connection read timeout callback handler. + void commTimeoutHandler(const CommTimeoutCbParams &); + /// The comm_close callback handler. void commCloseHandler(const CommCloseCbParams ¶ms); - /// Inform us that the connection is closed. Does the required clean-up. - void connectionClosed(const char *reason); - - /// Sets up TCP socket-related notification callbacks if things go wrong. - /// If socket already closed return false, else install the comm_close - /// handler to monitor the socket. - bool prepareSocket(); - /// \returns true on successful TLS session initialization virtual bool initialize(Security::SessionPointer &); @@ -160,12 +136,18 @@ /// mimics FwdState to minimize changes to FwdState::initiate/negotiateSsl Comm::ConnectionPointer const &serverConnection() const { return serverConn; } - void bail(ErrorState *error); ///< Return an error to the PeerConnector caller + /// sends the given error to the initiator + void bail(ErrorState *error); + + /// sends the encrypted connection to the initiator + void sendSuccess(); - /// Callback the caller class, and pass the ready to communicate secure - /// connection or an error if PeerConnector failed. + /// a bail(), sendSuccess() helper: sends results to the initiator void callBack(); + /// a bail(), sendSuccess() helper: stops monitoring the connection + void disconnect(); + /// If called the certificates validator will not used void bypassCertValidator() {useCertValidator_ = false;} diff -u -r -N squid-5.0.2/src/ssl/bio.cc squid-5.0.3/src/ssl/bio.cc --- squid-5.0.2/src/ssl/bio.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ssl/bio.cc 2020-06-08 03:33:42.000000000 +1200 @@ -250,7 +250,8 @@ parsedHandshake(false), parseError(false), bumpMode_(bumpNone), - rbufConsumePos(0) + rbufConsumePos(0), + parser_(Security::HandshakeParser::fromServer) { } @@ -554,6 +555,13 @@ return parser_.resumingSession; } +bool +Ssl::ServerBio::encryptedCertificates() const +{ + return parser_.details->tlsSupportedVersion && + Security::Tls1p3orLater(parser_.details->tlsSupportedVersion); +} + /// initializes BIO table after allocation static int squid_bio_create(BIO *bi) @@ -717,6 +725,12 @@ SSL_set_options(ssl, SSL_OP_NO_COMPRESSION); #endif +#if defined(SSL_OP_NO_TLSv1_3) + // avoid "inappropriate fallback" OpenSSL error messages + if (details->tlsSupportedVersion && Security::Tls1p2orEarlier(details->tlsSupportedVersion)) + SSL_set_options(ssl, SSL_OP_NO_TLSv1_3); +#endif + #if defined(TLSEXT_STATUSTYPE_ocsp) if (details->tlsStatusRequest) SSL_set_tlsext_status_type(ssl, TLSEXT_STATUSTYPE_ocsp); diff -u -r -N squid-5.0.2/src/ssl/bio.h squid-5.0.3/src/ssl/bio.h --- squid-5.0.2/src/ssl/bio.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ssl/bio.h 2020-06-08 03:33:42.000000000 +1200 @@ -143,6 +143,10 @@ bool resumingSession(); + /// whether the server encrypts its certificate (e.g., TLS v1.3) + /// \retval false the server uses plain certs or its intent is unknown + bool encryptedCertificates() const; + /// The write hold state bool holdWrite() const {return holdWrite_;} /// Enables or disables the write hold state diff -u -r -N squid-5.0.2/src/ssl/ErrorDetail.cc squid-5.0.3/src/ssl/ErrorDetail.cc --- squid-5.0.2/src/ssl/ErrorDetail.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ssl/ErrorDetail.cc 2020-06-08 03:33:42.000000000 +1200 @@ -233,6 +233,9 @@ "X509_V_ERR_SUBTREE_MINMAX" }, #endif + { X509_V_ERR_APPLICATION_VERIFICATION, //50 + "X509_V_ERR_APPLICATION_VERIFICATION" + }, #if defined(X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE) { X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE, //51 @@ -257,9 +260,132 @@ "X509_V_ERR_CRL_PATH_VALIDATION_ERROR" }, #endif - { X509_V_ERR_APPLICATION_VERIFICATION, - "X509_V_ERR_APPLICATION_VERIFICATION" +#if defined(X509_V_ERR_PATH_LOOP) + { + X509_V_ERR_PATH_LOOP, //55 + "X509_V_ERR_PATH_LOOP" + }, +#endif +#if defined(X509_V_ERR_SUITE_B_INVALID_VERSION) + { + X509_V_ERR_SUITE_B_INVALID_VERSION, //56 + "X509_V_ERR_SUITE_B_INVALID_VERSION" + }, +#endif +#if defined(X509_V_ERR_SUITE_B_INVALID_ALGORITHM) + { + X509_V_ERR_SUITE_B_INVALID_ALGORITHM, //57 + "X509_V_ERR_SUITE_B_INVALID_ALGORITHM" + }, +#endif +#if defined(X509_V_ERR_SUITE_B_INVALID_CURVE) + { + X509_V_ERR_SUITE_B_INVALID_CURVE, //58 + "X509_V_ERR_SUITE_B_INVALID_CURVE" + }, +#endif +#if defined(X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM) + { + X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM, //59 + "X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM" + }, +#endif +#if defined(X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED) + { + X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED, //60 + "X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED" + }, +#endif +#if defined(X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256) + { + X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256, //61 + "X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256" + }, +#endif +#if defined(X509_V_ERR_HOSTNAME_MISMATCH) + { + X509_V_ERR_HOSTNAME_MISMATCH, //62 + "X509_V_ERR_HOSTNAME_MISMATCH" + }, +#endif +#if defined(X509_V_ERR_EMAIL_MISMATCH) + { + X509_V_ERR_EMAIL_MISMATCH, //63 + "X509_V_ERR_EMAIL_MISMATCH" + }, +#endif +#if defined(X509_V_ERR_IP_ADDRESS_MISMATCH) + { + X509_V_ERR_IP_ADDRESS_MISMATCH, //64 + "X509_V_ERR_IP_ADDRESS_MISMATCH" + }, +#endif +#if defined(X509_V_ERR_DANE_NO_MATCH) + { + X509_V_ERR_DANE_NO_MATCH, //65 + "X509_V_ERR_DANE_NO_MATCH" }, +#endif +#if defined(X509_V_ERR_EE_KEY_TOO_SMALL) + { + X509_V_ERR_EE_KEY_TOO_SMALL, //66 + "X509_V_ERR_EE_KEY_TOO_SMALL" + }, +#endif +#if defined(X509_V_ERR_CA_KEY_TOO_SMALL) + { + X509_V_ERR_CA_KEY_TOO_SMALL, //67 + "X509_V_ERR_CA_KEY_TOO_SMALL" + }, +#endif +#if defined(X509_V_ERR_CA_MD_TOO_WEAK) + { + X509_V_ERR_CA_MD_TOO_WEAK, //68 + "X509_V_ERR_CA_MD_TOO_WEAK" + }, +#endif +#if defined(X509_V_ERR_INVALID_CALL) + { + X509_V_ERR_INVALID_CALL, //69 + "X509_V_ERR_INVALID_CALL" + }, +#endif +#if defined(X509_V_ERR_STORE_LOOKUP) + { + X509_V_ERR_STORE_LOOKUP, //70 + "X509_V_ERR_STORE_LOOKUP" + }, +#endif +#if defined(X509_V_ERR_NO_VALID_SCTS) + { + X509_V_ERR_NO_VALID_SCTS, //71 + "X509_V_ERR_NO_VALID_SCTS" + }, +#endif +#if defined(X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION) + { + X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION, //72 + "X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION" + }, +#endif +#if defined(X509_V_ERR_OCSP_VERIFY_NEEDED) + { + X509_V_ERR_OCSP_VERIFY_NEEDED, //73 + "X509_V_ERR_OCSP_VERIFY_NEEDED" + }, +#endif +#if defined(X509_V_ERR_OCSP_VERIFY_FAILED) + { + X509_V_ERR_OCSP_VERIFY_FAILED, //74 + "X509_V_ERR_OCSP_VERIFY_FAILED" + }, +#endif +#if defined(X509_V_ERR_OCSP_CERT_UNKNOWN) + { + X509_V_ERR_OCSP_CERT_UNKNOWN, //75 + "X509_V_ERR_OCSP_CERT_UNKNOWN" + }, +#endif { SSL_ERROR_NONE, "SSL_ERROR_NONE"}, {SSL_ERROR_NONE, NULL} }; @@ -286,6 +412,27 @@ "X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX", "X509_V_ERR_UNSUPPORTED_NAME_SYNTAX", "X509_V_ERR_CRL_PATH_VALIDATION_ERROR", + "X509_V_ERR_PATH_LOOP", + "X509_V_ERR_SUITE_B_INVALID_VERSION", + "X509_V_ERR_SUITE_B_INVALID_ALGORITHM", + "X509_V_ERR_SUITE_B_INVALID_CURVE", + "X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM", + "X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED", + "X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256", + "X509_V_ERR_HOSTNAME_MISMATCH", + "X509_V_ERR_EMAIL_MISMATCH", + "X509_V_ERR_IP_ADDRESS_MISMATCH", + "X509_V_ERR_DANE_NO_MATCH", + "X509_V_ERR_EE_KEY_TOO_SMALL", + "X509_V_ERR_CA_KEY_TOO_SMALL", + "X509_V_ERR_CA_MD_TOO_WEAK", + "X509_V_ERR_INVALID_CALL", + "X509_V_ERR_STORE_LOOKUP", + "X509_V_ERR_NO_VALID_SCTS", + "X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION", + "X509_V_ERR_OCSP_VERIFY_NEEDED", + "X509_V_ERR_OCSP_VERIFY_FAILED", + "X509_V_ERR_OCSP_CERT_UNKNOWN", NULL }; @@ -390,7 +537,7 @@ return false; // not reached } -const char *Ssl::GetErrorName(Security::ErrorCode value) +const char *Ssl::GetErrorName(Security::ErrorCode value, const bool prefixRawCode) { if (TheSslErrors.empty()) loadSslErrorMap(); @@ -399,7 +546,9 @@ if (it != TheSslErrors.end()) return it->second->name; - return NULL; + static char tmpBuffer[128]; + snprintf(tmpBuffer, sizeof(tmpBuffer), "%s%d", prefixRawCode ? "SSL_ERR=" : "", (int)value); + return tmpBuffer; } bool @@ -529,21 +678,14 @@ */ const char *Ssl::ErrorDetail::err_code() const { - static char tmpBuffer[64]; // We can use the GetErrorName but using the detailEntry is faster, // so try it first. - const char *err = detailEntry.name.termedBuf(); + if (const char *err = detailEntry.name.termedBuf()) + return err; // error details not loaded yet or not defined in error_details.txt, // try the GetErrorName... - if (!err) - err = GetErrorName(error_no); - - if (!err) { - snprintf(tmpBuffer, 64, "%d", (int)error_no); - err = tmpBuffer; - } - return err; + return GetErrorName(error_no); } /** diff -u -r -N squid-5.0.2/src/ssl/ErrorDetail.h squid-5.0.3/src/ssl/ErrorDetail.h --- squid-5.0.2/src/ssl/ErrorDetail.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ssl/ErrorDetail.h 2020-06-08 03:33:42.000000000 +1200 @@ -26,8 +26,9 @@ /// The Security::ErrorCode code of the error described by "name". Security::ErrorCode GetErrorCode(const char *name); -/// The string representation of the TLS error "value" -const char *GetErrorName(Security::ErrorCode value); +/// \return string representation of a known TLS error (or a raw error code) +/// \param prefixRawCode whether to prefix raw codes with "SSL_ERR=" +const char *GetErrorName(Security::ErrorCode value, const bool prefixRawCode = false); /// A short description of the TLS error "value" const char *GetErrorDescr(Security::ErrorCode value); diff -u -r -N squid-5.0.2/src/ssl/PeekingPeerConnector.cc squid-5.0.3/src/ssl/PeekingPeerConnector.cc --- squid-5.0.2/src/ssl/PeekingPeerConnector.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ssl/PeekingPeerConnector.cc 2020-06-08 03:33:42.000000000 +1200 @@ -92,8 +92,9 @@ al->ssl.bumpMode = finalAction; if (finalAction == Ssl::bumpTerminate) { - serverConn->close(); + bail(new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scForbidden, request.getRaw(), al)); clientConn->close(); + clientConn = nullptr; } else if (finalAction != Ssl::bumpSplice) { //Allow write, proceed with the connection srvBio->holdWrite(false); @@ -140,11 +141,13 @@ if (!Security::PeerConnector::initialize(serverSession)) return false; + // client connection supplies TLS client details and is also used if we + // need to splice or terminate the client and server connections + if (!Comm::IsConnOpen(clientConn)) + return false; + if (ConnStateData *csd = request->clientConnectionManager.valid()) { - // client connection is required in the case we need to splice - // or terminate client and server connections - assert(clientConn != NULL); SBuf *hostName = NULL; //Enable Status_request TLS extension, required to bump some clients @@ -245,6 +248,10 @@ if (!error) { serverCertificateVerified(); if (splice) { + if (!Comm::IsConnOpen(clientConn)) { + bail(new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw(), al)); + throw TextException("from-client connection gone", Here()); + } switchToTunnel(request.getRaw(), clientConn, serverConn); tunnelInsteadOfNegotiating(); } @@ -276,18 +283,32 @@ BIO *b = SSL_get_rbio(session.get()); Ssl::ServerBio *srvBio = static_cast(BIO_get_data(b)); - // In Peek mode, the ClientHello message sent to the server. If the - // server resuming a previous (spliced) SSL session with the client, - // then probably we are here because local SSL object does not know - // anything about the session being resumed. - // - if (srvBio->bumpMode() == Ssl::bumpPeek && (resumingSession = srvBio->resumingSession())) { - // we currently splice all resumed sessions unconditionally - // if (const bool spliceResumed = true) { - bypassCertValidator(); - checkForPeekAndSpliceMatched(Ssl::bumpSplice); - return; - // } // else fall through to find a matching ssl_bump action (with limited info) + if (srvBio->bumpMode() == Ssl::bumpPeek) { + auto bypassValidator = false; + if (srvBio->encryptedCertificates()) { + // it is pointless to peek at encrypted certificates + // + // we currently splice all sessions with encrypted certificates + // if (const auto spliceEncryptedCertificates = true) { + bypassValidator = true; + // } // else fall through to find a matching ssl_bump action (with limited info) + } else if (srvBio->resumingSession()) { + // In peek mode, the ClientHello message is forwarded to the server. + // If the server is resuming a previous (spliced) SSL session with + // the client, then probably we are here because our local SSL + // object does not know anything about the session being resumed. + // + // we currently splice all resumed sessions + // if (const auto spliceResumed = true) { + bypassValidator = true; + // } // else fall through to find a matching ssl_bump action (with limited info) + } + + if (bypassValidator) { + bypassCertValidator(); + checkForPeekAndSpliceMatched(Ssl::bumpSplice); + return; + } } // If we are in peek-and-splice mode and still we did not write to diff -u -r -N squid-5.0.2/src/ssl/PeekingPeerConnector.h squid-5.0.3/src/ssl/PeekingPeerConnector.h --- squid-5.0.2/src/ssl/PeekingPeerConnector.h 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/ssl/PeekingPeerConnector.h 2020-06-08 03:33:42.000000000 +1200 @@ -30,7 +30,6 @@ Security::PeerConnector(aServerConn, aCallback, alp, timeout), clientConn(aClientConn), splice(false), - resumingSession(false), serverCertificateHandled(false) { request = aRequest; @@ -75,7 +74,6 @@ Comm::ConnectionPointer clientConn; ///< TCP connection to the client AsyncCall::Pointer closeHandler; ///< we call this when the connection closed bool splice; ///< whether we are going to splice or not - bool resumingSession; ///< whether it is an SSL resuming session connection bool serverCertificateHandled; ///< whether handleServerCertificate() succeeded }; diff -u -r -N squid-5.0.2/src/store/id_rewriters/file/storeid_file_rewrite.8 squid-5.0.3/src/store/id_rewriters/file/storeid_file_rewrite.8 --- squid-5.0.2/src/store/id_rewriters/file/storeid_file_rewrite.8 2020-04-20 00:20:16.000000000 +1200 +++ squid-5.0.3/src/store/id_rewriters/file/storeid_file_rewrite.8 2020-06-09 19:09:21.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "STOREID_FILE_REWRITE 8" -.TH STOREID_FILE_REWRITE 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH STOREID_FILE_REWRITE 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-5.0.2/src/tests/stub_libanyp.cc squid-5.0.3/src/tests/stub_libanyp.cc --- squid-5.0.2/src/tests/stub_libanyp.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/tests/stub_libanyp.cc 2020-06-08 03:33:42.000000000 +1200 @@ -18,6 +18,7 @@ void AnyP::Uri::host(const char *) STUB static SBuf nil; const SBuf &AnyP::Uri::path() const STUB_RETVAL(nil) +void AnyP::Uri::addRelativePath(const char *) STUB const SBuf &AnyP::Uri::SlashPath() { static SBuf slash("/"); @@ -33,11 +34,9 @@ void urlInitialize() STUB const char *urlCanonicalFakeHttps(const HttpRequest *) STUB_RETVAL(nullptr) bool urlIsRelative(const char *) STUB_RETVAL(false) -char *urlMakeAbsolute(const HttpRequest *, const char *)STUB_RETVAL(nullptr) char *urlRInternal(const char *, unsigned short, const char *, const char *) STUB_RETVAL(nullptr) char *urlInternal(const char *, const char *) STUB_RETVAL(nullptr) int matchDomainName(const char *, const char *, enum MatchDomainNameFlags) STUB_RETVAL(0) int urlCheckRequest(const HttpRequest *) STUB_RETVAL(0) -char *urlHostname(const char *) STUB_RETVAL(nullptr) void urlExtMethodConfigure() STUB diff -u -r -N squid-5.0.2/src/tests/stub_libsecurity.cc squid-5.0.3/src/tests/stub_libsecurity.cc --- squid-5.0.2/src/tests/stub_libsecurity.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/tests/stub_libsecurity.cc 2020-06-08 03:33:42.000000000 +1200 @@ -28,7 +28,7 @@ std::ostream &Security::operator <<(std::ostream &os, const Security::EncryptorAnswer &) STUB_RETVAL(os) #include "security/Handshake.h" -Security::HandshakeParser::HandshakeParser() STUB +Security::HandshakeParser::HandshakeParser(MessageSource) STUB bool Security::HandshakeParser::parseHello(const SBuf &) STUB_RETVAL(false) #include "security/KeyData.h" @@ -50,14 +50,12 @@ { PeerConnector::PeerConnector(const Comm::ConnectionPointer &, AsyncCall::Pointer &, const AccessLogEntryPointer &, const time_t) : AsyncJob("Security::PeerConnector") {STUB} -PeerConnector::~PeerConnector() {STUB} void PeerConnector::start() STUB bool PeerConnector::doneAll() const STUB_RETVAL(true) void PeerConnector::swanSong() STUB const char *PeerConnector::status() const STUB_RETVAL("") void PeerConnector::commCloseHandler(const CommCloseCbParams &) STUB -void PeerConnector::connectionClosed(const char *) STUB -bool PeerConnector::prepareSocket() STUB_RETVAL(false) +void PeerConnector::commTimeoutHandler(const CommTimeoutCbParams &) STUB bool PeerConnector::initialize(Security::SessionPointer &) STUB_RETVAL(false) void PeerConnector::negotiate() STUB bool PeerConnector::sslFinalized() STUB_RETVAL(false) @@ -67,7 +65,9 @@ void PeerConnector::noteNegotiationError(const int, const int, const int) STUB // virtual Security::ContextPointer getTlsContext() = 0; void PeerConnector::bail(ErrorState *) STUB +void PeerConnector::sendSuccess() STUB void PeerConnector::callBack() STUB +void PeerConnector::disconnect() STUB void PeerConnector::recordNegotiationDetails() STUB } diff -u -r -N squid-5.0.2/src/tests/stub_StatHist.cc squid-5.0.3/src/tests/stub_StatHist.cc --- squid-5.0.2/src/tests/stub_StatHist.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/tests/stub_StatHist.cc 2020-06-08 03:33:42.000000000 +1200 @@ -16,7 +16,7 @@ void StatHist::dump(StoreEntry * sentry, StatHistBinDumper * bd) const STUB void StatHist::enumInit(unsigned int i) STUB_NOP -void StatHist::count(double d) STUB_NOP +void StatHist::count(double) {/* STUB_NOP */} double statHistDeltaMedian(const StatHist & A, const StatHist & B) STUB_RETVAL(0.0) double statHistDeltaPctile(const StatHist & A, const StatHist & B, double pctile) STUB_RETVAL(0.0) void StatHist::logInit(unsigned int i, double d1, double d2) STUB_NOP diff -u -r -N squid-5.0.2/src/tunnel.cc squid-5.0.3/src/tunnel.cc --- squid-5.0.2/src/tunnel.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/tunnel.cc 2020-06-08 03:33:42.000000000 +1200 @@ -111,9 +111,12 @@ /// starts connecting to the next hop, either for the first time or while /// recovering from the previous connect failure void startConnecting(); + void closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason); + + void retryOrBail(); /// called when negotiations with the peer have been successfully completed - void notePeerReadyToShovel(); + void notePeerReadyToShovel(const Comm::ConnectionPointer &); class Connection { @@ -179,7 +182,8 @@ void copyRead(Connection &from, IOCB *completion); /// continue to set up connection to a peer, going async for SSL peers - void connectToPeer(); + void connectToPeer(const Comm::ConnectionPointer &); + void secureConnectionToPeer(const Comm::ConnectionPointer &); /* PeerSelectionInitiator API */ virtual void noteDestination(Comm::ConnectionPointer conn) override; @@ -233,8 +237,15 @@ void usePinned(); - /// callback handler after connection setup (including any encryption) - void connectedToPeer(Security::EncryptorAnswer &answer); + /// callback handler for the Security::PeerConnector encryptor + void noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &); + + /// called after connection setup (including any encryption) + void connectedToPeer(const Comm::ConnectionPointer &); + void establishTunnelThruProxy(const Comm::ConnectionPointer &); + + template + void advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep); /// details of the "last tunneling attempt" failure (if it failed) ErrorState *savedError = nullptr; @@ -680,6 +691,15 @@ } void +TunnelStateData::closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason) +{ + debugs(26, 3, "because " << reason << "; " << conn); + assert(!server.conn); + if (IsConnOpen(conn)) + conn->close(); +} + +void TunnelStateData::Connection::closeIfOpen() { if (Comm::IsConnOpen(conn)) @@ -776,6 +796,11 @@ tunnelStartShoveling(TunnelStateData *tunnelState) { assert(!tunnelState->waitingForConnectExchange); + assert(tunnelState->server.conn); + AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout", + CommTimeoutCbPtrFun(tunnelTimeout, tunnelState)); + commSetConnTimeout(tunnelState->server.conn, Config.Timeout.read, timeoutCall); + *tunnelState->status_ptr = Http::scOkay; if (tunnelState->logTag_ptr) tunnelState->logTag_ptr->update(LOG_TCP_TUNNEL); @@ -838,35 +863,51 @@ waitingForConnectExchange = false; - if (answer.positive()) { + auto sawProblem = false; + + if (!answer.positive()) { + sawProblem = true; + Must(!Comm::IsConnOpen(answer.conn)); + } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) { + sawProblem = true; + closePendingConnection(answer.conn, "conn was closed while waiting for tunnelEstablishmentDone"); + } + + if (!sawProblem) { + assert(answer.positive()); // paranoid // copy any post-200 OK bytes to our buffer preReadServerData = answer.leftovers; - notePeerReadyToShovel(); + notePeerReadyToShovel(answer.conn); return; } - // TODO: Reuse to-peer connections after a CONNECT error response. - - // TODO: We can and, hence, should close now, but tunnelServerClosed() - // cannot yet tell whether ErrorState is still writing an error response. - // server.closeIfOpen(); - if (!clientExpectsConnectResponse()) { // closing the non-HTTP client connection is the best we can do - debugs(50, 3, server.conn << " closing on CONNECT-to-peer error"); - server.closeIfOpen(); + debugs(50, 3, client.conn << " closing on CONNECT-to-peer error"); + client.closeIfOpen(); return; } - ErrorState *error = answer.squidError.get(); - Must(error); - answer.squidError.clear(); // preserve error for errorSendComplete() - sendError(error, "tunneler returns error"); + ErrorState *error = nullptr; + if (answer.positive()) { + error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al); + } else { + error = answer.squidError.get(); + Must(error); + answer.squidError.clear(); // preserve error for errorSendComplete() + } + assert(error); + saveError(error); + retryOrBail(); } void -TunnelStateData::notePeerReadyToShovel() +TunnelStateData::notePeerReadyToShovel(const Comm::ConnectionPointer &conn) { + server.conn = conn; + // Start monitoring for server-side connection problems + comm_add_close_handler(server.conn->fd, tunnelServerClosed, this); + if (!clientExpectsConnectResponse()) tunnelStartShoveling(this); // ssl-bumped connection, be quiet else { @@ -896,17 +937,42 @@ tunnelState->server.conn->close(); } +/// reacts to the current destination failure +void +TunnelStateData::retryOrBail() +{ + // Since no TCP payload has been passed to client or server, we may + // TCP-connect to other destinations (including alternate IPs). + + if (!FwdState::EnoughTimeToReForward(startTime)) + return sendError(savedError, "forwarding timeout"); + + if (!destinations->empty()) + return startConnecting(); + + if (!PeerSelectionInitiator::subscribed) + return sendError(savedError, "tried all destinations"); +} + void TunnelStateData::noteConnection(HappyConnOpener::Answer &answer) { calls.connector = nullptr; connOpener.clear(); - if (const auto error = answer.error.get()) { + ErrorState *error = nullptr; + if ((error = answer.error.get())) { + Must(!Comm::IsConnOpen(answer.conn)); syncHierNote(answer.conn, request->url.host()); + answer.error.clear(); + } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) { + error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al); + closePendingConnection(answer.conn, "conn was closed while waiting for noteConnection"); + } + + if (error) { saveError(error); - answer.error.clear(); // savedError has it now - sendError(savedError, "tried all destinations"); + retryOrBail(); return; } @@ -917,17 +983,12 @@ TunnelStateData::connectDone(const Comm::ConnectionPointer &conn, const char *origin, const bool reused) { Must(Comm::IsConnOpen(conn)); - server.conn = conn; if (reused) ResetMarkingsToServer(request.getRaw(), *conn); // else Comm::ConnOpener already applied proper/current markings - syncHierNote(server.conn, request->url.host()); - - request->hier.resetPeerNotes(conn, origin); - if (al) - al->hier.resetPeerNotes(conn, origin); + syncHierNote(conn, origin); #if USE_DELAY_POOLS /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */ @@ -938,7 +999,6 @@ netdbPingSite(request->url.host()); request->peer_host = conn->getPeer() ? conn->getPeer()->host : nullptr; - comm_add_close_handler(conn->fd, tunnelServerClosed, this); bool toOrigin = false; // same semantics as StateFlags::toOrigin if (const auto * const peer = conn->getPeer()) { @@ -950,14 +1010,10 @@ } if (!toOrigin) - connectToPeer(); + connectToPeer(conn); else { - notePeerReadyToShovel(); + notePeerReadyToShovel(conn); } - - AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout", - CommTimeoutCbPtrFun(tunnelTimeout, this)); - commSetConnTimeout(conn, Config.Timeout.read, timeoutCall); } void @@ -1007,38 +1063,85 @@ } void -TunnelStateData::connectToPeer() +TunnelStateData::connectToPeer(const Comm::ConnectionPointer &conn) { - if (CachePeer *p = server.conn->getPeer()) { - if (p->secure.encryptTransport) { - AsyncCall::Pointer callback = asyncCall(5,4, - "TunnelStateData::ConnectedToPeer", - MyAnswerDialer(&TunnelStateData::connectedToPeer, this)); - auto *connector = new Security::BlindPeerConnector(request, server.conn, callback, al); - AsyncJob::Start(connector); // will call our callback - return; - } + if (const auto p = conn->getPeer()) { + if (p->secure.encryptTransport) + return advanceDestination("secure connection to peer", conn, [this,&conn] { + secureConnectionToPeer(conn); + }); } - Security::EncryptorAnswer nil; - connectedToPeer(nil); + connectedToPeer(conn); } +/// encrypts an established TCP connection to peer void -TunnelStateData::connectedToPeer(Security::EncryptorAnswer &answer) +TunnelStateData::secureConnectionToPeer(const Comm::ConnectionPointer &conn) +{ + AsyncCall::Pointer callback = asyncCall(5,4, "TunnelStateData::noteSecurityPeerConnectorAnswer", + MyAnswerDialer(&TunnelStateData::noteSecurityPeerConnectorAnswer, this)); + const auto connector = new Security::BlindPeerConnector(request, conn, callback, al); + AsyncJob::Start(connector); // will call our callback +} + +/// starts a preparation step for an established connection; retries on failures +template +void +TunnelStateData::advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep) +{ + // TODO: Extract destination-specific handling from TunnelStateData so that + // all the awkward, limited-scope advanceDestination() calls can be replaced + // with a single simple try/catch,retry block. + try { + startStep(); + // now wait for the step callback + } catch (...) { + debugs (26, 2, "exception while trying to " << stepDescription << ": " << CurrentException); + closePendingConnection(conn, "connection preparation exception"); + if (!savedError) + saveError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al)); + retryOrBail(); + } +} + +/// callback handler for the connection encryptor +void +TunnelStateData::noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &answer) { if (ErrorState *error = answer.error.get()) { + Must(!Comm::IsConnOpen(answer.conn)); answer.error.clear(); // sendError() will own the error sendError(error, "TLS peer connection error"); return; } + if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) { + sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al), "connecion gone"); + closePendingConnection(answer.conn, "conn was closed while waiting for noteSecurityPeerConnectorAnswer"); + return; + } + + connectedToPeer(answer.conn); +} + +void +TunnelStateData::connectedToPeer(const Comm::ConnectionPointer &conn) +{ + advanceDestination("establish tunnel through proxy", conn, [this,&conn] { + establishTunnelThruProxy(conn); + }); +} + +void +TunnelStateData::establishTunnelThruProxy(const Comm::ConnectionPointer &conn) +{ assert(!waitingForConnectExchange); AsyncCall::Pointer callback = asyncCall(5,4, "TunnelStateData::tunnelEstablishmentDone", Http::Tunneler::CbDialer(&TunnelStateData::tunnelEstablishmentDone, this)); - const auto tunneler = new Http::Tunneler(server.conn, request, callback, Config.Timeout.lifetime, al); + const auto tunneler = new Http::Tunneler(conn, request, callback, Config.Timeout.lifetime, al); #if USE_DELAY_POOLS tunneler->setDelayId(server.delayId); #endif @@ -1240,6 +1343,9 @@ void switchToTunnel(HttpRequest *request, Comm::ConnectionPointer &clientConn, Comm::ConnectionPointer &srvConn) { + Must(Comm::IsConnOpen(clientConn)); + Must(Comm::IsConnOpen(srvConn)); + debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd); /* Create state structure. */ @@ -1277,10 +1383,6 @@ else request->prepForDirect(); - AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout", - CommTimeoutCbPtrFun(tunnelTimeout, tunnelState)); - commSetConnTimeout(srvConn, Config.Timeout.read, timeoutCall); - // we drain any already buffered from-server data below (rBufData) fd_table[srvConn->fd].useDefaultIo(); diff -u -r -N squid-5.0.2/src/urn.cc squid-5.0.3/src/urn.cc --- squid-5.0.2/src/urn.cc 2020-04-19 22:50:48.000000000 +1200 +++ squid-5.0.3/src/urn.cc 2020-06-08 03:33:42.000000000 +1200 @@ -387,7 +387,6 @@ { char *buf = xstrdup(inbuf); char *token; - char *host; url_entry *list; url_entry *old; int n = 32; @@ -406,24 +405,23 @@ safe_free(old); } - host = urlHostname(token); - - if (NULL == host) + AnyP::Uri uri; + if (!uri.parse(m, SBuf(token)) || !*uri.host()) continue; #if USE_ICMP - list[i].rtt = netdbHostRtt(host); + list[i].rtt = netdbHostRtt(uri.host()); if (0 == list[i].rtt) { - debugs(52, 3, "urnParseReply: Pinging " << host); - netdbPingSite(host); + debugs(52, 3, "Pinging " << uri.host()); + netdbPingSite(uri.host()); } #else list[i].rtt = 0; #endif - list[i].url = xstrdup(token); - list[i].host = xstrdup(host); + list[i].url = xstrdup(uri.absolute().c_str()); + list[i].host = xstrdup(uri.host()); // TODO: Use storeHas() or lock/unlock entry to avoid creating unlocked // ones. list[i].flags.cached = storeGetPublic(list[i].url, m) ? 1 : 0; diff -u -r -N squid-5.0.2/tools/CharacterSet.cc squid-5.0.3/tools/CharacterSet.cc --- squid-5.0.2/tools/CharacterSet.cc 2020-04-20 00:20:18.000000000 +1200 +++ squid-5.0.3/tools/CharacterSet.cc 2020-06-09 19:09:23.000000000 +1200 @@ -51,6 +51,13 @@ } CharacterSet & +CharacterSet::remove(const unsigned char c) +{ + chars_[static_cast(c)] = 0; + return *this; +} + +CharacterSet & CharacterSet::addRange(unsigned char low, unsigned char high) { //manual loop splitting is needed to cover case where high is 255 diff -u -r -N squid-5.0.2/tools/helper-mux/helper-mux.8 squid-5.0.3/tools/helper-mux/helper-mux.8 --- squid-5.0.2/tools/helper-mux/helper-mux.8 2020-04-20 00:20:19.000000000 +1200 +++ squid-5.0.3/tools/helper-mux/helper-mux.8 2020-06-09 19:09:23.000000000 +1200 @@ -1,4 +1,4 @@ -.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35) +.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35) .\" .\" Standard preamble: .\" ======================================================================== @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "HELPER-MUX 8" -.TH HELPER-MUX 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation" +.TH HELPER-MUX 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l