diff -u -r -N squid-5.0.2/acinclude/os-deps.m4 squid-5.0.3/acinclude/os-deps.m4
--- squid-5.0.2/acinclude/os-deps.m4 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/acinclude/os-deps.m4 2020-06-08 03:33:42.000000000 +1200
@@ -925,11 +925,13 @@
## Solaris 10+ backported IPv6 NAT to their IPFilter v4.1 instead of using v5
AC_CHECK_MEMBERS([
struct natlookup.nl_inipaddr.in6,
- struct natlookup.nl_realipaddr.in6
- ],,,[
+ struct natlookup.nl_realipaddr.in6],,,[
#if USE_SOLARIS_IPFILTER_MINOR_T_HACK
#define minor_t fubar
#endif
+#if HAVE_SYS_PARAM_H
+#include
+#endif
#if HAVE_SYS_TYPES_H
#include
#endif
@@ -955,7 +957,11 @@
#elif HAVE_NETINET_IP_FIL_H
#include
#endif
+#if HAVE_IP_NAT_H
#include
+#elif HAVE_NETINET_IP_NAT_H
+#include
+#endif
])
])
diff -u -r -N squid-5.0.2/ChangeLog squid-5.0.3/ChangeLog
--- squid-5.0.2/ChangeLog 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/ChangeLog 2020-06-08 03:33:42.000000000 +1200
@@ -1,3 +1,13 @@
+Changes in squid-5.0.3 (05 Jun 2020):
+
+ - Bug 5046: FreeBSD lacks open(2) O_DSYNC flag
+ - Happy Eyeballs: Do not discard viable reforwarding destinations
+ - Reduced startup time with large rock cache_dirs
+ - Fix the ABA problem with Ipc::Mem::PageStack::pop() in v5.0.1
+ - Fix sending of unknown validation errors to certificate validator
+ - ... and several debug improvements
+ - ... and all fixes from 4.12
+
Changes in squid-5.0.2 (18 Apr 2020):
- Bug 5030: Negative responses are never cached
@@ -9,6 +19,7 @@
- Fix PURGE error responses
- ... and several documentation changes
- ... and some compile fixes
+ - ... and all fixes from 4.11
Changes in squid-5.0.1 (14 Jan 2020):
@@ -56,6 +67,21 @@
- Centralized PagePool/PageStack ID generation
- ... and many documentation changes
- ... and much code cleanup and polishing
+ - ... and all fixes from 4.10
+
+Changes to squid-4.12 (05 Jun 2020):
+
+ - Regression Fix: Revert to slow search for new SMP shm pages
+ - Bug 5045: ext_edirectory_userip_acl is missing include files
+ - Bug 5041: Missing Debug::Extra breaks build on hosts with systemd
+ - Bug 5030: Negative responses are never cached
+ - HTTP: validate Content-Length value prefix
+ - HTTP: add flexible RFC 3986 URI encoder
+ - SslBump: disable OpenSSL TLSv1.3 support for older TLS traffic
+ - Tests: Support passing a custom config.cache to test builds
+ - Fix IPFilter IPv6 detection, especially on NetBSD
+ - Fix stall if transaction overwrites a recently active cache entry
+ - ... and some compile fixes
Changes to squid-4.11 (18 Apr 2020):
diff -u -r -N squid-5.0.2/configure squid-5.0.3/configure
--- squid-5.0.2/configure 2020-04-20 00:08:55.000000000 +1200
+++ squid-5.0.3/configure 2020-06-09 18:58:26.000000000 +1200
@@ -1,7 +1,7 @@
#! /bin/sh
# From configure.ac Revision.
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for Squid Web Proxy 5.0.2.
+# Generated by GNU Autoconf 2.69 for Squid Web Proxy 5.0.3.
#
# Report bugs to .
#
@@ -595,8 +595,8 @@
# Identity of this package.
PACKAGE_NAME='Squid Web Proxy'
PACKAGE_TARNAME='squid'
-PACKAGE_VERSION='5.0.2'
-PACKAGE_STRING='Squid Web Proxy 5.0.2'
+PACKAGE_VERSION='5.0.3'
+PACKAGE_STRING='Squid Web Proxy 5.0.3'
PACKAGE_BUGREPORT='http://bugs.squid-cache.org/'
PACKAGE_URL=''
@@ -1662,7 +1662,7 @@
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures Squid Web Proxy 5.0.2 to adapt to many kinds of systems.
+\`configure' configures Squid Web Proxy 5.0.3 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1733,7 +1733,7 @@
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of Squid Web Proxy 5.0.2:";;
+ short | recursive ) echo "Configuration of Squid Web Proxy 5.0.3:";;
esac
cat <<\_ACEOF
@@ -2166,7 +2166,7 @@
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-Squid Web Proxy configure 5.0.2
+Squid Web Proxy configure 5.0.3
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@@ -3270,7 +3270,7 @@
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by Squid Web Proxy $as_me 5.0.2, which was
+It was created by Squid Web Proxy $as_me 5.0.3, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@@ -4137,7 +4137,7 @@
# Define the identity of the package.
PACKAGE='squid'
- VERSION='5.0.2'
+ VERSION='5.0.3'
cat >>confdefs.h <<_ACEOF
@@ -21097,8 +21097,9 @@
$as_echo "$as_me: With dl" >&6;}
fi
+
+
## check for atomics library before anything that might need it
-# AC_SEARCH_LIBS pollutes LIBS
# save state, key is LIBATOMIC
LIBATOMIC_CFLAGS="${CFLAGS}"
@@ -21115,63 +21116,76 @@
eval "${squid_util_var_tosave2}=\"${squid_util_var_tosave}\""
done
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing __atomic_load_8" >&5
-$as_echo_n "checking for library containing __atomic_load_8... " >&6; }
-if ${ac_cv_search___atomic_load_8+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- ac_func_search_save_LIBS=$LIBS
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether linking without -latomic works" >&5
+$as_echo_n "checking whether linking without -latomic works... " >&6; }
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char __atomic_load_8 ();
-int
-main ()
-{
-return __atomic_load_8 ();
- ;
- return 0;
-}
+
+#include
+#include
+ int
+ main(int, char **) {
+ return std::atomic{}.is_lock_free() ? 0 : 1;
+ }
+
_ACEOF
-for ac_lib in '' atomic; do
- if test -z "$ac_lib"; then
- ac_res="none required"
- else
- ac_res=-l$ac_lib
- LIBS="-l$ac_lib $ac_func_search_save_LIBS"
- fi
- if ac_fn_cxx_try_link "$LINENO"; then :
- ac_cv_search___atomic_load_8=$ac_res
+if ac_fn_cxx_try_link "$LINENO"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ libatomic_checker_result="yes"
+else
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ libatomic_checker_result="no"
+
fi
rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext
- if ${ac_cv_search___atomic_load_8+:} false; then :
- break
-fi
-done
-if ${ac_cv_search___atomic_load_8+:} false; then :
+ conftest$ac_exeext conftest.$ac_ext
+if test "x$libatomic_checker_result" = "xno"; then :
+
+ LIBS="$LIBS -latomic"
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether linking with -latomic works" >&5
+$as_echo_n "checking whether linking with -latomic works... " >&6; }
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+
+#include
+#include
+ int
+ main(int, char **) {
+ return std::atomic{}.is_lock_free() ? 0 : 1;
+ }
+_ACEOF
+if ac_fn_cxx_try_link "$LINENO"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ libatomic_checker_result="yes"
else
- ac_cv_search___atomic_load_8=no
-fi
-rm conftest.$ac_ext
-LIBS=$ac_func_search_save_LIBS
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search___atomic_load_8" >&5
-$as_echo "$ac_cv_search___atomic_load_8" >&6; }
-ac_res=$ac_cv_search___atomic_load_8
-if test "$ac_res" != no; then :
- test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
- test "$ac_res" = "none required" || ATOMICLIB=$ac_res
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ libatomic_checker_result="no"
+
fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ if test "x$libatomic_checker_result" = "xyes"; then :
+
+ ATOMICLIB="-latomic"
+else
+
+ as_fn_error $? "Required library libatomic not found." "$LINENO" 5
+fi
+fi
# rollback state, key is LIBATOMIC
CFLAGS="${LIBATOMIC_CFLAGS}"
@@ -42692,6 +42706,9 @@
#if USE_SOLARIS_IPFILTER_MINOR_T_HACK
#define minor_t fubar
#endif
+#if HAVE_SYS_PARAM_H
+#include
+#endif
#if HAVE_SYS_TYPES_H
#include
#endif
@@ -42717,7 +42734,11 @@
#elif HAVE_NETINET_IP_FIL_H
#include
#endif
+#if HAVE_IP_NAT_H
#include
+#elif HAVE_NETINET_IP_NAT_H
+#include
+#endif
"
if test "x$ac_cv_member_struct_natlookup_nl_inipaddr_in6" = xyes; then :
@@ -42728,11 +42749,13 @@
fi
-ac_fn_cxx_check_member "$LINENO" "struct natlookup" "nl_realipaddr.in6"
- "ac_cv_member_struct_natlookup_nl_realipaddr_in6___" "
+ac_fn_cxx_check_member "$LINENO" "struct natlookup" "nl_realipaddr.in6" "ac_cv_member_struct_natlookup_nl_realipaddr_in6" "
#if USE_SOLARIS_IPFILTER_MINOR_T_HACK
#define minor_t fubar
#endif
+#if HAVE_SYS_PARAM_H
+#include
+#endif
#if HAVE_SYS_TYPES_H
#include
#endif
@@ -42758,13 +42781,17 @@
#elif HAVE_NETINET_IP_FIL_H
#include
#endif
+#if HAVE_IP_NAT_H
#include
+#elif HAVE_NETINET_IP_NAT_H
+#include
+#endif
"
-if test "x$ac_cv_member_struct_natlookup_nl_realipaddr_in6___" = xyes; then :
+if test "x$ac_cv_member_struct_natlookup_nl_realipaddr_in6" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6___ 1
+#define HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6 1
_ACEOF
@@ -44931,7 +44958,7 @@
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by Squid Web Proxy $as_me 5.0.2, which was
+This file was extended by Squid Web Proxy $as_me 5.0.3, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -44997,7 +45024,7 @@
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-Squid Web Proxy config.status 5.0.2
+Squid Web Proxy config.status 5.0.3
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
diff -u -r -N squid-5.0.2/configure.ac squid-5.0.3/configure.ac
--- squid-5.0.2/configure.ac 2020-04-20 00:08:55.000000000 +1200
+++ squid-5.0.3/configure.ac 2020-06-09 18:58:26.000000000 +1200
@@ -5,7 +5,7 @@
## Please see the COPYING and CONTRIBUTORS files for details.
##
-AC_INIT([Squid Web Proxy],[5.0.2],[http://bugs.squid-cache.org/],[squid])
+AC_INIT([Squid Web Proxy],[5.0.3],[http://bugs.squid-cache.org/],[squid])
AC_PREREQ(2.61)
AC_CONFIG_HEADERS([include/autoconf.h])
AC_CONFIG_AUX_DIR(cfgaux)
@@ -440,11 +440,33 @@
AC_MSG_NOTICE([With dl])
fi
+AC_DEFUN([LIBATOMIC_CHECKER],[
+ AC_MSG_CHECKING(whether linking $1 -latomic works)
+ AC_LINK_IFELSE([
+ AC_LANG_SOURCE([[
+#include
+#include
+ int
+ main(int, char **) {
+ return std::atomic{}.is_lock_free() ? 0 : 1;
+ }
+ ]])],[
+ AC_MSG_RESULT(yes)
+ libatomic_checker_result="yes"],[
+ AC_MSG_RESULT(no)
+ libatomic_checker_result="no"
+])])
+
## check for atomics library before anything that might need it
-# AC_SEARCH_LIBS pollutes LIBS
SQUID_STATE_SAVE(LIBATOMIC)
-AC_SEARCH_LIBS([__atomic_load_8],[atomic],[
- test "$ac_res" = "none required" || ATOMICLIB=$ac_res],[])
+LIBATOMIC_CHECKER(without)
+AS_IF([test "x$libatomic_checker_result" = "xno"],[
+ LIBS="$LIBS -latomic"
+ LIBATOMIC_CHECKER(with)
+ AS_IF([test "x$libatomic_checker_result" = "xyes"],[
+ ATOMICLIB="-latomic"],[
+ AC_MSG_ERROR([Required library libatomic not found.])
+])])
SQUID_STATE_ROLLBACK(LIBATOMIC)
AC_SUBST(ATOMICLIB)
diff -u -r -N squid-5.0.2/CONTRIBUTORS squid-5.0.3/CONTRIBUTORS
--- squid-5.0.2/CONTRIBUTORS 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/CONTRIBUTORS 2020-06-08 03:33:42.000000000 +1200
@@ -25,6 +25,7 @@
Alex Wu
Alin Nastac
Alter
+ Amit Klein
Amos Jeffries
Amos Jeffries
Amos Jeffries
diff -u -r -N squid-5.0.2/doc/release-notes/release-5.html squid-5.0.3/doc/release-notes/release-5.html
--- squid-5.0.2/doc/release-notes/release-5.html 2020-04-20 00:20:13.000000000 +1200
+++ squid-5.0.3/doc/release-notes/release-5.html 2020-06-09 19:09:17.000000000 +1200
@@ -1,12 +1,12 @@
-
+
- Squid 5.0.2 release notes
+ Squid 5.0.3 release notes
-Squid 5.0.2 release notes
+Squid 5.0.3 release notes
Squid Developers
@@ -61,7 +61,7 @@
-The Squid Team are pleased to announce the release of Squid-5.0.2 for testing.
+The Squid Team are pleased to announce the release of Squid-5.0.3 for testing.
This new release is available for download from
http://www.squid-cache.org/Versions/v5/ or the
mirrors.
@@ -326,6 +326,10 @@
New code %A to display Squid listening IP address the client
TCP connection was connected to.
+http_port
+New worker-queues option to have TCP stack maintain dedicated
+listening queue for each worker in SMP.
+
logformat
New ssl::<cert macro code to display received server X.509
certificate in PEM format.
@@ -402,6 +406,10 @@
--disable-optimizations
No longer implies --disable-inline option (which is removed).
+--enable-external-acl-helpers
+New helper type kerberos_sid_group to match group=
+annotations AD Domain group SID.
+
4.3 Removed options
diff -u -r -N squid-5.0.2/include/autoconf.h.in squid-5.0.3/include/autoconf.h.in
--- squid-5.0.2/include/autoconf.h.in 2020-04-20 00:08:47.000000000 +1200
+++ squid-5.0.3/include/autoconf.h.in 2020-06-09 18:58:17.000000000 +1200
@@ -1041,8 +1041,8 @@
/* Define to 1 if `nl_inipaddr.in6' is a member of `struct natlookup'. */
#undef HAVE_STRUCT_NATLOOKUP_NL_INIPADDR_IN6
-/* Define to 1 if `nl_realipaddr.in6' is a member of `struct natlookup '. */
-#undef HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6___
+/* Define to 1 if `nl_realipaddr.in6' is a member of `struct natlookup'. */
+#undef HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6
/* The system provides struct rusage */
#undef HAVE_STRUCT_RUSAGE
diff -u -r -N squid-5.0.2/include/version.h squid-5.0.3/include/version.h
--- squid-5.0.2/include/version.h 2020-04-20 00:08:55.000000000 +1200
+++ squid-5.0.3/include/version.h 2020-06-09 18:58:26.000000000 +1200
@@ -7,7 +7,7 @@
*/
#ifndef SQUID_RELEASE_TIME
-#define SQUID_RELEASE_TIME 1587298122
+#define SQUID_RELEASE_TIME 1591685891
#endif
/*
diff -u -r -N squid-5.0.2/RELEASENOTES.html squid-5.0.3/RELEASENOTES.html
--- squid-5.0.2/RELEASENOTES.html 2020-04-20 00:20:13.000000000 +1200
+++ squid-5.0.3/RELEASENOTES.html 2020-06-09 19:09:17.000000000 +1200
@@ -1,12 +1,12 @@
-
+
- Squid 5.0.2 release notes
+ Squid 5.0.3 release notes
-Squid 5.0.2 release notes
+Squid 5.0.3 release notes
Squid Developers
@@ -61,7 +61,7 @@
-
The Squid Team are pleased to announce the release of Squid-5.0.2 for testing.
+The Squid Team are pleased to announce the release of Squid-5.0.3 for testing.
This new release is available for download from
http://www.squid-cache.org/Versions/v5/ or the
mirrors.
@@ -326,6 +326,10 @@
New code %A to display Squid listening IP address the client
TCP connection was connected to.
+http_port
+New worker-queues option to have TCP stack maintain dedicated
+listening queue for each worker in SMP.
+
logformat
New ssl::<cert macro code to display received server X.509
certificate in PEM format.
@@ -402,6 +406,10 @@
--disable-optimizations
No longer implies --disable-inline option (which is removed).
+--enable-external-acl-helpers
+New helper type kerberos_sid_group to match group=
+annotations AD Domain group SID.
+
4.3 Removed options
diff -u -r -N squid-5.0.2/src/acl/external/delayer/ext_delayer_acl.8 squid-5.0.3/src/acl/external/delayer/ext_delayer_acl.8
--- squid-5.0.2/src/acl/external/delayer/ext_delayer_acl.8 2020-04-20 00:20:15.000000000 +1200
+++ squid-5.0.3/src/acl/external/delayer/ext_delayer_acl.8 2020-06-09 19:09:20.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "EXT_DELAYER_ACL 8"
-.TH EXT_DELAYER_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH EXT_DELAYER_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc squid-5.0.3/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc
--- squid-5.0.2/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc 2020-06-08 03:33:42.000000000 +1200
@@ -69,6 +69,12 @@
#ifdef HAVE_NETDB_H
#include
#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include
+#endif
#ifdef HELPER_INPUT_BUFFER
#define EDUI_MAXLEN HELPER_INPUT_BUFFER
diff -u -r -N squid-5.0.2/src/acl/external/kerberos_ldap_group/support_krb5.cc squid-5.0.3/src/acl/external/kerberos_ldap_group/support_krb5.cc
--- squid-5.0.2/src/acl/external/kerberos_ldap_group/support_krb5.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/acl/external/kerberos_ldap_group/support_krb5.cc 2020-06-08 03:33:42.000000000 +1200
@@ -467,10 +467,15 @@
}
// overwrite limitation of enctypes
+#if USE_HEIMDAL_KRB5
+ creds->session.keytype = 0;
+ if (creds->session.keyvalue.length > 0)
+ krb5_free_keyblock_contents(kparam.context, &creds->session);
+#else
creds->keyblock.enctype = 0;
if (creds->keyblock.contents)
krb5_free_keyblock_contents(kparam.context, &creds->keyblock);
-
+#endif
code = krb5_get_credentials(kparam.context, 0, kparam.cc[ccindex], creds, &tgt_creds);
if (code) {
k5_error("Error while getting tgt", code);
diff -u -r -N squid-5.0.2/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 squid-5.0.3/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8
--- squid-5.0.2/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 2020-04-20 00:20:15.000000000 +1200
+++ squid-5.0.3/src/acl/external/kerberos_sid_group/ext_kerberos_sid_group_acl.8 2020-06-09 19:09:20.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "EXT_KERBEROS_SID_GROUP_ACL 8"
-.TH EXT_KERBEROS_SID_GROUP_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH EXT_KERBEROS_SID_GROUP_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/acl/external/session/ext_session_acl.cc squid-5.0.3/src/acl/external/session/ext_session_acl.cc
--- squid-5.0.2/src/acl/external/session/ext_session_acl.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/acl/external/session/ext_session_acl.cc 2020-06-08 03:33:42.000000000 +1200
@@ -137,6 +137,10 @@
}
}
#elif USE_TRIVIALDB
+#if _SQUID_FREEBSD_ && !defined(O_DSYNC)
+ // FreeBSD lacks O_DSYNC, O_SYNC is closest to correct behaviour
+#define O_DSYNC O_SYNC
+#endif
db = tdb_open(db_path, 0, TDB_CLEAR_IF_FIRST, O_CREAT|O_DSYNC, 0666);
#endif
if (!db) {
diff -u -r -N squid-5.0.2/src/acl/external/SQL_session/ext_sql_session_acl.8 squid-5.0.3/src/acl/external/SQL_session/ext_sql_session_acl.8
--- squid-5.0.2/src/acl/external/SQL_session/ext_sql_session_acl.8 2020-04-20 00:20:16.000000000 +1200
+++ squid-5.0.3/src/acl/external/SQL_session/ext_sql_session_acl.8 2020-06-09 19:09:20.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "EXT_SQL_SESSION_ACL 8"
-.TH EXT_SQL_SESSION_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH EXT_SQL_SESSION_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 squid-5.0.3/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8
--- squid-5.0.2/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2020-04-20 00:20:16.000000000 +1200
+++ squid-5.0.3/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2020-06-09 19:09:20.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "EXT_WBINFO_GROUP_ACL 8"
-.TH EXT_WBINFO_GROUP_ACL 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH EXT_WBINFO_GROUP_ACL 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/anyp/ProtocolVersion.h squid-5.0.3/src/anyp/ProtocolVersion.h
--- squid-5.0.2/src/anyp/ProtocolVersion.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/anyp/ProtocolVersion.h 2020-06-08 03:33:42.000000000 +1200
@@ -40,6 +40,9 @@
unsigned int major; ///< major version number
unsigned int minor; ///< minor version number
+ /// whether the version is "known" (e.g., has been parsed or explicitly set)
+ explicit operator bool() const { return protocol != PROTO_NONE; }
+
bool operator==(const ProtocolVersion& that) const {
if (this->protocol != that.protocol)
return false;
diff -u -r -N squid-5.0.2/src/anyp/Uri.cc squid-5.0.3/src/anyp/Uri.cc
--- squid-5.0.2/src/anyp/Uri.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/anyp/Uri.cc 2020-06-08 03:33:42.000000000 +1200
@@ -30,6 +30,56 @@
"[:]"
;
+/// Characters which are valid within a URI userinfo section
+static const CharacterSet &
+UserInfoChars()
+{
+ /*
+ * RFC 3986 section 3.2.1
+ *
+ * userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
+ * unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
+ * pct-encoded = "%" HEXDIG HEXDIG
+ * sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
+ */
+ static const auto userInfoValid = CharacterSet("userinfo", ":-._~%!$&'()*+,;=") +
+ CharacterSet::ALPHA +
+ CharacterSet::DIGIT;
+ return userInfoValid;
+}
+
+/**
+ * Governed by RFC 3986 section 2.1
+ */
+SBuf
+AnyP::Uri::Encode(const SBuf &buf, const CharacterSet &ignore)
+{
+ if (buf.isEmpty())
+ return buf;
+
+ Parser::Tokenizer tk(buf);
+ SBuf goodSection;
+ // optimization for the arguably common "no encoding necessary" case
+ if (tk.prefix(goodSection, ignore) && tk.atEnd())
+ return buf;
+
+ SBuf output;
+ output.reserveSpace(buf.length() * 3); // worst case: encode all chars
+ output.append(goodSection); // may be empty
+
+ while (!tk.atEnd()) {
+ // TODO: Add Tokenizer::parseOne(void).
+ const auto ch = tk.remaining()[0];
+ output.appendf("%%%02X", static_cast(ch)); // TODO: Optimize using a table
+ (void)tk.skip(ch);
+
+ if (tk.prefix(goodSection, ignore))
+ output.append(goodSection);
+ }
+
+ return output;
+}
+
const SBuf &
AnyP::Uri::Asterisk()
{
@@ -557,7 +607,10 @@
getScheme() == AnyP::PROTO_UNKNOWN;
if (allowUserInfo && !userInfo().isEmpty()) {
- absolute_.append(userInfo());
+ static const CharacterSet uiChars = CharacterSet(UserInfoChars())
+ .remove('%')
+ .rename("userinfo-reserved");
+ absolute_.append(Encode(userInfo(), uiChars));
absolute_.append("@", 1);
}
absolute_.append(authority());
@@ -565,7 +618,7 @@
absolute_.append(host());
absolute_.append(":", 1);
}
- absolute_.append(path());
+ absolute_.append(path()); // TODO: Encode each URI subcomponent in path_ as needed.
}
return absolute_;
@@ -619,102 +672,76 @@
return request->canonicalCleanUrl();
}
-/*
- * Test if a URL is relative.
+/**
+ * Test if a URL is a relative reference.
+ *
+ * Governed by RFC 3986 section 4.2
+ *
+ * relative-ref = relative-part [ "?" query ] [ "#" fragment ]
*
- * RFC 2396, Section 5 (Page 17) implies that in a relative URL, a '/' will
- * appear before a ':'.
+ * relative-part = "//" authority path-abempty
+ * / path-absolute
+ * / path-noscheme
+ * / path-empty
*/
bool
urlIsRelative(const char *url)
{
- const char *p;
+ if (!url)
+ return false; // no URL
- if (url == NULL) {
- return (false);
- }
- if (*url == '\0') {
- return (false);
- }
+ /*
+ * RFC 3986 section 5.2.3
+ *
+ * path = path-abempty ; begins with "/" or is empty
+ * / path-absolute ; begins with "/" but not "//"
+ * / path-noscheme ; begins with a non-colon segment
+ * / path-rootless ; begins with a segment
+ * / path-empty ; zero characters
+ */
- for (p = url; *p != '\0' && *p != ':' && *p != '/'; ++p);
+ if (*url == '\0')
+ return true; // path-empty
- if (*p == ':') {
- return (false);
+ if (*url == '/') {
+ // RFC 3986 section 5.2.3
+ // path-absolute ; begins with "/" but not "//"
+ if (url[1] == '/')
+ return true; // network-path reference, aka. 'scheme-relative URI'
+ else
+ return true; // path-absolute, aka 'absolute-path reference'
}
- return (true);
-}
-
-/*
- * Convert a relative URL to an absolute URL using the context of a given
- * request.
- *
- * It is assumed that you have already ensured that the URL is relative.
- *
- * If NULL is returned it is an indication that the method in use in the
- * request does not distinguish between relative and absolute and you should
- * use the url unchanged.
- *
- * If non-NULL is returned, it is up to the caller to free the resulting
- * memory using safe_free().
- */
-char *
-urlMakeAbsolute(const HttpRequest * req, const char *relUrl)
-{
- if (req->method.id() == Http::METHOD_CONNECT) {
- return (NULL);
+ for (const auto *p = url; *p != '\0' && *p != '/' && *p != '?' && *p != '#'; ++p) {
+ if (*p == ':')
+ return false; // colon is forbidden in first segment
}
- char *urlbuf = (char *)xmalloc(MAX_URL * sizeof(char));
+ return true; // path-noscheme, path-abempty, path-rootless
+}
- if (req->url.getScheme() == AnyP::PROTO_URN) {
- // XXX: this is what the original code did, but it seems to break the
- // intended behaviour of this function. It returns the stored URN path,
- // not converting the given one into a URN...
- snprintf(urlbuf, MAX_URL, SQUIDSBUFPH, SQUIDSBUFPRINT(req->url.absolute()));
- return (urlbuf);
- }
+void
+AnyP::Uri::addRelativePath(const char *relUrl)
+{
+ // URN cannot be merged
+ if (getScheme() == AnyP::PROTO_URN)
+ return;
- SBuf authorityForm = req->url.authority(); // host[:port]
- const SBuf &scheme = req->url.getScheme().image();
- size_t urllen = snprintf(urlbuf, MAX_URL, SQUIDSBUFPH "://" SQUIDSBUFPH "%s" SQUIDSBUFPH,
- SQUIDSBUFPRINT(scheme),
- SQUIDSBUFPRINT(req->url.userInfo()),
- !req->url.userInfo().isEmpty() ? "@" : "",
- SQUIDSBUFPRINT(authorityForm));
+ // TODO: Handle . and .. segment normalization
- // if the first char is '/' assume its a relative path
- // XXX: this breaks on scheme-relative URLs,
- // but we should not see those outside ESI, and rarely there.
- // XXX: also breaks on any URL containing a '/' in the query-string portion
- if (relUrl[0] == '/') {
- xstrncpy(&urlbuf[urllen], relUrl, MAX_URL - urllen - 1);
+ const auto lastSlashPos = path_.rfind('/');
+ // TODO: To optimize and simplify, add and use SBuf::replace().
+ const auto relUrlLength = strlen(relUrl);
+ if (lastSlashPos == SBuf::npos) {
+ // start replacing the whole path
+ path_.reserveCapacity(1 + relUrlLength);
+ path_.assign("/", 1);
} else {
- SBuf path = req->url.path();
- SBuf::size_type lastSlashPos = path.rfind('/');
-
- if (lastSlashPos == SBuf::npos) {
- // replace the whole path with the given bit(s)
- urlbuf[urllen] = '/';
- ++urllen;
- xstrncpy(&urlbuf[urllen], relUrl, MAX_URL - urllen - 1);
- } else {
- // replace only the last (file?) segment with the given bit(s)
- ++lastSlashPos;
- if (lastSlashPos > MAX_URL - urllen - 1) {
- // XXX: crops bits in the middle of the combined URL.
- lastSlashPos = MAX_URL - urllen - 1;
- }
- SBufToCstring(&urlbuf[urllen], path.substr(0,lastSlashPos));
- urllen += lastSlashPos;
- if (urllen + 1 < MAX_URL) {
- xstrncpy(&urlbuf[urllen], relUrl, MAX_URL - urllen - 1);
- }
- }
+ // start replacing just the last segment
+ path_.reserveCapacity(lastSlashPos + 1 + relUrlLength);
+ path_.chop(0, lastSlashPos+1);
}
-
- return (urlbuf);
+ path_.append(relUrl, relUrlLength);
}
int
@@ -890,105 +917,6 @@
return rc;
}
-/*
- * Quick-n-dirty host extraction from a URL. Steps:
- * Look for a colon
- * Skip any '/' after the colon
- * Copy the next SQUID_MAXHOSTNAMELEN bytes to host[]
- * Look for an ending '/' or ':' and terminate
- * Look for login info preceeded by '@'
- */
-
-class URLHostName
-{
-
-public:
- char * extract(char const *url);
-
-private:
- static char Host [SQUIDHOSTNAMELEN];
- void init(char const *);
- void findHostStart();
- void trimTrailingChars();
- void trimAuth();
- char const *hostStart;
- char const *url;
-};
-
-char *
-urlHostname(const char *url)
-{
- return URLHostName().extract(url);
-}
-
-char URLHostName::Host[SQUIDHOSTNAMELEN];
-
-void
-URLHostName::init(char const *aUrl)
-{
- Host[0] = '\0';
- url = aUrl;
-}
-
-void
-URLHostName::findHostStart()
-{
- if (NULL == (hostStart = strchr(url, ':')))
- return;
-
- ++hostStart;
-
- while (*hostStart != '\0' && *hostStart == '/')
- ++hostStart;
-
- if (*hostStart == ']')
- ++hostStart;
-}
-
-void
-URLHostName::trimTrailingChars()
-{
- char *t;
-
- if ((t = strchr(Host, '/')))
- *t = '\0';
-
- if ((t = strrchr(Host, ':')))
- *t = '\0';
-
- if ((t = strchr(Host, ']')))
- *t = '\0';
-}
-
-void
-URLHostName::trimAuth()
-{
- char *t;
-
- if ((t = strrchr(Host, '@'))) {
- ++t;
- memmove(Host, t, strlen(t) + 1);
- }
-}
-
-char *
-URLHostName::extract(char const *aUrl)
-{
- init(aUrl);
- findHostStart();
-
- if (hostStart == NULL)
- return NULL;
-
- xstrncpy(Host, hostStart, SQUIDHOSTNAMELEN);
-
- trimTrailingChars();
-
- trimAuth();
-
- return Host;
-}
-
AnyP::Uri::Uri(AnyP::UriScheme const &aScheme) :
scheme_(aScheme),
hostIsNumeric_(false),
diff -u -r -N squid-5.0.2/src/anyp/Uri.h squid-5.0.3/src/anyp/Uri.h
--- squid-5.0.2/src/anyp/Uri.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/anyp/Uri.h 2020-06-08 03:33:42.000000000 +1200
@@ -77,6 +77,8 @@
}
void userInfo(const SBuf &s) {userInfo_=s; touch();}
+ /// \returns raw userinfo subcomponent (or an empty string)
+ /// the caller is responsible for caller-specific encoding
const SBuf &userInfo() const {return userInfo_;}
void host(const char *src);
@@ -98,12 +100,27 @@
void path(const SBuf &p) {path_=p; touch();}
const SBuf &path() const;
+ /**
+ * Merge a relative-path URL into the existing URI details.
+ * Implements RFC 3986 section 5.2.3
+ *
+ * The caller must ensure relUrl is a valid relative-path.
+ *
+ * NP: absolute-path are also accepted, but path() method
+ * should be used instead when possible.
+ */
+ void addRelativePath(const char *relUrl);
+
/// the static '/' default URL-path
static const SBuf &SlashPath();
/// the static '*' pseudo-URI
static const SBuf &Asterisk();
+ /// %-encode characters in a buffer which do not conform to
+ /// the provided set of expected characters.
+ static SBuf Encode(const SBuf &, const CharacterSet &expected);
+
/**
* The authority-form URI for currently stored values.
*
@@ -199,7 +216,6 @@
char *urlCanonicalCleanWithoutRequest(const SBuf &url, const HttpRequestMethod &, const AnyP::UriScheme &);
const char *urlCanonicalFakeHttps(const HttpRequest * request);
bool urlIsRelative(const char *);
-char *urlMakeAbsolute(const HttpRequest *, const char *);
char *urlRInternal(const char *host, unsigned short port, const char *dir, const char *name);
char *urlInternal(const char *dir, const char *name);
bool urlAppendDomain(char *host); ///< apply append_domain config to the given hostname
@@ -245,7 +261,6 @@
*/
int matchDomainName(const char *host, const char *domain, MatchDomainNameFlags flags = mdnNone);
int urlCheckRequest(const HttpRequest *);
-char *urlHostname(const char *url);
void urlExtMethodConfigure(void);
#endif /* SQUID_SRC_ANYP_URI_H */
diff -u -r -N squid-5.0.2/src/auth/basic/DB/basic_db_auth.8 squid-5.0.3/src/auth/basic/DB/basic_db_auth.8
--- squid-5.0.2/src/auth/basic/DB/basic_db_auth.8 2020-04-20 00:20:16.000000000 +1200
+++ squid-5.0.3/src/auth/basic/DB/basic_db_auth.8 2020-06-09 19:09:21.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "BASIC_DB_AUTH 8"
-.TH BASIC_DB_AUTH 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH BASIC_DB_AUTH 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/auth/basic/POP3/basic_pop3_auth.8 squid-5.0.3/src/auth/basic/POP3/basic_pop3_auth.8
--- squid-5.0.2/src/auth/basic/POP3/basic_pop3_auth.8 2020-04-20 00:20:17.000000000 +1200
+++ squid-5.0.3/src/auth/basic/POP3/basic_pop3_auth.8 2020-06-09 19:09:21.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "BASIC_POP3_AUTH 8"
-.TH BASIC_POP3_AUTH 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH BASIC_POP3_AUTH 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/base/CharacterSet.cc squid-5.0.3/src/base/CharacterSet.cc
--- squid-5.0.2/src/base/CharacterSet.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/base/CharacterSet.cc 2020-06-08 03:33:42.000000000 +1200
@@ -51,6 +51,13 @@
}
CharacterSet &
+CharacterSet::remove(const unsigned char c)
+{
+ chars_[static_cast(c)] = 0;
+ return *this;
+}
+
+CharacterSet &
CharacterSet::addRange(unsigned char low, unsigned char high)
{
//manual loop splitting is needed to cover case where high is 255
diff -u -r -N squid-5.0.2/src/base/CharacterSet.h squid-5.0.3/src/base/CharacterSet.h
--- squid-5.0.2/src/base/CharacterSet.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/base/CharacterSet.h 2020-06-08 03:33:42.000000000 +1200
@@ -41,6 +41,9 @@
/// add a given character to the character set
CharacterSet & add(const unsigned char c);
+ /// remove a given character from the character set
+ CharacterSet & remove(const unsigned char c);
+
/// add a list of character ranges, expressed as pairs [low,high], including both ends
CharacterSet & addRange(unsigned char low, unsigned char high);
diff -u -r -N squid-5.0.2/src/ClientInfo.h squid-5.0.3/src/ClientInfo.h
--- squid-5.0.2/src/ClientInfo.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ClientInfo.h 2020-06-08 03:33:42.000000000 +1200
@@ -78,6 +78,8 @@
unsigned int quotaPeekReserv() const; ///< returns the next reserv. to pop
void quotaDequeue(); ///< pops queue head from queue
void kickQuotaQueue(); ///< schedule commHandleWriteHelper call
+ /// either selects the head descriptor for writing or calls quotaDequeue()
+ void writeOrDequeue();
/* BandwidthBucket API */
virtual int quota() override; ///< allocate quota for a just dequeued client
diff -u -r -N squid-5.0.2/src/clients/Client.cc squid-5.0.3/src/clients/Client.cc
--- squid-5.0.2/src/clients/Client.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/clients/Client.cc 2020-06-08 03:33:42.000000000 +1200
@@ -426,7 +426,7 @@
static bool
sameUrlHosts(const char *url1, const char *url2)
{
- // XXX: Want urlHostname() here, but it uses static storage and copying
+ // XXX: Want AnyP::Uri::parse() here, but it uses static storage and copying
const char *host1 = strchr(url1, ':');
const char *host2 = strchr(url2, ':');
@@ -455,33 +455,40 @@
static void
purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
{
- const char *hdrUrl, *absUrl;
-
- absUrl = NULL;
- hdrUrl = rep->header.getStr(hdr);
- if (hdrUrl == NULL) {
+ const auto hdrUrl = rep->header.getStr(hdr);
+ if (!hdrUrl)
return;
- }
/*
* If the URL is relative, make it absolute so we can find it.
* If it's absolute, make sure the host parts match to avoid DOS attacks
* as per RFC 2616 13.10.
*/
+ SBuf absUrlMaker;
+ const char *absUrl = nullptr;
if (urlIsRelative(hdrUrl)) {
- absUrl = urlMakeAbsolute(req, hdrUrl);
- if (absUrl != NULL) {
- hdrUrl = absUrl;
+ if (req->method.id() == Http::METHOD_CONNECT)
+ absUrl = hdrUrl; // TODO: merge authority-uri and hdrUrl
+ else if (req->url.getScheme() == AnyP::PROTO_URN)
+ absUrl = req->url.absolute().c_str();
+ else {
+ AnyP::Uri tmpUrl = req->url;
+ if (*hdrUrl == '/') {
+ // RFC 3986 section 4.2: absolute-path reference
+ // for this logic replace the entire request-target URI path
+ tmpUrl.path(hdrUrl);
+ } else {
+ tmpUrl.addRelativePath(reqUrl);
+ }
+ absUrlMaker = tmpUrl.absolute();
+ absUrl = absUrlMaker.c_str();
}
} else if (!sameUrlHosts(reqUrl, hdrUrl)) {
return;
- }
+ } else
+ absUrl = hdrUrl;
- purgeEntriesByUrl(req, hdrUrl);
-
- if (absUrl != NULL) {
- safe_free(absUrl);
- }
+ purgeEntriesByUrl(req, absUrl);
}
// some HTTP methods should purge matching cache entries
diff -u -r -N squid-5.0.2/src/clients/HttpTunnelerAnswer.cc squid-5.0.3/src/clients/HttpTunnelerAnswer.cc
--- squid-5.0.2/src/clients/HttpTunnelerAnswer.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/clients/HttpTunnelerAnswer.cc 2020-06-08 03:33:42.000000000 +1200
@@ -32,6 +32,9 @@
if (answer.peerResponseStatus != Http::scNone)
os << ' ' << answer.peerResponseStatus;
+ if (answer.conn)
+ os << ' ' << answer.conn;
+
os << ']';
return os;
}
diff -u -r -N squid-5.0.2/src/clients/HttpTunnelerAnswer.h squid-5.0.3/src/clients/HttpTunnelerAnswer.h
--- squid-5.0.2/src/clients/HttpTunnelerAnswer.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/clients/HttpTunnelerAnswer.h 2020-06-08 03:33:42.000000000 +1200
@@ -43,6 +43,8 @@
/// the status code of the successfully parsed CONNECT response (or scNone)
StatusCode peerResponseStatus = scNone;
+
+ Comm::ConnectionPointer conn;
};
std::ostream &operator <<(std::ostream &, const Http::TunnelerAnswer &);
diff -u -r -N squid-5.0.2/src/clients/HttpTunneler.cc squid-5.0.3/src/clients/HttpTunneler.cc
--- squid-5.0.2/src/clients/HttpTunneler.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/clients/HttpTunneler.cc 2020-06-08 03:33:42.000000000 +1200
@@ -18,6 +18,8 @@
#include "http/one/ResponseParser.h"
#include "http/StateFlags.h"
#include "HttpRequest.h"
+#include "neighbors.h"
+#include "pconn.h"
#include "SquidConfig.h"
#include "StatCounters.h"
@@ -25,6 +27,7 @@
Http::Tunneler::Tunneler(const Comm::ConnectionPointer &conn, const HttpRequest::Pointer &req, AsyncCall::Pointer &aCallback, time_t timeout, const AccessLogEntryPointer &alp):
AsyncJob("Http::Tunneler"),
+ noteFwdPconnUse(false),
connection(conn),
request(req),
callback(aCallback),
@@ -41,6 +44,7 @@
assert(callback);
assert(dynamic_cast(callback->getDialer()));
url = request->url.authority();
+ watchForClosures();
}
Http::Tunneler::~Tunneler()
@@ -73,11 +77,22 @@
Must(url.length());
Must(lifetimeLimit >= 0);
+ // we own this Comm::Connection object and its fd exclusively, but must bail
+ // if others started closing the socket while we were waiting to start()
+ assert(Comm::IsConnOpen(connection));
+ if (fd_table[connection->fd].closing()) {
+ bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw(), al));
+ return;
+ }
+
const auto peer = connection->getPeer();
- Must(peer); // bail if our peer was reconfigured away
+ // bail if our peer was reconfigured away
+ if (!peer) {
+ bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scInternalServerError, request.getRaw(), al));
+ return;
+ }
request->prepForPeering(*peer);
- watchForClosures();
writeRequest();
startReadingResponse();
}
@@ -85,8 +100,8 @@
void
Http::Tunneler::handleConnectionClosure(const CommCloseCbParams ¶ms)
{
- mustStop("server connection gone");
- callback = nullptr; // the caller must monitor closures
+ closer = nullptr;
+ bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw(), al));
}
/// make sure we quit if/when the connection is gone
@@ -104,12 +119,11 @@
comm_add_close_handler(connection->fd, closer);
}
+/// The connection read timeout callback handler.
void
-Http::Tunneler::handleException(const std::exception& e)
+Http::Tunneler::handleTimeout(const CommTimeoutCbParams &)
{
- debugs(83, 2, e.what() << status());
- connection->close();
- bailWith(new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw(), al));
+ bailWith(new ErrorState(ERR_CONNECT_FAIL, Http::scGatewayTimeout, request.getRaw(), al));
}
void
@@ -255,8 +269,11 @@
Comm::Read(connection, reader);
AsyncCall::Pointer nil;
+ typedef CommCbMemFunT TimeoutDialer;
+ AsyncCall::Pointer timeoutCall = JobCallback(93, 5,
+ TimeoutDialer, this, Http::Tunneler::handleTimeout);
const auto timeout = Comm::MortalReadTimeout(startTime, lifetimeLimit);
- commSetConnTimeout(connection, timeout, nil);
+ commSetConnTimeout(connection, timeout, timeoutCall);
}
/// Parses [possibly incomplete] CONNECT response and reacts to it.
@@ -342,13 +359,51 @@
{
Must(error);
answer().squidError = error;
+
+ if (const auto p = connection->getPeer())
+ peerConnectFailed(p);
+
callBack();
+ disconnect();
+
+ if (noteFwdPconnUse)
+ fwdPconnPool->noteUses(fd_table[connection->fd].pconn.uses);
+ // TODO: Reuse to-peer connections after a CONNECT error response.
+ connection->close();
+ connection = nullptr;
+}
+
+void
+Http::Tunneler::sendSuccess()
+{
+ assert(answer().positive());
+ callBack();
+ disconnect();
+}
+
+void
+Http::Tunneler::disconnect()
+{
+ if (closer) {
+ comm_remove_close_handler(connection->fd, closer);
+ closer = nullptr;
+ }
+
+ if (reader) {
+ Comm::ReadCancel(connection->fd, reader);
+ reader = nullptr;
+ }
+
+ // remove connection timeout handler
+ commUnsetConnTimeout(connection);
}
void
Http::Tunneler::callBack()
{
debugs(83, 5, connection << status());
+ if (answer().positive())
+ answer().conn = connection;
auto cb = callback;
callback = nullptr;
ScheduleCallHere(cb);
@@ -361,8 +416,7 @@
if (callback) {
if (requestWritten && tunnelEstablished) {
- assert(answer().positive());
- callBack(); // success
+ sendSuccess();
} else {
// we should have bailed when we discovered the job-killing problem
debugs(83, DBG_IMPORTANT, "BUG: Unexpected state while establishing a CONNECT tunnel " << connection << status());
@@ -370,16 +424,6 @@
}
assert(!callback);
}
-
- if (closer) {
- comm_remove_close_handler(connection->fd, closer);
- closer = nullptr;
- }
-
- if (reader) {
- Comm::ReadCancel(connection->fd, reader);
- reader = nullptr;
- }
}
const char *
diff -u -r -N squid-5.0.2/src/clients/HttpTunneler.h squid-5.0.3/src/clients/HttpTunneler.h
--- squid-5.0.2/src/clients/HttpTunneler.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/clients/HttpTunneler.h 2020-06-08 03:33:42.000000000 +1200
@@ -26,15 +26,9 @@
namespace Http
{
-/// Establishes an HTTP CONNECT tunnel through a forward proxy.
-///
-/// The caller receives a call back with Http::TunnelerAnswer.
-///
-/// The caller must monitor the connection for closure because this job will not
-/// inform the caller about such events.
-///
-/// This job never closes the connection, even on errors. If a 3rd-party closes
-/// the connection, this job simply quits without informing the caller.
+/// Negotiates an HTTP CONNECT tunnel through a forward proxy using a given
+/// (open and, if needed, encrypted) TCP connection to that proxy. Owns the
+/// connection during these negotiations. The caller receives TunnelerAnswer.
class Tunneler: virtual public AsyncJob
{
CBDATA_CLASS(Tunneler);
@@ -71,6 +65,9 @@
void setDelayId(DelayId delay_id) {delayId = delay_id;}
#endif
+ /// hack: whether the connection requires fwdPconnPool->noteUses()
+ bool noteFwdPconnUse;
+
protected:
/* AsyncJob API */
virtual ~Tunneler();
@@ -81,7 +78,7 @@
void handleConnectionClosure(const CommCloseCbParams&);
void watchForClosures();
- void handleException(const std::exception&);
+ void handleTimeout(const CommTimeoutCbParams &);
void startReadingResponse();
void writeRequest();
void handleWrittenRequest(const CommIoCbParams&);
@@ -89,9 +86,19 @@
void readMore();
void handleResponse(const bool eof);
void bailOnResponseError(const char *error, HttpReply *);
+
+ /// sends the given error to the initiator
void bailWith(ErrorState*);
+
+ /// sends the ready-to-use tunnel to the initiator
+ void sendSuccess();
+
+ /// a bailWith(), sendSuccess() helper: sends results to the initiator
void callBack();
+ /// a bailWith(), sendSuccess() helper: stops monitoring the connection
+ void disconnect();
+
TunnelerAnswer &answer();
private:
diff -u -r -N squid-5.0.2/src/client_side.cc squid-5.0.3/src/client_side.cc
--- squid-5.0.2/src/client_side.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/client_side.cc 2020-06-08 03:33:42.000000000 +1200
@@ -2187,6 +2187,7 @@
bodyParser(nullptr),
#if USE_OPENSSL
sslBumpMode(Ssl::bumpEnd),
+ tlsParser(Security::HandshakeParser::fromClient),
#endif
needProxyProtocolHeader_(false),
#if USE_OPENSSL
diff -u -r -N squid-5.0.2/src/client_side_reply.cc squid-5.0.3/src/client_side_reply.cc
--- squid-5.0.2/src/client_side_reply.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/client_side_reply.cc 2020-06-08 03:33:42.000000000 +1200
@@ -913,7 +913,7 @@
const cache_key *key = storeKeyPublic(url, m);
debugs(88, 5, m << ' ' << url << ' ' << storeKeyText(key));
#if USE_HTCP
- neighborsHtcpClear(nullptr, url, req, m, HTCP_CLR_INVALIDATION);
+ neighborsHtcpClear(nullptr, req, m, HTCP_CLR_INVALIDATION);
#endif
Store::Root().evictIfFound(key);
}
@@ -1051,7 +1051,7 @@
/* Release the cached URI */
debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
#if USE_HTCP
- neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
+ neighborsHtcpClear(newEntry, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
#endif
newEntry->release(true);
purgeStatus = Http::scOkay;
@@ -1067,7 +1067,7 @@
if (newEntry) {
debugs(88, 4, "HEAD " << newEntry->url());
#if USE_HTCP
- neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
+ neighborsHtcpClear(newEntry, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
#endif
newEntry->release(true);
purgeStatus = Http::scOkay;
@@ -1083,7 +1083,7 @@
if (entry) {
debugs(88, 4, "Vary GET " << entry->url());
#if USE_HTCP
- neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
+ neighborsHtcpClear(entry, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
#endif
entry->release(true);
purgeStatus = Http::scOkay;
@@ -1094,7 +1094,7 @@
if (entry) {
debugs(88, 4, "Vary HEAD " << entry->url());
#if USE_HTCP
- neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
+ neighborsHtcpClear(entry, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
#endif
entry->release(true);
purgeStatus = Http::scOkay;
diff -u -r -N squid-5.0.2/src/comm/forward.h squid-5.0.3/src/comm/forward.h
--- squid-5.0.2/src/comm/forward.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/comm/forward.h 2020-06-08 03:33:42.000000000 +1200
@@ -26,8 +26,6 @@
typedef RefCount ConnectionPointer;
-typedef std::vector ConnectionList;
-
bool IsConnOpen(const Comm::ConnectionPointer &conn);
// callback handler to process an FD which is available for writing.
diff -u -r -N squid-5.0.2/src/comm.cc squid-5.0.3/src/comm.cc
--- squid-5.0.2/src/comm.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/comm.cc 2020-06-08 03:33:42.000000000 +1200
@@ -1275,30 +1275,38 @@
assert(clientInfo);
assert(clientInfo->hasQueue());
assert(clientInfo->hasQueue(queue));
- assert(!clientInfo->selectWaiting);
assert(clientInfo->eventWaiting);
clientInfo->eventWaiting = false;
do {
- // check that the head descriptor is still relevant
- const int head = clientInfo->quotaPeekFd();
- Comm::IoCallback *ccb = COMMIO_FD_WRITECB(head);
+ clientInfo->writeOrDequeue();
+ if (clientInfo->selectWaiting)
+ return;
+ } while (clientInfo->hasQueue());
- if (fd_table[head].clientInfo == clientInfo &&
- clientInfo->quotaPeekReserv() == ccb->quotaQueueReserv &&
- !fd_table[head].closing()) {
+ debugs(77, 3, "emptied queue");
+}
+
+void
+ClientInfo::writeOrDequeue()
+{
+ assert(!selectWaiting);
+ const auto head = quotaPeekFd();
+ const auto &headFde = fd_table[head];
+ CallBack(headFde.codeContext, [&] {
+ const auto ccb = COMMIO_FD_WRITECB(head);
+ // check that the head descriptor is still relevant
+ if (headFde.clientInfo == this &&
+ quotaPeekReserv() == ccb->quotaQueueReserv &&
+ !headFde.closing()) {
// wait for the head descriptor to become ready for writing
Comm::SetSelect(head, COMM_SELECT_WRITE, Comm::HandleWrite, ccb, 0);
- clientInfo->selectWaiting = true;
- return;
+ selectWaiting = true;
+ } else {
+ quotaDequeue(); // remove the no longer relevant descriptor
}
-
- clientInfo->quotaDequeue(); // remove the no longer relevant descriptor
- // and continue looking for a relevant one
- } while (clientInfo->hasQueue());
-
- debugs(77,3, HERE << "emptied queue");
+ });
}
bool
@@ -1474,6 +1482,7 @@
debugs(77,5, "clt" << (const char*)clientInfo->key <<
": FD " << fd << " with qqid" << (ins+1) << ' ' << fds.size());
fds.push_back(fd);
+ fd_table[fd].codeContext = CodeContext::Current();
return ++ins;
}
@@ -1639,6 +1648,7 @@
debugs(5, 5, HERE << "adding FD " << fd << " to " << *TheHalfClosed);
assert(isOpen(fd) && !commHasHalfClosedMonitor(fd));
(void)TheHalfClosed->add(fd); // could also assert the result
+ fd_table[fd].codeContext = CodeContext::Current();
commPlanHalfClosedCheck(); // may schedule check if we added the first FD
}
@@ -1666,10 +1676,12 @@
Comm::ConnectionPointer c = new Comm::Connection; // XXX: temporary. make HalfClosed a list of these.
c->fd = *i;
if (!fd_table[c->fd].halfClosedReader) { // not reading already
- AsyncCall::Pointer call = commCbCall(5,4, "commHalfClosedReader",
- CommIoCbPtrFun(&commHalfClosedReader, NULL));
- Comm::Read(c, call);
- fd_table[c->fd].halfClosedReader = call;
+ CallBack(fd_table[c->fd].codeContext, [&c] {
+ AsyncCall::Pointer call = commCbCall(5,4, "commHalfClosedReader",
+ CommIoCbPtrFun(&commHalfClosedReader, nullptr));
+ Comm::Read(c, call);
+ fd_table[c->fd].halfClosedReader = call;
+ });
} else
c->fd = -1; // XXX: temporary. prevent c replacement erase closing listed FD
}
diff -u -r -N squid-5.0.2/src/format/Format.cc squid-5.0.3/src/format/Format.cc
--- squid-5.0.2/src/format/Format.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/format/Format.cc 2020-06-08 03:33:42.000000000 +1200
@@ -341,15 +341,6 @@
*p = '\0';
}
-#if USE_OPENSSL
-static char *
-sslErrorName(Security::ErrorCode err, char *buf, size_t size)
-{
- snprintf(buf, size, "SSL_ERR=%d", err);
- return buf;
-}
-#endif
-
/// XXX: Misnamed. TODO: Split request && al->request->errType == ERR_SECURE_CONNECT_FAIL) {
- out = Ssl::GetErrorName(al->request->errDetail);
- if (!out)
- out = sslErrorName(al->request->errDetail, tmp, sizeof(tmp));
+ out = Ssl::GetErrorName(al->request->errDetail, true);
} else
#endif
if (al->request && al->request->errDetail != ERR_DETAIL_NONE) {
@@ -1312,10 +1301,7 @@
for (const Security::CertErrors *sslError = srvBump->sslErrors(); sslError; sslError = sslError->next) {
if (!sb.isEmpty())
sb.append(separator);
- if (const char *errorName = Ssl::GetErrorName(sslError->element.code))
- sb.append(errorName);
- else
- sb.append(sslErrorName(sslError->element.code, tmp, sizeof(tmp)));
+ sb.append(Ssl::GetErrorName(sslError->element.code, true));
if (sslError->element.depth >= 0)
sb.appendf("@depth=%d", sslError->element.depth);
}
diff -u -r -N squid-5.0.2/src/fs/rock/RockSwapDir.cc squid-5.0.3/src/fs/rock/RockSwapDir.cc
--- squid-5.0.2/src/fs/rock/RockSwapDir.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/fs/rock/RockSwapDir.cc 2020-06-08 03:33:42.000000000 +1200
@@ -1136,19 +1136,14 @@
mapOwners.push_back(mapOwner);
// TODO: somehow remove pool id and counters from PageStack?
+ Ipc::Mem::PageStack::Config config;
+ config.poolId = Ipc::Mem::PageStack::IdForSwapDirSpace(i);
+ config.pageSize = 0; // this is an index of slots on _disk_
+ config.capacity = capacity;
+ config.createFull = false; // Rebuild finds and pushes free slots
Ipc::Mem::Owner *const freeSlotsOwner =
- shm_new(Ipc::Mem::PageStack)(sd->freeSlotsPath(),
- Ipc::Mem::PageStack::IdForSwapDirSpace(i),
- capacity,
- 0);
+ shm_new(Ipc::Mem::PageStack)(sd->freeSlotsPath(), config);
freeSlotsOwners.push_back(freeSlotsOwner);
-
- // TODO: add method to initialize PageStack with no free pages
- while (true) {
- Ipc::Mem::PageId pageId;
- if (!freeSlotsOwner->object()->pop(pageId))
- break;
- }
}
}
}
diff -u -r -N squid-5.0.2/src/FwdState.cc squid-5.0.3/src/FwdState.cc
--- squid-5.0.2/src/FwdState.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/FwdState.cc 2020-06-08 03:33:42.000000000 +1200
@@ -119,6 +119,18 @@
}
void
+FwdState::closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason)
+{
+ debugs(17, 3, "because " << reason << "; " << conn);
+ assert(!serverConn);
+ assert(!closeHandler);
+ if (IsConnOpen(conn)) {
+ fwdPconnPool->noteUses(fd_table[conn->fd].pconn.uses);
+ conn->close();
+ }
+}
+
+void
FwdState::closeServerConnection(const char *reason)
{
debugs(17, 3, "because " << reason << "; " << serverConn);
@@ -753,6 +765,24 @@
retryOrBail();
}
+/// starts a preparation step for an established connection; retries on failures
+template
+void
+FwdState::advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep)
+{
+ // TODO: Extract destination-specific handling from FwdState so that all the
+ // awkward, limited-scope advanceDestination() calls can be replaced with a
+ // single simple try/catch,retry block.
+ try {
+ startStep();
+ // now wait for the step callback
+ } catch (...) {
+ debugs (17, 2, "exception while trying to " << stepDescription << ": " << CurrentException);
+ closePendingConnection(conn, "connection preparation exception");
+ retryOrBail();
+ }
+}
+
/// called when a to-peer connection has been successfully obtained or
/// when all candidate destinations have been tried and all have failed
void
@@ -764,22 +794,31 @@
Must(n_tries <= answer.n_tries); // n_tries cannot decrease
n_tries = answer.n_tries;
- if (const auto error = answer.error.get()) {
+ ErrorState *error = nullptr;
+ if ((error = answer.error.get())) {
flags.dont_retry = true; // or HappyConnOpener would not have given up
syncHierNote(answer.conn, request->url.host());
- fail(error);
+ Must(!Comm::IsConnOpen(answer.conn));
answer.error.clear(); // preserve error for errorSendComplete()
+ } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
+ syncHierNote(answer.conn, request->url.host());
+ closePendingConnection(answer.conn, "conn was closed while waiting for noteConnection");
+ error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al);
+ }
+
+ if (error) {
+ fail(error);
retryOrBail(); // will notice flags.dont_retry and bail
return;
}
- syncWithServerConn(answer.conn, request->url.host(), answer.reused);
-
- if (answer.reused)
+ if (answer.reused) {
+ syncWithServerConn(answer.conn, request->url.host(), answer.reused);
return dispatch();
+ }
// Check if we need to TLS before use
- if (const CachePeer *peer = serverConnection()->getPeer()) {
+ if (const auto *peer = answer.conn->getPeer()) {
// Assume that it is only possible for the client-first from the
// bumping modes to try connect to a remote server. The bumped
// requests with other modes are using pinned connections or fails.
@@ -793,20 +832,27 @@
if (originWantsEncryptedTraffic && // the "encrypted traffic" part
!peer->options.originserver && // the "through a proxy" part
!peer->secure.encryptTransport) // the "exclude HTTPS proxies" part
- return establishTunnelThruProxy();
+ return advanceDestination("establish tunnel through proxy", answer.conn, [this,&answer] {
+ establishTunnelThruProxy(answer.conn);
+ });
}
- secureConnectionToPeerIfNeeded();
+ secureConnectionToPeerIfNeeded(answer.conn);
}
void
-FwdState::establishTunnelThruProxy()
+FwdState::establishTunnelThruProxy(const Comm::ConnectionPointer &conn)
{
AsyncCall::Pointer callback = asyncCall(17,4,
"FwdState::tunnelEstablishmentDone",
Http::Tunneler::CbDialer(&FwdState::tunnelEstablishmentDone, this));
HttpRequest::Pointer requestPointer = request;
- const auto tunneler = new Http::Tunneler(serverConnection(), requestPointer, callback, connectingTimeout(serverConnection()), al);
+ const auto tunneler = new Http::Tunneler(conn, requestPointer, callback, connectingTimeout(serverConnection()), al);
+
+ // TODO: Replace this hack with proper Comm::Connection-Pool association
+ // that is not tied to fwdPconnPool and can handle disappearing pools.
+ tunneler->noteFwdPconnUse = true;
+
#if USE_DELAY_POOLS
Must(serverConnection()->getPeer());
if (!serverConnection()->getPeer()->options.no_delay)
@@ -820,11 +866,18 @@
void
FwdState::tunnelEstablishmentDone(Http::TunnelerAnswer &answer)
{
- if (answer.positive()) {
- if (answer.leftovers.isEmpty()) {
- secureConnectionToPeerIfNeeded();
- return;
- }
+ ErrorState *error = nullptr;
+ if (!answer.positive()) {
+ Must(!Comm::IsConnOpen(answer.conn));
+ error = answer.squidError.get();
+ Must(error);
+ answer.squidError.clear(); // preserve error for fail()
+ } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
+ // The socket could get closed while our callback was queued.
+ // We close Connection here to sync Connection::fd.
+ closePendingConnection(answer.conn, "conn was closed while waiting for tunnelEstablishmentDone");
+ error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al);
+ } else if (!answer.leftovers.isEmpty()) {
// This should not happen because TLS servers do not speak first. If we
// have to handle this, then pass answer.leftovers via a PeerConnector
// to ServerBio. See ClientBio::setReadBufData().
@@ -832,33 +885,26 @@
const auto level = (occurrences++ < 100) ? DBG_IMPORTANT : 2;
debugs(17, level, "ERROR: Early data after CONNECT response. " <<
"Found " << answer.leftovers.length() << " bytes. " <<
- "Closing " << serverConnection());
- fail(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request, al));
- closeServerConnection("found early data after CONNECT response");
+ "Closing " << answer.conn);
+ error = new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request, al);
+ closePendingConnection(answer.conn, "server spoke before tunnelEstablishmentDone");
+ }
+ if (error) {
+ fail(error);
retryOrBail();
return;
}
- // TODO: Reuse to-peer connections after a CONNECT error response.
-
- if (const auto peer = serverConnection()->getPeer())
- peerConnectFailed(peer);
-
- const auto error = answer.squidError.get();
- Must(error);
- answer.squidError.clear(); // preserve error for fail()
- fail(error);
- closeServerConnection("Squid-generated CONNECT error");
- retryOrBail();
+ secureConnectionToPeerIfNeeded(answer.conn);
}
/// handles an established TCP connection to peer (including origin servers)
void
-FwdState::secureConnectionToPeerIfNeeded()
+FwdState::secureConnectionToPeerIfNeeded(const Comm::ConnectionPointer &conn)
{
assert(!request->flags.pinned);
- const CachePeer *p = serverConnection()->getPeer();
+ const auto p = conn->getPeer();
const bool peerWantsTls = p && p->secure.encryptTransport;
// userWillTlsToPeerForUs assumes CONNECT == HTTPS
const bool userWillTlsToPeerForUs = p && p->options.originserver &&
@@ -872,56 +918,70 @@
const bool needTlsToOrigin = !p && request->url.getScheme() == AnyP::PROTO_HTTPS && !clientFirstBump;
if (needTlsToPeer || needTlsToOrigin || needsBump) {
- HttpRequest::Pointer requestPointer = request;
- AsyncCall::Pointer callback = asyncCall(17,4,
- "FwdState::ConnectedToPeer",
- FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this));
- const auto sslNegotiationTimeout = connectingTimeout(serverConnection());
- Security::PeerConnector *connector = nullptr;
-#if USE_OPENSSL
- if (request->flags.sslPeek)
- connector = new Ssl::PeekingPeerConnector(requestPointer, serverConnection(), clientConn, callback, al, sslNegotiationTimeout);
- else
-#endif
- connector = new Security::BlindPeerConnector(requestPointer, serverConnection(), callback, al, sslNegotiationTimeout);
- AsyncJob::Start(connector); // will call our callback
- return;
+ return advanceDestination("secure connection to peer", conn, [this,&conn] {
+ secureConnectionToPeer(conn);
+ });
}
// if not encrypting just run the post-connect actions
- successfullyConnectedToPeer();
+ successfullyConnectedToPeer(conn);
+}
+
+/// encrypts an established TCP connection to peer (including origin servers)
+void
+FwdState::secureConnectionToPeer(const Comm::ConnectionPointer &conn)
+{
+ HttpRequest::Pointer requestPointer = request;
+ AsyncCall::Pointer callback = asyncCall(17,4,
+ "FwdState::ConnectedToPeer",
+ FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this));
+ const auto sslNegotiationTimeout = connectingTimeout(conn);
+ Security::PeerConnector *connector = nullptr;
+#if USE_OPENSSL
+ if (request->flags.sslPeek)
+ connector = new Ssl::PeekingPeerConnector(requestPointer, conn, clientConn, callback, al, sslNegotiationTimeout);
+ else
+#endif
+ connector = new Security::BlindPeerConnector(requestPointer, conn, callback, al, sslNegotiationTimeout);
+ connector->noteFwdPconnUse = true;
+ AsyncJob::Start(connector); // will call our callback
}
/// called when all negotiations with the TLS-speaking peer have been completed
void
FwdState::connectedToPeer(Security::EncryptorAnswer &answer)
{
- if (ErrorState *error = answer.error.get()) {
- fail(error);
+ ErrorState *error = nullptr;
+ if ((error = answer.error.get())) {
+ Must(!Comm::IsConnOpen(answer.conn));
answer.error.clear(); // preserve error for errorSendComplete()
- if (CachePeer *p = serverConnection()->getPeer())
- peerConnectFailed(p);
- serverConnection()->close();
- return;
- }
-
- if (answer.tunneled) {
+ } else if (answer.tunneled) {
// TODO: When ConnStateData establishes tunnels, its state changes
// [in ways that may affect logging?]. Consider informing
// ConnStateData about our tunnel or otherwise unifying tunnel
// establishment [side effects].
- unregister(serverConn); // async call owns it now
complete(); // destroys us
return;
+ } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
+ closePendingConnection(answer.conn, "conn was closed while waiting for connectedToPeer");
+ error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al);
}
- successfullyConnectedToPeer();
+ if (error) {
+ fail(error);
+ retryOrBail();
+ return;
+ }
+
+ successfullyConnectedToPeer(answer.conn);
}
/// called when all negotiations with the peer have been completed
void
-FwdState::successfullyConnectedToPeer()
+FwdState::successfullyConnectedToPeer(const Comm::ConnectionPointer &conn)
{
+ syncWithServerConn(conn, request->url.host(), false);
+
// should reach ConnStateData before the dispatched Client job starts
CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData,
ConnStateData::notePeerConnection, serverConnection());
@@ -1146,7 +1206,7 @@
// Set the dont_retry flag because this is not a transient (network) error.
flags.dont_retry = true;
if (Comm::IsConnOpen(serverConn)) {
- serverConn->close();
+ serverConn->close(); // trigger cleanup
}
break;
}
@@ -1353,7 +1413,7 @@
}
void
-getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn)
+getOutgoingAddress(HttpRequest * request, const Comm::ConnectionPointer &conn)
{
// skip if an outgoing address is already set.
if (!conn->local.isAnyAddr()) return;
diff -u -r -N squid-5.0.2/src/FwdState.h squid-5.0.3/src/FwdState.h
--- squid-5.0.2/src/FwdState.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/FwdState.h 2020-06-08 03:33:42.000000000 +1200
@@ -102,6 +102,9 @@
void dontRetry(bool val) { flags.dont_retry = val; }
+ /// get rid of a to-server connection that failed to become serverConn
+ void closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason);
+
/** return a ConnectionPointer to the current server connection (may or may not be open) */
Comm::ConnectionPointer const & serverConnection() const { return serverConn; };
@@ -131,14 +134,18 @@
/// (in order to retry or reforward a failed request)
bool pinnedCanRetry() const;
+ template
+ void advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep);
+
ErrorState *makeConnectingError(const err_type type) const;
void connectedToPeer(Security::EncryptorAnswer &answer);
static void RegisterWithCacheManager(void);
- void establishTunnelThruProxy();
+ void establishTunnelThruProxy(const Comm::ConnectionPointer &);
void tunnelEstablishmentDone(Http::TunnelerAnswer &answer);
- void secureConnectionToPeerIfNeeded();
- void successfullyConnectedToPeer();
+ void secureConnectionToPeerIfNeeded(const Comm::ConnectionPointer &);
+ void secureConnectionToPeer(const Comm::ConnectionPointer &);
+ void successfullyConnectedToPeer(const Comm::ConnectionPointer &);
/// stops monitoring server connection for closure and updates pconn stats
void closeServerConnection(const char *reason);
@@ -197,7 +204,7 @@
PconnRace pconnRace; ///< current pconn race state
};
-void getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn);
+void getOutgoingAddress(HttpRequest * request, const Comm::ConnectionPointer &conn);
/// a collection of previously used persistent Squid-to-peer HTTP(S) connections
extern PconnPool *fwdPconnPool;
diff -u -r -N squid-5.0.2/src/HappyConnOpener.cc squid-5.0.3/src/HappyConnOpener.cc
--- squid-5.0.2/src/HappyConnOpener.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/HappyConnOpener.cc 2020-06-08 03:33:42.000000000 +1200
@@ -397,15 +397,11 @@
// TODO: Find an automated, faster way to kill no-longer-needed jobs.
if (prime) {
- if (prime.connector)
- prime.connector->cancel("HappyConnOpener object destructed");
- prime.clear();
+ cancelAttempt(prime, "job finished during a prime attempt");
}
if (spare) {
- if (spare.connector)
- spare.connector->cancel("HappyConnOpener object destructed");
- spare.clear();
+ cancelAttempt(spare, "job finished during a spare attempt");
if (gotSpareAllowance) {
TheSpareAllowanceGiver.jobDroppedAllowance();
gotSpareAllowance = false;
@@ -484,6 +480,15 @@
callback_ = nullptr;
}
+/// cancels the in-progress attempt, making its path a future candidate
+void
+HappyConnOpener::cancelAttempt(Attempt &attempt, const char *reason)
+{
+ Must(attempt);
+ destinations->retryPath(attempt.path); // before attempt.cancel() clears path
+ attempt.cancel(reason);
+}
+
/// inform the initiator about our failure to connect (if needed)
void
HappyConnOpener::sendFailure()
@@ -563,6 +568,7 @@
attempt.path = dest;
attempt.connector = callConnect;
+ attempt.opener = cs;
AsyncJob::Start(cs);
}
@@ -578,9 +584,9 @@
Must(itWasPrime != itWasSpare);
if (itWasPrime) {
- prime.clear();
+ prime.finish();
} else {
- spare.clear();
+ spare.finish();
if (gotSpareAllowance) {
TheSpareAllowanceGiver.jobUsedAllowance();
gotSpareAllowance = false;
@@ -869,3 +875,13 @@
return false;
}
+void
+HappyConnOpener::Attempt::cancel(const char *reason)
+{
+ if (connector) {
+ connector->cancel(reason);
+ CallJobHere(17, 3, opener, Comm::ConnOpener, noteAbort);
+ }
+ clear();
+}
+
diff -u -r -N squid-5.0.2/src/HappyConnOpener.h squid-5.0.3/src/HappyConnOpener.h
--- squid-5.0.2/src/HappyConnOpener.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/HappyConnOpener.h 2020-06-08 03:33:42.000000000 +1200
@@ -157,10 +157,20 @@
class Attempt {
public:
explicit operator bool() const { return static_cast(path); }
- void clear() { path = nullptr; connector = nullptr; }
+
+ /// reacts to a natural attempt completion (successful or otherwise)
+ void finish() { clear(); }
+
+ /// aborts an in-progress attempt
+ void cancel(const char *reason);
Comm::ConnectionPointer path; ///< the destination we are connecting to
- AsyncCall::Pointer connector; ///< our Comm::ConnOpener callback
+ AsyncCall::Pointer connector; ///< our opener callback
+ Comm::ConnOpener::Pointer opener; ///< connects to path and calls us
+
+ private:
+ /// cleans up after the attempt ends (successfully or otherwise)
+ void clear() { path = nullptr; connector = nullptr; opener = nullptr; }
};
/* AsyncJob API */
@@ -194,6 +204,7 @@
Answer *futureAnswer(const Comm::ConnectionPointer &);
void sendSuccess(const Comm::ConnectionPointer &conn, bool reused, const char *connKind);
void sendFailure();
+ void cancelAttempt(Attempt &, const char *reason);
const time_t fwdStart; ///< requestor start time
diff -u -r -N squid-5.0.2/src/htcp.cc squid-5.0.3/src/htcp.cc
--- squid-5.0.2/src/htcp.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/htcp.cc 2020-06-08 03:33:42.000000000 +1200
@@ -145,7 +145,7 @@
public:
const char *method = nullptr;
- char *uri = nullptr;
+ const char *uri = nullptr;
char *version = nullptr;
char *req_hdrs = nullptr;
size_t reqHdrsSz = 0; ///< size of the req_hdrs content
@@ -816,6 +816,7 @@
if (spec) {
stuff.S.method = spec->method;
+ stuff.S.request = spec->request;
stuff.S.uri = spec->uri;
stuff.S.version = spec->version;
stuff.S.req_hdrs = spec->req_hdrs;
@@ -850,15 +851,15 @@
hdr.clean();
#if USE_ICMP
- if (char *host = urlHostname(spec->uri)) {
+ if (const char *host = spec->request->url.host()) {
int rtt = 0;
int hops = 0;
int samp = 0;
netdbHostData(host, &samp, &rtt, &hops);
if (rtt || hops) {
- char cto_buf[128];
- snprintf(cto_buf, 128, "%s %d %f %d",
+ char cto_buf[SQUIDHOSTNAMELEN+128];
+ snprintf(cto_buf, sizeof(cto_buf), "%s %d %f %d",
host, samp, 0.001 * rtt, hops);
hdr.putExt("Cache-to-Origin", cto_buf);
}
@@ -1563,7 +1564,7 @@
* Send an HTCP CLR message for a specified item to a given CachePeer.
*/
void
-htcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &, CachePeer * p, htcp_clr_reason reason)
+htcpClear(StoreEntry * e, HttpRequest * req, const HttpRequestMethod &, CachePeer * p, htcp_clr_reason reason)
{
static char pkt[8192];
ssize_t pktlen;
@@ -1585,14 +1586,9 @@
SBuf sb = req->method.image();
stuff.S.method = sb.c_str();
- if (e == NULL || e->mem_obj == NULL) {
- if (uri == NULL) {
- return;
- }
- stuff.S.uri = xstrdup(uri);
- } else {
- stuff.S.uri = (char *) e->url();
- }
+ stuff.S.request = req;
+ SBuf uri = req->effectiveRequestUri();
+ stuff.S.uri = uri.c_str();
stuff.S.version = vbuf;
if (reason != HTCP_CLR_INVALIDATION) {
HttpStateData::httpBuildRequestHeader(req, e, NULL, &hdr, flags);
@@ -1607,9 +1603,6 @@
if (reason != HTCP_CLR_INVALIDATION) {
mb.clean();
}
- if (e == NULL) {
- xfree(stuff.S.uri);
- }
if (!pktlen) {
debugs(31, 3, "htcpClear: htcpBuildPacket() failed");
return;
diff -u -r -N squid-5.0.2/src/htcp.h squid-5.0.3/src/htcp.h
--- squid-5.0.2/src/htcp.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/htcp.h 2020-06-08 03:33:42.000000000 +1200
@@ -61,7 +61,7 @@
int htcpQuery(StoreEntry * e, HttpRequest * req, CachePeer * p);
/// \ingroup ServerProtocolHTCP
-void htcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &method, CachePeer * p, htcp_clr_reason reason);
+void htcpClear(StoreEntry * e, HttpRequest * req, const HttpRequestMethod &method, CachePeer * p, htcp_clr_reason reason);
/// \ingroup ServerProtocolHTCP
void htcpSocketShutdown(void);
diff -u -r -N squid-5.0.2/src/http/ContentLengthInterpreter.cc squid-5.0.3/src/http/ContentLengthInterpreter.cc
--- squid-5.0.2/src/http/ContentLengthInterpreter.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/http/ContentLengthInterpreter.cc 2020-06-08 03:33:42.000000000 +1200
@@ -29,6 +29,24 @@
{
}
+/// checks whether all characters before the Content-Length number are allowed
+/// \returns the start of the digit sequence (or nil on errors)
+const char *
+Http::ContentLengthInterpreter::findDigits(const char *prefix, const char * const valueEnd) const
+{
+ // skip leading OWS in RFC 7230's `OWS field-value OWS`
+ const CharacterSet &whitespace = Http::One::Parser::WhitespaceCharacters();
+ while (prefix < valueEnd) {
+ const auto ch = *prefix;
+ if (CharacterSet::DIGIT[ch])
+ return prefix; // common case: a pre-trimmed field value
+ if (!whitespace[ch])
+ return nullptr; // (trimmed) length does not start with a digit
+ ++prefix;
+ }
+ return nullptr; // empty or whitespace-only value
+}
+
/// checks whether all characters after the Content-Length are allowed
bool
Http::ContentLengthInterpreter::goodSuffix(const char *suffix, const char * const end) const
@@ -53,10 +71,19 @@
{
Must(!sawBad);
+ const auto valueEnd = rawValue + valueSize;
+
+ const auto digits = findDigits(rawValue, valueEnd);
+ if (!digits) {
+ debugs(55, debugLevel, "WARNING: Leading garbage or empty value in" << Raw("Content-Length", rawValue, valueSize));
+ sawBad = true;
+ return false;
+ }
+
int64_t latestValue = -1;
char *suffix = nullptr;
- // TODO: Handle malformed values with leading signs (e.g., "-0" or "+1").
- if (!httpHeaderParseOffset(rawValue, &latestValue, &suffix)) {
+
+ if (!httpHeaderParseOffset(digits, &latestValue, &suffix)) {
debugs(55, DBG_IMPORTANT, "WARNING: Malformed" << Raw("Content-Length", rawValue, valueSize));
sawBad = true;
return false;
@@ -69,7 +96,7 @@
}
// check for garbage after the number
- if (!goodSuffix(suffix, rawValue + valueSize)) {
+ if (!goodSuffix(suffix, valueEnd)) {
debugs(55, debugLevel, "WARNING: Trailing garbage in" << Raw("Content-Length", rawValue, valueSize));
sawBad = true;
return false;
diff -u -r -N squid-5.0.2/src/http/ContentLengthInterpreter.h squid-5.0.3/src/http/ContentLengthInterpreter.h
--- squid-5.0.2/src/http/ContentLengthInterpreter.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/http/ContentLengthInterpreter.h 2020-06-08 03:33:42.000000000 +1200
@@ -67,6 +67,7 @@
bool sawGood;
protected:
+ const char *findDigits(const char *prefix, const char *valueEnd) const;
bool goodSuffix(const char *suffix, const char * const end) const;
bool checkValue(const char *start, const int size);
bool checkList(const String &list);
diff -u -r -N squid-5.0.2/src/http/url_rewriters/LFS/url_lfs_rewrite.8 squid-5.0.3/src/http/url_rewriters/LFS/url_lfs_rewrite.8
--- squid-5.0.2/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2020-04-20 00:20:17.000000000 +1200
+++ squid-5.0.3/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2020-06-09 19:09:22.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "URL_LFS_REWRITE 8"
-.TH URL_LFS_REWRITE 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH URL_LFS_REWRITE 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/http.cc squid-5.0.3/src/http.cc
--- squid-5.0.2/src/http.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/http.cc 2020-06-08 03:33:42.000000000 +1200
@@ -253,7 +253,7 @@
if (pe != NULL) {
assert(e != pe);
#if USE_HTCP
- neighborsHtcpClear(e, nullptr, e->mem_obj->request.getRaw(), e->mem_obj->method, HTCP_CLR_INVALIDATION);
+ neighborsHtcpClear(e, e->mem_obj->request.getRaw(), e->mem_obj->method, HTCP_CLR_INVALIDATION);
#endif
pe->release(true);
}
@@ -270,7 +270,7 @@
if (pe != NULL) {
assert(e != pe);
#if USE_HTCP
- neighborsHtcpClear(e, nullptr, e->mem_obj->request.getRaw(), HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_INVALIDATION);
+ neighborsHtcpClear(e, e->mem_obj->request.getRaw(), HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_INVALIDATION);
#endif
pe->release(true);
}
diff -u -r -N squid-5.0.2/src/ip/Intercept.cc squid-5.0.3/src/ip/Intercept.cc
--- squid-5.0.2/src/ip/Intercept.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ip/Intercept.cc 2020-06-08 03:33:42.000000000 +1200
@@ -204,7 +204,7 @@
memset(&natLookup, 0, sizeof(natLookup));
// for NAT lookup set local and remote IP:port's
if (newConn->remote.isIPv6()) {
-#if HAVE_NATLOOKUP_NL_INIPADDR_IN6
+#if HAVE_STRUCT_NATLOOKUP_NL_INIPADDR_IN6
natLookup.nl_v = 6;
newConn->local.getInAddr(natLookup.nl_inipaddr.in6);
newConn->remote.getInAddr(natLookup.nl_outipaddr.in6);
@@ -292,7 +292,7 @@
debugs(89, 9, HERE << "address: " << newConn);
return false;
} else {
-#if HAVE_NATLOOKUP_NL_REALIPADDR_IN6
+#if HAVE_STRUCT_NATLOOKUP_NL_REALIPADDR_IN6
if (newConn->remote.isIPv6())
newConn->local = natLookup.nl_realipaddr.in6;
else
diff -u -r -N squid-5.0.2/src/ipc/Makefile.am squid-5.0.3/src/ipc/Makefile.am
--- squid-5.0.2/src/ipc/Makefile.am 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ipc/Makefile.am 2020-06-08 03:33:42.000000000 +1200
@@ -71,3 +71,6 @@
install-data-local:
$(mkinstalldirs) $(DESTDIR)$(localstatedir)/run/squid;
+
+libipc_la_LIBADD = $(ATOMICLIB)
+
diff -u -r -N squid-5.0.2/src/ipc/Makefile.in squid-5.0.3/src/ipc/Makefile.in
--- squid-5.0.2/src/ipc/Makefile.in 2020-04-20 00:08:53.000000000 +1200
+++ squid-5.0.3/src/ipc/Makefile.in 2020-06-09 18:58:23.000000000 +1200
@@ -163,7 +163,8 @@
CONFIG_CLEAN_FILES =
CONFIG_CLEAN_VPATH_FILES =
LTLIBRARIES = $(noinst_LTLIBRARIES)
-libipc_la_LIBADD =
+am__DEPENDENCIES_1 =
+libipc_la_DEPENDENCIES = $(am__DEPENDENCIES_1)
am__dirstamp = $(am__leading_dot)dirstamp
am_libipc_la_OBJECTS = FdNotes.lo Kid.lo Kids.lo MemMap.lo Queue.lo \
ReadWriteLock.lo StartListening.lo StoreMap.lo StrandCoord.lo \
@@ -801,6 +802,7 @@
mem/Segment.h \
mem/forward.h
+libipc_la_LIBADD = $(ATOMICLIB)
all: all-am
.SUFFIXES:
diff -u -r -N squid-5.0.2/src/ipc/mem/PagePool.cc squid-5.0.3/src/ipc/mem/PagePool.cc
--- squid-5.0.2/src/ipc/mem/PagePool.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ipc/mem/PagePool.cc 2020-06-08 03:33:42.000000000 +1200
@@ -18,7 +18,12 @@
Ipc::Mem::PagePool::Owner *
Ipc::Mem::PagePool::Init(const char *const shmId, const Ipc::Mem::PoolId stackId, const unsigned int capacity, const size_t pageSize)
{
- return shm_new(PageStack)(shmId, stackId, capacity, pageSize);
+ PageStack::Config config;
+ config.poolId = stackId;
+ config.pageSize = pageSize; // the pages are stored in Ipc::Mem::Pages
+ config.capacity = capacity;
+ config.createFull = true; // all pages are initially available
+ return shm_new(PageStack)(shmId, config);
}
Ipc::Mem::PagePool::PagePool(const char *const id):
diff -u -r -N squid-5.0.2/src/ipc/mem/PageStack.cc squid-5.0.3/src/ipc/mem/PageStack.cc
--- squid-5.0.2/src/ipc/mem/PageStack.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ipc/mem/PageStack.cc 2020-06-08 03:33:42.000000000 +1200
@@ -10,54 +10,430 @@
#include "squid.h"
-#include "base/TextException.h"
#include "Debug.h"
#include "ipc/mem/Page.h"
#include "ipc/mem/PageStack.h"
-/* Ipc::Mem::PageStackStorageSlot */
+#include
+#include
-static_assert(sizeof(Ipc::Mem::PageStackStorageSlot::Pointer) ==
- sizeof(decltype(Ipc::Mem::PageId::number)), "page indexing types are consistent");
+/*
+
+Ipc::Mem::IdSet and related code maintains a perfect full binary tree structure:
+
+ (l,r)
+ /\
+ (ll,lr) (rl,rr)
+ /\ /\
+ L1 L2 L3 L4
+
+where
+
+ * (l,r) is an always-present root node;
+ * inner nodes, including the root one, count the total number of available
+ IDs in the leaf nodes of the left and right subtrees (e.g., r = rl + rr);
+ * leaf nodes are bitsets of available IDs (e.g., rl = number of 1s in L3);
+ all leaf nodes are always present.
+
+The above sample tree would be stored as seven 64-bit atomic integers:
+ (l,r), (ll,lr), (rl,rr), L1, L2, L3, L4
+
+*/
+
+namespace Ipc
+{
+
+namespace Mem
+{
+
+/// the maximum number of pages that a leaf node can store
+static const IdSet::size_type BitsPerLeaf = 64;
+
+class IdSetPosition
+{
+public:
+ using size_type = IdSet::size_type;
+
+ IdSetPosition() = default; ///< root node position
+ IdSetPosition(size_type aLevel, size_type anOffset);
+
+ /// whether we are at the top of the tree
+ bool atRoot() const { return !level && !offset; }
+
+ /// which direction is this position from our parent node
+ IdSetNavigationDirection ascendDirection() const;
+
+ /// the number of levels above us (e.g., zero for the root node)
+ IdSet::size_type level = 0;
+ /// the number of nodes (at our level) to the left of us
+ IdSet::size_type offset = 0;
+};
+
+/// a helper class to perform inner node manipulation for IdSet
+class IdSetInnerNode
+{
+public:
+ using size_type = IdSet::size_type;
+ typedef uint64_t Packed; ///< (atomically) stored serialized value
+
+ /// de-serializes a given value
+ static IdSetInnerNode Unpack(Packed packed);
+
+ IdSetInnerNode() = default;
+ IdSetInnerNode(size_type left, size_type right);
+
+ /// returns a serializes value suitable for shared memory storage
+ Packed pack() const { return (static_cast(left) << 32) | right; }
+
+ size_type left = 0; ///< the number of available IDs in the left subtree
+ size_type right = 0; ///< the number of available IDs in the right subtree
+};
+
+} // namespace Mem
+
+} // namespace Ipc
+
+/* Ipc::Mem::IdSetPosition */
+
+Ipc::Mem::IdSetPosition::IdSetPosition(size_type aLevel, size_type anOffset):
+ level(aLevel),
+ offset(anOffset)
+{
+}
+
+Ipc::Mem::IdSetNavigationDirection
+Ipc::Mem::IdSetPosition::ascendDirection() const
+{
+ return (offset % 2 == 0) ? dirLeft : dirRight;
+}
+
+/* Ipc::Mem::IdSetMeasurements */
+
+Ipc::Mem::IdSetMeasurements::IdSetMeasurements(const size_type aCapacity)
+{
+ capacity = aCapacity;
+
+ // For simplicity, we want a perfect full binary tree with root and leaves.
+ // We could compute all this with log2() calls, but rounding and honoring
+ // root+leaves minimums make that approach more complex than this fast loop.
+ requestedLeafNodeCount = (capacity + (BitsPerLeaf-1))/BitsPerLeaf;
+ treeHeight = 1+1; // the root level plus the leaf nodes level
+ leafNodeCount = 2; // the root node can have only two leaf nodes
+ while (leafNodeCount < requestedLeafNodeCount) {
+ leafNodeCount *= 2;
+ ++treeHeight;
+ }
+ innerLevelCount = treeHeight - 1;
+
+ debugs(54, 5, "rounded capacity up from " << capacity << " to " << (leafNodeCount*BitsPerLeaf));
+
+ // we do (1 << level) when computing 32-bit IdSetInnerNode::left
+ assert(treeHeight < 32);
+}
+
+/* Ipc::Mem::IdSetInnerNode */
+
+Ipc::Mem::IdSetInnerNode::IdSetInnerNode(size_type aLeft, size_type aRight):
+ left(aLeft),
+ right(aRight)
+{
+}
+
+Ipc::Mem::IdSetInnerNode
+Ipc::Mem::IdSetInnerNode::Unpack(Packed packed)
+{
+ // truncation during the cast is intentional here
+ return IdSetInnerNode(packed >> 32, static_cast(packed));
+}
+
+/* Ipc::Mem::IdSet */
+
+Ipc::Mem::IdSet::IdSet(const size_type capacity):
+ measurements(capacity),
+ nodes_(capacity)
+{
+ // For valueAddress() to be able to return a raw uint64_t pointer, the
+ // atomic wrappers in nodes_ must be zero-size. Check the best we can. Once.
+ static_assert(sizeof(StoredNode) == sizeof(Node), "atomic locks use no storage");
+ assert(StoredNode().is_lock_free());
+}
void
-Ipc::Mem::PageStackStorageSlot::take()
+Ipc::Mem::IdSet::makeFullBeforeSharing()
{
- const auto nxt = nextOrMarker.exchange(TakenPage);
- assert(nxt != TakenPage);
+ // initially, all IDs are marked as available
+ fillAllNodes();
+
+ // ... but IDs beyond the requested capacity should not be available
+ if (measurements.capacity != measurements.leafNodeCount*BitsPerLeaf)
+ truncateExtras();
}
+/// populates the entire allocated tree with available IDs
+/// may exceed the requested capacity; \see truncateExtras()
void
-Ipc::Mem::PageStackStorageSlot::put(const PointerOrMarker expected, const Pointer nxt)
+Ipc::Mem::IdSet::fillAllNodes()
{
- assert(nxt != TakenPage);
- const auto old = nextOrMarker.exchange(nxt);
- assert(old == expected);
+ // leaf nodes
+ auto pos = Position(measurements.treeHeight-1, 0);
+ const auto allOnes = ~uint64_t(0);
+ std::fill_n(valueAddress(pos), measurements.leafNodeCount, allOnes);
+
+ // inner nodes, starting from the bottom of the tree
+ auto nodesAtLevel = measurements.leafNodeCount/2;
+ auto pagesBelow = BitsPerLeaf;
+ do {
+ pos = ascend(pos);
+ const auto value = IdSetInnerNode(pagesBelow, pagesBelow).pack();
+ std::fill_n(valueAddress(pos), nodesAtLevel, value);
+ nodesAtLevel /= 2;
+ pagesBelow *= 2;
+ } while (!pos.atRoot());
}
-/* Ipc::Mem::PageStack */
+/// effectively removes IDs that exceed the requested capacity after makeFull()
+void
+Ipc::Mem::IdSet::truncateExtras()
+{
+ // leaf nodes
+ // start with the left-most leaf that should have some 0s; it may even have
+ // no 1s at all (i.e. be completely unused)
+ auto pos = Position(measurements.treeHeight-1, measurements.capacity/BitsPerLeaf);
+ leafTruncate(pos, measurements.capacity % BitsPerLeaf);
+ const auto rightLeaves = measurements.leafNodeCount - measurements.requestedLeafNodeCount;
+ // this zeroing of the leaf nodes to the right from pos is only necessary to
+ // trigger asserts if the code dealing with the inner node counters is buggy
+ if (rightLeaves > 1)
+ std::fill_n(valueAddress(pos) + 1, rightLeaves-1, 0);
+
+ // inner nodes, starting from the bottom of the tree; optimization: only
+ // adjusting nodes on the way up from the first leaf-with-0s position
+ auto toSubtract = BitsPerLeaf - (measurements.capacity % BitsPerLeaf);
+ do {
+ const auto direction = pos.ascendDirection();
+ pos = ascend(pos);
+ toSubtract = innerTruncate(pos, direction, toSubtract);
+ } while (!pos.atRoot());
+}
-Ipc::Mem::PageStack::PageStack(const PoolId aPoolId, const PageCount aCapacity, const size_t aPageSize):
- thePoolId(aPoolId), capacity_(aCapacity), thePageSize(aPageSize),
- size_(0),
- head_(Slot::NilPtr),
- slots_(aCapacity)
+/// fill the leaf node at a given position with 0s, leaving only idsToKeep IDs
+void
+Ipc::Mem::IdSet::leafTruncate(const Position pos, const size_type idsToKeep)
+{
+ Node &node = *valueAddress(pos); // no auto to simplify the asserts() below
+ assert(node == std::numeric_limits::max()); // all 1s
+ static_assert(std::is_unsigned::value, "right shift prepends 0s");
+ node >>= BitsPerLeaf - idsToKeep;
+ // node can be anything here, including all 0s and all 1s
+}
+
+/// accounts for toSubtract IDs removal from a subtree in the given direction of
+/// the given position
+/// \returns the number of IDs to subtract from the parent node
+Ipc::Mem::IdSet::size_type
+Ipc::Mem::IdSet::innerTruncate(const Position pos, const NavigationDirection dir, const size_type toSubtract)
+{
+ auto *valuePtr = valueAddress(pos);
+ auto value = IdSetInnerNode::Unpack(*valuePtr);
+ size_type toSubtractNext = 0;
+ if (dir == dirLeft) {
+ toSubtractNext = toSubtract + value.right;
+ assert(value.left >= toSubtract);
+ value.left -= toSubtract;
+ value.right = 0;
+ } else {
+ assert(dir == dirRight);
+ toSubtractNext = toSubtract;
+ assert(value.right >= toSubtract);
+ // value.left is unchanged; we have only adjusted the right branch
+ value.right -= toSubtract;
+ }
+ *valuePtr = value.pack();
+ return toSubtractNext;
+}
+
+/// accounts for an ID added to subtree in the given dir from the given position
+void
+Ipc::Mem::IdSet::innerPush(const Position pos, const NavigationDirection dir)
+{
+ // either left or right component will be true/1; the other will be false/0
+ const auto increment = IdSetInnerNode(dir == dirLeft, dir == dirRight).pack();
+ const auto previousValue = nodeAt(pos).fetch_add(increment);
+ // no overflows
+ assert(previousValue <= std::numeric_limits::max() - increment);
+}
+
+/// accounts for future ID removal from a subtree of the given position
+/// \returns the direction of the subtree chosen to relinquish the ID
+Ipc::Mem::IdSet::NavigationDirection
+Ipc::Mem::IdSet::innerPop(const Position pos)
+{
+ NavigationDirection direction = dirNone;
+
+ auto &node = nodeAt(pos);
+ auto oldValue = node.load();
+ IdSetInnerNode newValue;
+ do {
+ newValue = IdSetInnerNode::Unpack(oldValue);
+ if (newValue.left) {
+ --newValue.left;
+ direction = dirLeft;
+ } else if (newValue.right) {
+ --newValue.right;
+ direction = dirRight;
+ } else {
+ return dirEnd;
+ }
+ } while (!node.compare_exchange_weak(oldValue, newValue.pack()));
+
+ assert(direction == dirLeft || direction == dirRight);
+ return direction;
+}
+
+/// adds the given ID to the leaf node at the given position
+void
+Ipc::Mem::IdSet::leafPush(const Position pos, const size_type id)
+{
+ const auto mask = Node(1) << (id % BitsPerLeaf);
+ const auto oldValue = nodeAt(pos).fetch_or(mask);
+ // this was a new entry
+ assert((oldValue & mask) == 0);
+}
+
+// TODO: After switching to C++20, use countr_zero() which may compile to a
+// single TZCNT assembly instruction on modern CPUs.
+/// a temporary C++20 countr_zero() replacement
+static inline
+int trailingZeros(uint64_t x)
+{
+ if (!x)
+ return 64;
+ int count = 0;
+ for (uint64_t mask = 1; !(x & mask); mask <<= 1)
+ ++count;
+ return count;
+}
+
+/// extracts and returns an ID from the leaf node at the given position
+Ipc::Mem::IdSet::size_type
+Ipc::Mem::IdSet::leafPop(const Position pos)
+{
+ auto &node = nodeAt(pos);
+ auto oldValue = node.load();
+ Node newValue;
+ do {
+ assert(oldValue > 0);
+ const auto mask = oldValue - 1; // flips the rightmost 1 and trailing 0s
+ newValue = oldValue & mask; // clears the rightmost 1
+ } while (!node.compare_exchange_weak(oldValue, newValue));
+
+ return pos.offset*BitsPerLeaf + trailingZeros(oldValue);
+}
+
+/// \returns the position of a parent node of the node at the given position
+Ipc::Mem::IdSet::Position
+Ipc::Mem::IdSet::ascend(Position pos)
+{
+ assert(pos.level > 0);
+ --pos.level;
+ pos.offset /= 2;
+ return pos;
+}
+
+/// \returns the position of a child node in the given direction of the parent
+/// node at the given position
+Ipc::Mem::IdSet::Position
+Ipc::Mem::IdSet::descend(Position pos, const NavigationDirection direction)
{
- assert(thePoolId);
+ assert(pos.level < measurements.treeHeight);
+ ++pos.level;
- assert(capacity_ < Slot::TakenPage);
- assert(capacity_ < Slot::NilPtr);
+ pos.offset *= 2;
+ if (direction == dirRight)
+ ++pos.offset;
+ else
+ assert(direction == dirLeft);
- // initially, all pages are free
- if (capacity_) {
- const auto lastIndex = capacity_-1;
- // FlexibleArray cannot construct its phantom elements so, technically,
- // all slots (except the very first one) are uninitialized until now.
- for (Slot::Pointer i = 0; i < lastIndex; ++i)
- (void)new(&slots_[i])Slot(i+1);
- (void)new(&slots_[lastIndex])Slot(Slot::NilPtr);
- size_ = capacity_;
- head_ = 0;
+ return pos;
+}
+
+/// \returns the atomic node (either inner or leaf) at the given position
+Ipc::Mem::IdSet::StoredNode &
+Ipc::Mem::IdSet::nodeAt(const Position pos)
+{
+ assert(pos.level < measurements.treeHeight);
+ // n = 2^(h+1) - 1 with h = level-1
+ const auto nodesAbove = (1U << pos.level) - 1;
+
+ // the second clause is for the special case of a root node
+ assert(pos.offset < nodesAbove*2 || (pos.atRoot() && nodesAbove == 0));
+ const auto nodesToTheLeft = pos.offset;
+
+ const size_t nodesBefore = nodesAbove + nodesToTheLeft;
+ assert(nodesBefore < measurements.nodeCount());
+ return nodes_[nodesBefore];
+}
+
+/// \returns the location of the raw (inner or leaf) node at the given position
+Ipc::Mem::IdSet::Node *
+Ipc::Mem::IdSet::valueAddress(const Position pos)
+{
+ // IdSet() constructor asserts that this frequent reinterpret_cast is safe
+ return &reinterpret_cast(nodeAt(pos));
+}
+
+bool
+Ipc::Mem::IdSet::pop(size_type &id)
+{
+ Position rootPos;
+ const auto directionFromRoot = innerPop(rootPos);
+ if (directionFromRoot == dirEnd)
+ return false; // an empty tree
+
+ auto pos = descend(rootPos, directionFromRoot);
+ for (size_t level = 1; level < measurements.innerLevelCount; ++level) {
+ const auto direction = innerPop(pos);
+ pos = descend(pos, direction);
+ }
+
+ id = leafPop(pos);
+ return true;
+}
+
+void
+Ipc::Mem::IdSet::push(const size_type id)
+{
+ const auto offsetAtLeafLevel = id/BitsPerLeaf;
+ auto pos = Position(measurements.innerLevelCount, offsetAtLeafLevel);
+ leafPush(pos, id);
+
+ do {
+ const auto direction = pos.ascendDirection();
+ pos = ascend(pos);
+ innerPush(pos, direction);
+ } while (!pos.atRoot());
+}
+
+size_t
+Ipc::Mem::IdSet::MemorySize(const size_type capacity)
+{
+ const IdSetMeasurements measurements(capacity);
+ // Adding sizeof(IdSet) double-counts the first node but it is better to
+ // overestimate (a little) than to underestimate our memory needs due to
+ // padding, new data members, etc.
+ return sizeof(IdSet) + measurements.nodeCount() * sizeof(StoredNode);
+}
+
+/* Ipc::Mem::PageStack */
+
+Ipc::Mem::PageStack::PageStack(const Config &config):
+ config_(config),
+ size_(0),
+ ids_(config_.capacity)
+{
+ if (config.createFull) {
+ ids_.makeFullBeforeSharing();
+ size_ = config_.capacity;
}
}
@@ -66,23 +442,21 @@
{
assert(!page);
- Slot::Pointer current = head_.load();
+ if (!config_.capacity)
+ return false;
- auto nextFree = Slot::NilPtr;
- do {
- if (current == Slot::NilPtr)
- return false;
- nextFree = slots_[current].next();
- } while (!head_.compare_exchange_weak(current, nextFree));
+ IdSet::size_type pageIndex = 0;
+ if (!ids_.pop(pageIndex))
+ return false;
// must decrement after removing the page to avoid underflow
const auto newSize = --size_;
- assert(newSize < capacity_);
+ assert(newSize < config_.capacity);
- slots_[current].take();
- page.number = current + 1;
- page.pool = thePoolId;
+ page.number = pageIndex + 1;
+ page.pool = config_.poolId;
debugs(54, 8, page << " size: " << newSize);
+ assert(pageIdIsValid(page));
return true;
}
@@ -93,19 +467,12 @@
assert(page);
assert(pageIdIsValid(page));
- const auto pageIndex = page.number - 1;
- auto &slot = slots_[pageIndex];
-
// must increment before inserting the page to avoid underflow in pop()
const auto newSize = ++size_;
- assert(newSize <= capacity_);
+ assert(newSize <= config_.capacity);
- auto current = head_.load();
- auto expected = Slot::TakenPage;
- do {
- slot.put(expected, current);
- expected = current;
- } while (!head_.compare_exchange_weak(current, pageIndex));
+ const auto pageIndex = page.number - 1;
+ ids_.push(pageIndex);
debugs(54, 8, page << " size: " << newSize);
page = PageId();
@@ -114,34 +481,37 @@
bool
Ipc::Mem::PageStack::pageIdIsValid(const PageId &page) const
{
- return page.pool == thePoolId &&
+ return page.pool == config_.poolId &&
0 < page.number && page.number <= capacity();
}
size_t
Ipc::Mem::PageStack::sharedMemorySize() const
{
- return SharedMemorySize(thePoolId, capacity_, thePageSize);
+ return SharedMemorySize(config_);
}
size_t
-Ipc::Mem::PageStack::SharedMemorySize(const PoolId, const PageCount capacity, const size_t pageSize)
+Ipc::Mem::PageStack::SharedMemorySize(const Config &cfg)
{
const auto levelsSize = PageId::maxPurpose * sizeof(Levels_t);
- const size_t pagesDataSize = capacity * pageSize;
- return StackSize(capacity) + LevelsPaddingSize(capacity) + levelsSize + pagesDataSize;
+ const size_t pagesDataSize = cfg.capacity * cfg.pageSize;
+ return StackSize(cfg.capacity) + pagesDataSize + levelsSize;
}
size_t
Ipc::Mem::PageStack::StackSize(const PageCount capacity)
{
- return sizeof(PageStack) + capacity * sizeof(Slot);
+ // Adding sizeof(PageStack) double-counts the fixed portion of the ids_ data
+ // member but it is better to overestimate (a little) than to underestimate
+ // our memory needs due to padding, new data members, etc.
+ return sizeof(PageStack) + IdSet::MemorySize(capacity);
}
size_t
Ipc::Mem::PageStack::stackSize() const
{
- return StackSize(capacity_);
+ return StackSize(config_.capacity);
}
size_t
diff -u -r -N squid-5.0.2/src/ipc/mem/PageStack.h squid-5.0.3/src/ipc/mem/PageStack.h
--- squid-5.0.2/src/ipc/mem/PageStack.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ipc/mem/PageStack.h 2020-06-08 03:33:42.000000000 +1200
@@ -23,39 +23,83 @@
class PageId;
-/// reflects the dual nature of PageStack storage:
-/// - for free pages, this is a pointer to the next free page
-/// - for used pages, this is a "used page" marker
-class PageStackStorageSlot
+class IdSetPosition;
+typedef enum { dirNone, dirLeft, dirRight, dirEnd } IdSetNavigationDirection;
+
+/// basic IdSet storage parameters, extracted here to keep them constant
+class IdSetMeasurements
+{
+public:
+ /// we need to fit two size_type counters into one 64-bit lockless atomic
+ typedef uint32_t size_type;
+
+ explicit IdSetMeasurements(size_type capacity);
+
+ /// the maximum number of pages our tree is allowed to store
+ size_type capacity = 0;
+
+ /// the number of leaf nodes that satisfy capacity requirements
+ size_type requestedLeafNodeCount = 0;
+
+ size_type treeHeight = 0; ///< total number of levels, including the leaf level
+ size_type leafNodeCount = 0; ///< the number of nodes at the leaf level
+ size_type innerLevelCount = 0; ///< all levels except the leaf level
+
+ /// the total number of nodes at all levels
+ size_type nodeCount() const { return leafNodeCount ? leafNodeCount*2 -1 : 0; }
+};
+
+/// a shareable set of positive uint32_t IDs with O(1) insertion/removal ops
+class IdSet
{
public:
- // We are using uint32_t for Pointer because PageId::number is uint32_t.
- // PageId::number should probably be uint64_t to accommodate caches with
- // page numbers exceeding UINT32_MAX.
- typedef uint32_t PointerOrMarker;
- typedef PointerOrMarker Pointer;
- typedef PointerOrMarker Marker;
-
- /// represents a nil next slot pointer
- static const Pointer NilPtr = std::numeric_limits::max();
- /// marks a slot of a used (i.e. take()n) page
- static const Marker TakenPage = std::numeric_limits::max() - 1;
- static_assert(TakenPage != NilPtr, "magic PointerOrMarker values do not clash");
-
- explicit PageStackStorageSlot(const Pointer nxt = NilPtr): nextOrMarker(nxt) {}
-
- /// returns a (possibly nil) pointer to the next free page
- Pointer next() const { return nextOrMarker.load(); }
-
- /// marks our page as used
- void take();
-
- /// marks our page as free, to be used before the given `nxt` page;
- /// also checks that the slot state matches the caller expectations
- void put(const PointerOrMarker expected, const Pointer nxt);
+ using size_type = IdSetMeasurements::size_type;
+ using Position = IdSetPosition;
+ using NavigationDirection = IdSetNavigationDirection;
+
+ /// memory size required to store a tree with the given capacity
+ static size_t MemorySize(size_type capacity);
+
+ explicit IdSet(size_type capacity);
+
+ /// populates the allocated tree with the requested capacity IDs
+ /// optimized to run without atomic protection
+ void makeFullBeforeSharing();
+
+ /// finds/extracts (into the given `id`) an ID value and returns true
+ /// \retval false no IDs are left
+ bool pop(size_type &id);
+
+ /// makes `id` value available to future pop() callers
+ void push(size_type id);
+
+ const IdSetMeasurements measurements;
private:
- std::atomic nextOrMarker;
+ typedef uint64_t Node; ///< either leaf or intermediate node
+ typedef std::atomic StoredNode; ///< a Node stored in shared memory
+
+ /* optimization: these initialization methods bypass atomic protections */
+ void fillAllNodes();
+ void truncateExtras();
+ Node *valueAddress(Position);
+ size_type innerTruncate(Position pos, NavigationDirection dir, size_type toSubtract);
+ void leafTruncate(Position pos, size_type idsToKeep);
+
+ void innerPush(Position, NavigationDirection);
+ NavigationDirection innerPop(Position);
+
+ void leafPush(Position, size_type id);
+ size_type leafPop(Position);
+
+ Position ascend(Position);
+ Position descend(Position, NavigationDirection);
+
+ StoredNode &nodeAt(Position);
+
+ /// the entire binary tree flattened into an array
+ FlexibleArray nodes_;
+ // No more data members should follow! See FlexibleArray<> for details.
};
/// Atomic container of "free" PageIds. Detects (some) double-free bugs.
@@ -72,10 +116,24 @@
/// the number of (free and/or used) pages in a stack
typedef unsigned int PageCount;
- PageStack(const PoolId aPoolId, const PageCount aCapacity, const size_t aPageSize);
+ // XXX: poolId, pageSize look misplaced due to messy separation of PagePool
+ // (which should support multiple Segments but does not) and PageStack
+ // (which should not calculate the Segment size but does) duties.
+ /// PageStack construction and SharedMemorySize calculation parameters
+ class Config {
+ public:
+ uint32_t poolId = 0; ///< pool ID
+ size_t pageSize = 0; ///< page size, used to calculate shared memory size
+ PageCount capacity = 0; ///< the maximum number of pages
+
+ /// whether a newly created PageStack should be prefilled with PageIds
+ bool createFull = false;
+ };
+
+ explicit PageStack(const Config &);
- PageCount capacity() const { return capacity_; }
- size_t pageSize() const { return thePageSize; }
+ PageCount capacity() const { return config_.capacity; }
+ size_t pageSize() const { return config_.pageSize; }
/// an approximate number of free pages
PageCount size() const { return size_.load(); }
@@ -87,7 +145,7 @@
bool pageIdIsValid(const PageId &page) const;
/// total shared memory size required to share
- static size_t SharedMemorySize(const PoolId aPoolId, const PageCount capacity, const size_t pageSize);
+ static size_t SharedMemorySize(const Config &);
size_t sharedMemorySize() const;
/// shared memory size required only by PageStack, excluding
@@ -97,7 +155,7 @@
/// \returns the number of padding bytes to align PagePool::theLevels array
static size_t LevelsPaddingSize(const PageCount capacity);
- size_t levelsPaddingSize() const { return LevelsPaddingSize(capacity_); }
+ size_t levelsPaddingSize() const { return LevelsPaddingSize(config_.capacity); }
/**
* The following functions return PageStack IDs for the corresponding
@@ -113,23 +171,12 @@
static PoolId IdForSwapDirSpace(const int dirIdx) { return 900 + dirIdx + 1; }
private:
- using Slot = PageStackStorageSlot;
-
- // XXX: theFoo members look misplaced due to messy separation of PagePool
- // (which should support multiple Segments but does not) and PageStack
- // (which should not calculate the Segment size but does) duties.
- const PoolId thePoolId; ///< pool ID
- const PageCount capacity_; ///< the maximum number of pages
- const size_t thePageSize; ///< page size, used to calculate shared memory size
+ const Config config_;
/// a lower bound for the number of free pages (for debugging purposes)
std::atomic size_;
- /// the index of the first free stack element or nil
- std::atomic head_;
-
- /// slots indexed using their page number
- Ipc::Mem::FlexibleArray slots_;
- // No more data members should follow! See Ipc::Mem::FlexibleArray<> for details.
+ IdSet ids_; ///< free pages (valid with positive capacity_)
+ // No more data members should follow! See IdSet for details.
};
} // namespace Mem
diff -u -r -N squid-5.0.2/src/log/DB/log_db_daemon.8 squid-5.0.3/src/log/DB/log_db_daemon.8
--- squid-5.0.2/src/log/DB/log_db_daemon.8 2020-04-20 00:20:18.000000000 +1200
+++ squid-5.0.3/src/log/DB/log_db_daemon.8 2020-06-09 19:09:22.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "LOG_DB_DAEMON 8"
-.TH LOG_DB_DAEMON 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH LOG_DB_DAEMON 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/Makefile.am squid-5.0.3/src/Makefile.am
--- squid-5.0.2/src/Makefile.am 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/Makefile.am 2020-06-08 03:33:42.000000000 +1200
@@ -576,7 +576,6 @@
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
$(top_builddir)/lib/libmiscutil.la \
- $(ATOMICLIB) \
$(SSLLIB) \
$(EPOLL_LIBS) \
$(MINGW_LIBS) \
diff -u -r -N squid-5.0.2/src/Makefile.in squid-5.0.3/src/Makefile.in
--- squid-5.0.2/src/Makefile.in 2020-04-20 00:08:51.000000000 +1200
+++ squid-5.0.3/src/Makefile.in 2020-06-09 18:58:21.000000000 +1200
@@ -454,8 +454,8 @@
$(top_builddir)/lib/libmiscutil.la $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
- $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_3) \
- $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_6)
+ $(am__DEPENDENCIES_3) $(am__DEPENDENCIES_1) \
+ $(am__DEPENDENCIES_6)
squid_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
$(CXXFLAGS) $(squid_LDFLAGS) $(LDFLAGS) -o $@
@@ -3179,9 +3179,9 @@
$(ESI_LIBS) $(SNMP_LIBS) mem/libmem.la store/libstore.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
- $(top_builddir)/lib/libmiscutil.la $(ATOMICLIB) $(SSLLIB) \
- $(EPOLL_LIBS) $(MINGW_LIBS) $(KRB5LIBS) $(SYSTEMD_LIBS) \
- $(COMPAT_LIB) $(XTRA_LIBS) $(am__append_9)
+ $(top_builddir)/lib/libmiscutil.la $(SSLLIB) $(EPOLL_LIBS) \
+ $(MINGW_LIBS) $(KRB5LIBS) $(SYSTEMD_LIBS) $(COMPAT_LIB) \
+ $(XTRA_LIBS) $(am__append_9)
@ENABLE_LOADABLE_MODULES_TRUE@squid_LDFLAGS = -export-dynamic -dlopen force
unlinkd_SOURCES = unlinkd_daemon.cc
unlinkd_LDADD = \
diff -u -r -N squid-5.0.2/src/mem/PoolingAllocator.h squid-5.0.3/src/mem/PoolingAllocator.h
--- squid-5.0.2/src/mem/PoolingAllocator.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/mem/PoolingAllocator.h 2020-06-08 03:33:42.000000000 +1200
@@ -22,6 +22,23 @@
template PoolingAllocator(const PoolingAllocator &) noexcept {}
value_type *allocate(std::size_t n) { return static_cast(memAllocRigid(n*sizeof(value_type))); }
void deallocate(value_type *vp, std::size_t n) noexcept { memFreeRigid(vp, n*sizeof(value_type)); }
+
+ // The following declarations are only necessary for compilers that do not
+ // fully support C++11 Allocator-related APIs, such as GCC v4.8.
+ // TODO: Remove after dropping support for such compilers.
+
+ using size_type = size_t;
+ using pointer = Value*;
+ using const_pointer = const Value*;
+ using reference = Value&;
+ using const_reference = const Value&;
+
+ template
+ struct rebind {
+ typedef PoolingAllocator other;
+ };
+
+ template void destroy(OtherValue *p) { p->~OtherValue(); }
};
template
diff -u -r -N squid-5.0.2/src/MemStore.cc squid-5.0.3/src/MemStore.cc
--- squid-5.0.2/src/MemStore.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/MemStore.cc 2020-06-08 03:33:42.000000000 +1200
@@ -1008,10 +1008,14 @@
const int64_t entryLimit = MemStore::EntryLimit();
assert(entryLimit > 0);
+
+ Ipc::Mem::PageStack::Config spaceConfig;
+ spaceConfig.poolId = Ipc::Mem::PageStack::IdForMemStoreSpace(),
+ spaceConfig.pageSize = 0; // the pages are stored in Ipc::Mem::Pages
+ spaceConfig.capacity = entryLimit;
+ spaceConfig.createFull = true; // all pages are initially available
Must(!spaceOwner);
- spaceOwner = shm_new(Ipc::Mem::PageStack)(SpaceLabel,
- Ipc::Mem::PageStack::IdForMemStoreSpace(),
- entryLimit, 0);
+ spaceOwner = shm_new(Ipc::Mem::PageStack)(SpaceLabel, spaceConfig);
Must(!mapOwner);
mapOwner = MemStoreMap::Init(MapLabel, entryLimit);
Must(!extrasOwner);
diff -u -r -N squid-5.0.2/src/neighbors.cc squid-5.0.3/src/neighbors.cc
--- squid-5.0.2/src/neighbors.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/neighbors.cc 2020-06-08 03:33:42.000000000 +1200
@@ -63,6 +63,8 @@
static void peerCountMcastPeersDone(void *data);
static void peerCountMcastPeersStart(void *data);
static void peerCountMcastPeersSchedule(CachePeer * p, time_t when);
+static void peerCountMcastPeersAbort(PeerSelector *);
+static void peerCountMcastPeersCreateAndSend(CachePeer *p);
static IRCB peerCountHandleIcpReply;
static void neighborIgnoreNonPeer(const Ip::Address &, icp_opcode);
@@ -1393,9 +1395,19 @@
static void
peerCountMcastPeersStart(void *data)
{
+ const auto peer = static_cast(data);
+ CallContextCreator([peer] {
+ peerCountMcastPeersCreateAndSend(peer);
+ });
+ peerCountMcastPeersSchedule(peer, MCAST_COUNT_RATE);
+}
+
+/// initiates an ICP transaction to a multicast peer
+static void
+peerCountMcastPeersCreateAndSend(CachePeer * const p)
+{
// XXX: Do not create lots of complex fake objects (while abusing their
// APIs) to pass around a few basic data points like start_ping and ping!
- CachePeer *p = (CachePeer *)data;
MemObject *mem;
int reqnum;
// TODO: use class AnyP::Uri instead of constructing and re-parsing a string
@@ -1408,6 +1420,9 @@
const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initPeerMcast);
auto *req = HttpRequest::FromUrlXXX(url, mx);
assert(req != nullptr);
+ const AccessLogEntry::Pointer ale = new AccessLogEntry;
+ ale->request = req;
+ CodeContext::Reset(ale);
StoreEntry *fake = storeCreateEntry(url, url, RequestFlags(), Http::METHOD_GET);
const auto psstate = new PeerSelector(nullptr);
psstate->request = req;
@@ -1415,6 +1430,7 @@
psstate->entry = fake;
psstate->peerCountMcastPeerXXX = cbdataReference(p);
psstate->ping.start = current_time;
+ psstate->al = ale;
mem = fake->mem_obj;
mem->request = psstate->request;
mem->start_ping = current_time;
@@ -1431,13 +1447,23 @@
psstate,
Config.Timeout.mcast_icp_query / 1000.0, 1);
p->mcast.flags.counting = true;
- peerCountMcastPeersSchedule(p, MCAST_COUNT_RATE);
}
static void
peerCountMcastPeersDone(void *data)
{
const auto psstate = static_cast(data);
+ CallBack(psstate->al, [psstate] {
+ peerCountMcastPeersAbort(psstate);
+ delete psstate;
+ });
+}
+
+/// ends counting of multicast ICP replies
+/// to the ICP query initiated by peerCountMcastPeersCreateAndSend()
+static void
+peerCountMcastPeersAbort(PeerSelector * const psstate)
+{
StoreEntry *fake = psstate->entry;
if (cbdataReferenceValid(psstate->peerCountMcastPeerXXX)) {
@@ -1455,7 +1481,6 @@
fake->abort(); // sets ENTRY_ABORTED and initiates releated cleanup
fake->mem_obj->request = nullptr;
fake->unlock("peerCountMcastPeersDone");
- delete psstate;
}
static void
@@ -1786,7 +1811,7 @@
* Send HTCP CLR messages to all peers configured to receive them.
*/
void
-neighborsHtcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &method, htcp_clr_reason reason)
+neighborsHtcpClear(StoreEntry * e, HttpRequest * req, const HttpRequestMethod &method, htcp_clr_reason reason)
{
CachePeer *p;
char buf[128];
@@ -1802,7 +1827,7 @@
continue;
}
debugs(15, 3, "neighborsHtcpClear: sending CLR to " << p->in_addr.toUrl(buf, 128));
- htcpClear(e, uri, req, method, p, reason);
+ htcpClear(e, req, method, p, reason);
}
}
diff -u -r -N squid-5.0.2/src/neighbors.h squid-5.0.3/src/neighbors.h
--- squid-5.0.2/src/neighbors.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/neighbors.h 2020-06-08 03:33:42.000000000 +1200
@@ -40,7 +40,7 @@
void neighborAdd(const char *, const char *, int, int, int, int, int);
void neighbors_init(void);
#if USE_HTCP
-void neighborsHtcpClear(StoreEntry *, const char *, HttpRequest *, const HttpRequestMethod &, htcp_clr_reason);
+void neighborsHtcpClear(StoreEntry *, HttpRequest *, const HttpRequestMethod &, htcp_clr_reason);
#endif
CachePeer *peerFindByName(const char *);
CachePeer *peerFindByNameAndPort(const char *, unsigned short);
diff -u -r -N squid-5.0.2/src/ResolvedPeers.cc squid-5.0.3/src/ResolvedPeers.cc
--- squid-5.0.2/src/ResolvedPeers.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ResolvedPeers.cc 2020-06-08 03:33:42.000000000 +1200
@@ -24,59 +24,117 @@
{
debugs(17, 4, path);
assert(path);
- paths_.emplace(paths_.begin(), path);
+ // Cannot use pathsToSkip for a faster (reverse) search because there
+ // may be unavailable paths past pathsToSkip. We could remember
+ // the last extraction index, but, to completely avoid a linear search,
+ // extract*() methods should return the path index.
+ const auto found = std::find_if(paths_.begin(), paths_.end(),
+ [path](const ResolvedPeerPath &candidate) {
+ return candidate.connection == path; // (refcounted) pointer comparison
+ });
+ assert(found != paths_.end());
+ assert(!found->available);
+ found->available = true;
+ increaseAvailability();
+
+ // if we restored availability of a path that we used to skip, update
+ const auto pathsToTheLeft = static_cast(found - paths_.begin());
+ if (pathsToTheLeft < pathsToSkip) {
+ pathsToSkip = pathsToTheLeft;
+ } else {
+ // *found was unavailable so pathsToSkip could not end at it
+ Must(pathsToTheLeft != pathsToSkip);
+ }
}
void
ResolvedPeers::addPath(const Comm::ConnectionPointer &path)
{
paths_.emplace_back(path);
+ Must(paths_.back().available); // no pathsToSkip updates are needed
+ increaseAvailability();
+}
+
+/// \returns the beginning iterator for any available-path search
+ResolvedPeers::Paths::iterator
+ResolvedPeers::start()
+{
+ Must(pathsToSkip <= paths_.size());
+ return paths_.begin() + pathsToSkip; // may return end()
+}
+
+/// finalizes the iterator part of the given preliminary find*() result
+ResolvedPeers::Finding
+ResolvedPeers::makeFinding(const Paths::iterator &path, bool foundOther)
+{
+ return std::make_pair((foundOther ? paths_.end() : path), foundOther);
+}
+
+/// \returns the first available same-peer same-family Finding or
+ResolvedPeers::Finding
+ResolvedPeers::findPrime(const Comm::Connection ¤tPeer)
+{
+ const auto path = start();
+ const auto foundNextOrSpare = path != paths_.end() &&
+ (currentPeer.getPeer() != path->connection->getPeer() || // next peer
+ ConnectionFamily(currentPeer) != ConnectionFamily(*path->connection));
+ return makeFinding(path, foundNextOrSpare);
+}
+
+/// \returns the first available same-peer different-family Finding or
+ResolvedPeers::Finding
+ResolvedPeers::findSpare(const Comm::Connection ¤tPeer)
+{
+ const auto primeFamily = ConnectionFamily(currentPeer);
+ const auto primePeer = currentPeer.getPeer();
+ const auto path = std::find_if(start(), paths_.end(),
+ [primeFamily, primePeer](const ResolvedPeerPath &candidate) {
+ if (!candidate.available)
+ return false;
+ if (primePeer != candidate.connection->getPeer())
+ return true; // found next peer
+ if (primeFamily != ConnectionFamily(*candidate.connection))
+ return true; // found spare
+ return false;
+ });
+ const auto foundNext = path != paths_.end() &&
+ primePeer != path->connection->getPeer();
+ return makeFinding(path, foundNext);
+}
+
+/// \returns the first available same-peer Finding or
+ResolvedPeers::Finding
+ResolvedPeers::findPeer(const Comm::Connection ¤tPeer)
+{
+ const auto path = start();
+ const auto foundNext = path != paths_.end() &&
+ currentPeer.getPeer() != path->connection->getPeer();
+ return makeFinding(path, foundNext);
}
Comm::ConnectionPointer
ResolvedPeers::extractFront()
{
Must(!empty());
- return extractFound("first: ", paths_.begin());
+ return extractFound("first: ", start());
}
Comm::ConnectionPointer
ResolvedPeers::extractPrime(const Comm::Connection ¤tPeer)
{
- if (!empty()) {
- const auto peerToMatch = currentPeer.getPeer();
- const auto familyToMatch = ConnectionFamily(currentPeer);
- const auto &conn = paths_.front();
- if (conn->getPeer() == peerToMatch && familyToMatch == ConnectionFamily(*conn))
- return extractFound("same-peer same-family match: ", paths_.begin());
- }
+ const auto found = findPrime(currentPeer).first;
+ if (found != paths_.end())
+ return extractFound("same-peer same-family match: ", found);
debugs(17, 7, "no same-peer same-family paths");
return nullptr;
}
-/// If spare paths exist for currentPeer, returns the first spare path iterator.
-/// Otherwise, if there are paths for other peers, returns one of those.
-/// Otherwise, returns the end() iterator.
-Comm::ConnectionList::iterator
-ResolvedPeers::findSpareOrNextPeer(const Comm::Connection ¤tPeer)
-{
- const auto peerToMatch = currentPeer.getPeer();
- const auto familyToAvoid = ConnectionFamily(currentPeer);
- // Optimization: Also stop at the first mismatching peer because all
- // same-peer paths are grouped together.
- return std::find_if(paths_.begin(), paths_.end(),
- [peerToMatch, familyToAvoid](const Comm::ConnectionPointer &conn) {
- return peerToMatch != conn->getPeer() ||
- familyToAvoid != ConnectionFamily(*conn);
- });
-}
-
Comm::ConnectionPointer
ResolvedPeers::extractSpare(const Comm::Connection ¤tPeer)
{
- auto found = findSpareOrNextPeer(currentPeer);
- if (found != paths_.end() && currentPeer.getPeer() == (*found)->getPeer())
+ const auto found = findSpare(currentPeer).first;
+ if (found != paths_.end())
return extractFound("same-peer different-family match: ", found);
debugs(17, 7, "no same-peer different-family paths");
@@ -85,48 +143,58 @@
/// convenience method to finish a successful extract*() call
Comm::ConnectionPointer
-ResolvedPeers::extractFound(const char *description, const Comm::ConnectionList::iterator &found)
+ResolvedPeers::extractFound(const char *description, const Paths::iterator &found)
{
- const auto path = *found;
- paths_.erase(found);
- debugs(17, 7, description << path);
- return path;
+ auto &path = *found;
+ debugs(17, 7, description << path.connection);
+ assert(path.available);
+ path.available = false;
+ decreaseAvailability();
+
+ // if we extracted the left-most available path, find the next one
+ if (static_cast(found - paths_.begin()) == pathsToSkip) {
+ while (++pathsToSkip < paths_.size() && !paths_[pathsToSkip].available) {}
+ }
+
+ return path.connection;
}
bool
ResolvedPeers::haveSpare(const Comm::Connection ¤tPeer)
{
- const auto found = findSpareOrNextPeer(currentPeer);
- return found != paths_.end() &&
- currentPeer.getPeer() == (*found)->getPeer();
+ return findSpare(currentPeer).first != paths_.end();
+}
+
+/// whether paths_ have no (and will have no) Xs for the current peer based on
+/// the given findX(current peer) result
+bool
+ResolvedPeers::doneWith(const Finding &findings) const
+{
+ if (findings.first != paths_.end())
+ return false; // not done because the caller found a viable path X
+
+ // The caller did not find any path X. If the caller found any "other"
+ // paths, then we are done with paths X. If there are no "other" paths,
+ // then destinationsFinalized is the answer.
+ return findings.second ? true : destinationsFinalized;
}
bool
ResolvedPeers::doneWithSpares(const Comm::Connection ¤tPeer)
{
- const auto found = findSpareOrNextPeer(currentPeer);
- if (found == paths_.end())
- return destinationsFinalized;
- return currentPeer.getPeer() != (*found)->getPeer();
+ return doneWith(findSpare(currentPeer));
}
bool
-ResolvedPeers::doneWithPrimes(const Comm::Connection ¤tPeer) const
+ResolvedPeers::doneWithPrimes(const Comm::Connection ¤tPeer)
{
- const auto first = paths_.begin();
- if (first == paths_.end())
- return destinationsFinalized;
- return currentPeer.getPeer() != (*first)->getPeer() ||
- ConnectionFamily(currentPeer) != ConnectionFamily(**first);
+ return doneWith(findPrime(currentPeer));
}
bool
-ResolvedPeers::doneWithPeer(const Comm::Connection ¤tPeer) const
+ResolvedPeers::doneWithPeer(const Comm::Connection ¤tPeer)
{
- const auto first = paths_.begin();
- if (first == paths_.end())
- return destinationsFinalized;
- return currentPeer.getPeer() != (*first)->getPeer();
+ return doneWith(findPeer(currentPeer));
}
int
@@ -135,6 +203,22 @@
return conn.remote.isIPv4() ? AF_INET : AF_INET6;
}
+/// increments the number of available paths
+void
+ResolvedPeers::increaseAvailability()
+{
+ ++availablePaths;
+ assert(availablePaths <= paths_.size());
+}
+
+/// decrements the number of available paths
+void
+ResolvedPeers::decreaseAvailability()
+{
+ assert(availablePaths > 0);
+ --availablePaths;
+}
+
std::ostream &
operator <<(std::ostream &os, const ResolvedPeers &peers)
{
diff -u -r -N squid-5.0.2/src/ResolvedPeers.h squid-5.0.3/src/ResolvedPeers.h
--- squid-5.0.2/src/ResolvedPeers.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ResolvedPeers.h 2020-06-08 03:33:42.000000000 +1200
@@ -13,6 +13,16 @@
#include "comm/forward.h"
#include
+#include
+
+class ResolvedPeerPath
+{
+public:
+ explicit ResolvedPeerPath(const Comm::ConnectionPointer &conn) : connection(conn), available(true) {}
+
+ Comm::ConnectionPointer connection; ///< (the address of) a path
+ bool available; ///< whether this path may be used (i.e., has not been tried already)
+};
/// cache_peer and origin server addresses (a.k.a. paths)
/// selected and resolved by the peering code
@@ -21,17 +31,20 @@
MEMPROXY_CLASS(ResolvedPeers);
public:
+ // ResolvedPeerPaths in addPath() call order
+ typedef std::vector Paths;
+ using size_type = Paths::size_type;
typedef RefCount Pointer;
ResolvedPeers();
/// whether we lack any known candidate paths
- bool empty() const { return paths_.empty(); }
+ bool empty() const { return !availablePaths; }
/// add a candidate path to try after all the existing paths
void addPath(const Comm::ConnectionPointer &);
- /// add a candidate path to try before all the existing paths
+ /// re-inserts the previously extracted address into the same position
void retryPath(const Comm::ConnectionPointer &);
/// extracts and returns the first queued address
@@ -49,16 +62,16 @@
bool haveSpare(const Comm::Connection ¤tPeer);
/// whether extractPrime() returns and will continue to return nil
- bool doneWithPrimes(const Comm::Connection ¤tPeer) const;
+ bool doneWithPrimes(const Comm::Connection ¤tPeer);
/// whether extractSpare() returns and will continue to return nil
bool doneWithSpares(const Comm::Connection ¤tPeer);
/// whether doneWithPrimes() and doneWithSpares() are true for currentPeer
- bool doneWithPeer(const Comm::Connection ¤tPeer) const;
+ bool doneWithPeer(const Comm::Connection ¤tPeer);
/// the current number of candidate paths
- Comm::ConnectionList::size_type size() const { return paths_.size(); }
+ size_type size() const { return availablePaths; }
/// whether all of the available candidate paths received from DNS
bool destinationsFinalized = false;
@@ -67,13 +80,34 @@
bool notificationPending = false;
private:
+ /// A find*() result: An iterator of the found path (or paths_.end()) and
+ /// whether the "other" path was found instead.
+ typedef std::pair Finding;
+
/// The protocol family of the given path, AF_INET or AF_INET6
static int ConnectionFamily(const Comm::Connection &conn);
- Comm::ConnectionList::iterator findSpareOrNextPeer(const Comm::Connection ¤tPeer);
- Comm::ConnectionPointer extractFound(const char *description, const Comm::ConnectionList::iterator &found);
+ Paths::iterator start();
+ Finding findSpare(const Comm::Connection ¤tPeer);
+ Finding findPrime(const Comm::Connection ¤tPeer);
+ Finding findPeer(const Comm::Connection ¤tPeer);
+ Comm::ConnectionPointer extractFound(const char *description, const Paths::iterator &found);
+ Finding makeFinding(const Paths::iterator &found, bool foundOther);
+
+ bool doneWith(const Finding &findings) const;
+
+ void increaseAvailability();
+ void decreaseAvailability();
+
+ Paths paths_; ///< resolved addresses in (peer, family) order
+
+ /// the number of leading paths_ elements that are all currently unavailable
+ /// i.e. the size of the front paths_ segment comprised of unavailable items
+ /// i.e. the position of the first available path (or paths_.size())
+ size_type pathsToSkip = 0;
- Comm::ConnectionList paths_;
+ /// the total number of currently available elements in paths_
+ size_type availablePaths = 0;
};
/// summarized ResolvedPeers (for debugging)
diff -u -r -N squid-5.0.2/src/security/BlindPeerConnector.cc squid-5.0.3/src/security/BlindPeerConnector.cc
--- squid-5.0.2/src/security/BlindPeerConnector.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/security/BlindPeerConnector.cc 2020-06-08 03:33:42.000000000 +1200
@@ -7,6 +7,7 @@
*/
#include "squid.h"
+#include "AccessLogEntry.h"
#include "CachePeer.h"
#include "comm/Connection.h"
#include "errorpage.h"
diff -u -r -N squid-5.0.2/src/security/cert_validators/fake/security_fake_certverify.8 squid-5.0.3/src/security/cert_validators/fake/security_fake_certverify.8
--- squid-5.0.2/src/security/cert_validators/fake/security_fake_certverify.8 2020-04-20 00:20:18.000000000 +1200
+++ squid-5.0.3/src/security/cert_validators/fake/security_fake_certverify.8 2020-06-09 19:09:23.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "SECURITY_FAKE_CERTVERIFY 8"
-.TH SECURITY_FAKE_CERTVERIFY 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH SECURITY_FAKE_CERTVERIFY 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/security/Handshake.cc squid-5.0.3/src/security/Handshake.cc
--- squid-5.0.2/src/security/Handshake.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/security/Handshake.cc 2020-06-08 03:33:42.000000000 +1200
@@ -106,11 +106,11 @@
} // namespace Security
-/// Convenience helper: We parse ProtocolVersion but store "int".
+/// parse TLS ProtocolVersion (uint16) and convert it to AnyP::ProtocolVersion
static AnyP::ProtocolVersion
-ParseProtocolVersion(Parser::BinaryTokenizer &tk)
+ParseProtocolVersion(Parser::BinaryTokenizer &tk, const char *contextLabel = ".version")
{
- Parser::BinaryTokenizerContext context(tk, ".version");
+ Parser::BinaryTokenizerContext context(tk, contextLabel);
uint8_t vMajor = tk.uint8(".major");
uint8_t vMinor = tk.uint8(".minor");
if (vMajor == 0 && vMinor == 2)
@@ -187,10 +187,11 @@
/* Security::HandshakeParser */
-Security::HandshakeParser::HandshakeParser():
+Security::HandshakeParser::HandshakeParser(const MessageSource source):
details(new TlsDetails),
state(atHelloNone),
resumingSession(false),
+ messageSource(source),
currentContentType(0),
done(nullptr),
expectingModernRecords(false)
@@ -285,12 +286,19 @@
{
Must(currentContentType == ContentType::ctChangeCipherSpec);
// We are currently ignoring Change Cipher Spec Protocol messages.
- skipMessage("ChangeCipherCpec msg [fragment]");
+ skipMessage("ChangeCipherSpec msg [fragment]");
+
+ // In TLS v1.2 and earlier, ChangeCipherSpec is sent after Hello (when
+ // tlsSupportedVersion is already known) and indicates session resumption.
+ // In later TLS versions, ChangeCipherSpec may be sent before and after
+ // Hello, but it is unused for session resumption and should be ignored.
+ if (!details->tlsSupportedVersion || Tls1p3orLater(details->tlsSupportedVersion))
+ return;
- // Everything after the ChangeCipherCpec message may be encrypted.
- // Continuing parsing is pointless. Stop here.
resumingSession = true;
- done = "ChangeCipherCpec";
+
+ // Everything after the ChangeCipherSpec message may be encrypted. Stop.
+ done = "ChangeCipherSpec in v1.2-";
}
void
@@ -316,14 +324,19 @@
switch (message.msg_type) {
case HandshakeType::hskClientHello:
Must(state < atHelloReceived);
+ Must(messageSource == fromClient);
Security::HandshakeParser::parseClientHelloHandshakeMessage(message.msg_body);
state = atHelloReceived;
done = "ClientHello";
return;
case HandshakeType::hskServerHello:
Must(state < atHelloReceived);
+ Must(messageSource == fromServer);
parseServerHelloHandshakeMessage(message.msg_body);
state = atHelloReceived;
+ // for TLSv1.3 and later, anything after the server Hello is encrypted
+ if (Tls1p3orLater(details->tlsSupportedVersion))
+ done = "ServerHello in v1.3+";
return;
case HandshakeType::hskCertificate:
Must(state < atCertificatesReceived);
@@ -424,6 +437,10 @@
case 35: // SessionTicket TLS Extension; RFC 5077
details->tlsTicketsExtension = true;
details->hasTlsTicket = !extension.data.isEmpty();
+ break;
+ case 43: // supported_versions extension; RFC 8446
+ parseSupportedVersionsExtension(extension.data);
+ break;
case 13172: // Next Protocol Negotiation Extension (expired draft?)
default:
break;
@@ -504,6 +521,78 @@
return SBuf(); // SNI extension lacks host_name
}
+/// RFC 8446 Section 4.2.1: SupportedVersions extension
+void
+Security::HandshakeParser::parseSupportedVersionsExtension(const SBuf &extensionData) const
+{
+ // Upon detecting a quoted RFC MUST violation, this parser immediately
+ // returns, ignoring the entire extension and resulting in Squid relying on
+ // the legacy_version field value or another (valid) supported_versions
+ // extension. The alternative would be to reject the whole handshake as
+ // invalid. Deployment experience will show which alternative is the best.
+
+ // Please note that several of these MUSTs also imply certain likely
+ // handling of a hypothetical next TLS version (e.g., v1.4).
+
+ // RFC 8446 Section 4.1.2:
+ // In TLS 1.3, the client indicates its version preferences in the
+ // "supported_versions" extension (Section 4.2.1) and the legacy_version
+ // field MUST be set to 0x0303, which is the version number for TLS 1.2.
+ //
+ // RFC 8446 Section 4.2.1:
+ // A server which negotiates TLS 1.3 MUST respond by sending a
+ // "supported_versions" extension containing the selected version value
+ // (0x0304). It MUST set the ServerHello.legacy_version field to 0x0303
+ // (TLS 1.2).
+ //
+ // Ignore supported_versions senders violating legacy_version MUSTs above:
+ if (details->tlsSupportedVersion != AnyP::ProtocolVersion(AnyP::PROTO_TLS, 1, 2))
+ return;
+
+ AnyP::ProtocolVersion supportedVersionMax;
+ if (messageSource == fromClient) {
+ Parser::BinaryTokenizer tkList(extensionData);
+ Parser::BinaryTokenizer tkVersions(tkList.pstring8("SupportedVersions"));
+ while (!tkVersions.atEnd()) {
+ const auto version = ParseProtocolVersion(tkVersions, "supported_version");
+ if (!supportedVersionMax || TlsVersionEarlierThan(supportedVersionMax, version))
+ supportedVersionMax = version;
+ }
+
+ // ignore empty supported_versions
+ if (!supportedVersionMax)
+ return;
+
+ // supportedVersionMax here may be "earlier" than tlsSupportedVersion: A
+ // TLS v1.3 client may try to negotiate a _legacy_ version X with a TLS
+ // v1.3 server by sending supported_versions containing just X.
+ } else {
+ assert(messageSource == fromServer);
+ Parser::BinaryTokenizer tkVersion(extensionData);
+ const auto version = ParseProtocolVersion(tkVersion, "selected_version");
+ // RFC 8446 Section 4.2.1:
+ // A server which negotiates a version of TLS prior to TLS 1.3 [...]
+ // MUST NOT send the "supported_versions" extension.
+ if (Tls1p2orEarlier(version))
+ return;
+ supportedVersionMax = version;
+ }
+
+ // We overwrite Hello-derived legacy_version because the following MUSTs
+ // indicate that it is ignored in the presence of valid supported_versions
+ // as far as the negotiated version is concerned. For simplicity sake, we
+ // may also overwrite previous valid supported_versions extensions (if any).
+ //
+ // RFC 8446 Section 4.2.1:
+ // If this extension is present in the ClientHello, servers MUST NOT use the
+ // ClientHello.legacy_version value for version negotiation and MUST use
+ // only the "supported_versions" extension to determine client preferences.
+ // Servers MUST only select a version of TLS present in that extension
+ debugs(83, 7, "found " << supportedVersionMax);
+ assert(supportedVersionMax);
+ details->tlsSupportedVersion = supportedVersionMax;
+}
+
void
Security::HandshakeParser::skipMessage(const char *description)
{
@@ -647,6 +736,9 @@
#if defined(TLSEXT_TYPE_next_proto_neg) // 13172
extensions.insert(TLSEXT_TYPE_next_proto_neg);
#endif
+#if defined(TLSEXT_TYPE_supported_versions) // 43
+ extensions.insert(TLSEXT_TYPE_supported_versions);
+#endif
/*
* OpenSSL does not support these last extensions by default, but those
diff -u -r -N squid-5.0.2/src/security/Handshake.h squid-5.0.3/src/security/Handshake.h
--- squid-5.0.2/src/security/Handshake.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/security/Handshake.h 2020-06-08 03:33:42.000000000 +1200
@@ -29,7 +29,11 @@
std::ostream & print(std::ostream &os) const;
AnyP::ProtocolVersion tlsVersion; ///< The TLS hello message version
- AnyP::ProtocolVersion tlsSupportedVersion; ///< The requested/used TLS version
+
+ /// For most compliant TLS v1.3+ agents, this is supported_versions maximum.
+ /// For others agents, this is the legacy_version field.
+ AnyP::ProtocolVersion tlsSupportedVersion;
+
bool compressionSupported; ///< The requested/used compressed method
SBuf serverName; ///< The SNI hostname, if any
bool doHeartBeats;
@@ -59,7 +63,10 @@
/// The parsing states
typedef enum {atHelloNone = 0, atHelloStarted, atHelloReceived, atCertificatesReceived, atHelloDoneReceived, atNstReceived, atCcsReceived, atFinishReceived} ParserState;
- HandshakeParser();
+ /// the originator of the TLS handshake being parsed
+ typedef enum { fromClient = 0, fromServer } MessageSource;
+
+ explicit HandshakeParser(MessageSource);
/// Parses the initial sequence of raw bytes sent by the TLS/SSL agent.
/// Returns true upon successful completion (e.g., got HelloDone).
@@ -67,7 +74,7 @@
/// Throws on errors.
bool parseHello(const SBuf &data);
- TlsDetails::Pointer details; ///< TLS handshake meta info or nil.
+ TlsDetails::Pointer details; ///< TLS handshake meta info. Never nil.
Security::CertList serverCertificates; ///< parsed certificates chain
@@ -75,6 +82,9 @@
bool resumingSession; ///< True if this is a resuming session
+ /// whether we are parsing Server or Client TLS handshake messages
+ MessageSource messageSource;
+
private:
bool isSslv2Record(const SBuf &raw) const;
void parseRecord();
@@ -96,6 +106,7 @@
bool parseCompressionMethods(const SBuf &raw);
void parseExtensions(const SBuf &raw);
SBuf parseSniExtension(const SBuf &extensionData) const;
+ void parseSupportedVersionsExtension(const SBuf &extensionData) const;
void parseCiphers(const SBuf &raw);
void parseV23Ciphers(const SBuf &raw);
@@ -120,6 +131,40 @@
YesNoNone expectingModernRecords;
};
+/// whether the given protocol belongs to the TLS/SSL group of protocols
+inline bool
+TlsFamilyProtocol(const AnyP::ProtocolVersion &version)
+{
+ return (version.protocol == AnyP::PROTO_TLS || version.protocol == AnyP::PROTO_SSL);
+}
+
+/// whether TLS/SSL protocol `a` precedes TLS/SSL protocol `b`
+inline bool
+TlsVersionEarlierThan(const AnyP::ProtocolVersion &a, const AnyP::ProtocolVersion &b)
+{
+ Must(TlsFamilyProtocol(a));
+ Must(TlsFamilyProtocol(b));
+
+ if (a.protocol == b.protocol)
+ return a < b;
+
+ return a.protocol == AnyP::PROTO_SSL; // implies that b is TLS
+}
+
+/// whether the given TLS/SSL protocol is TLS v1.2 or earlier, including SSL
+inline bool
+Tls1p2orEarlier(const AnyP::ProtocolVersion &p)
+{
+ return TlsVersionEarlierThan(p, AnyP::ProtocolVersion(AnyP::PROTO_TLS, 1, 3));
+}
+
+/// whether the given TLS/SSL protocol is TLS v1.3 or later
+inline bool
+Tls1p3orLater(const AnyP::ProtocolVersion &p)
+{
+ return !Tls1p2orEarlier(p);
+}
+
}
#endif // SQUID_SECURITY_HANDSHAKE_H
diff -u -r -N squid-5.0.2/src/security/NegotiationHistory.cc squid-5.0.3/src/security/NegotiationHistory.cc
--- squid-5.0.2/src/security/NegotiationHistory.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/security/NegotiationHistory.cc 2020-06-08 03:33:42.000000000 +1200
@@ -25,7 +25,7 @@
const char *
Security::NegotiationHistory::printTlsVersion(AnyP::ProtocolVersion const &v) const
{
- if (v.protocol != AnyP::PROTO_SSL && v.protocol != AnyP::PROTO_TLS)
+ if (!TlsFamilyProtocol(v))
return nullptr;
static char buf[512];
diff -u -r -N squid-5.0.2/src/security/PeerConnector.cc squid-5.0.3/src/security/PeerConnector.cc
--- squid-5.0.2/src/security/PeerConnector.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/security/PeerConnector.cc 2020-06-08 03:33:42.000000000 +1200
@@ -15,8 +15,11 @@
#include "Downloader.h"
#include "errorpage.h"
#include "fde.h"
+#include "FwdState.h"
#include "http/Stream.h"
#include "HttpRequest.h"
+#include "neighbors.h"
+#include "pconn.h"
#include "security/NegotiationHistory.h"
#include "security/PeerConnector.h"
#include "SquidConfig.h"
@@ -31,6 +34,7 @@
Security::PeerConnector::PeerConnector(const Comm::ConnectionPointer &aServerConn, AsyncCall::Pointer &aCallback, const AccessLogEntryPointer &alp, const time_t timeout) :
AsyncJob("Security::PeerConnector"),
+ noteFwdPconnUse(false),
serverConn(aServerConn),
al(alp),
callback(aCallback),
@@ -39,14 +43,17 @@
useCertValidator_(true),
certsDownloads(0)
{
- debugs(83, 5, "Security::PeerConnector constructed, this=" << (void*)this);
+ debugs(83, 5, serverConn);
+
// if this throws, the caller's cb dialer is not our CbDialer
Must(dynamic_cast(callback->getDialer()));
-}
-Security::PeerConnector::~PeerConnector()
-{
- debugs(83, 5, "Security::PeerConnector destructed, this=" << (void*)this);
+ // watch for external connection closures
+ Must(Comm::IsConnOpen(serverConn));
+ Must(!fd_table[serverConn->fd].closing());
+ typedef CommCbMemFunT Dialer;
+ closeHandler = JobCallback(9, 5, Dialer, this, Security::PeerConnector::commCloseHandler);
+ comm_add_close_handler(serverConn->fd, closeHandler);
}
bool Security::PeerConnector::doneAll() const
@@ -61,8 +68,16 @@
AsyncJob::start();
debugs(83, 5, "this=" << (void*)this);
+ // we own this Comm::Connection object and its fd exclusively, but must bail
+ // if others started closing the socket while we were waiting to start()
+ assert(Comm::IsConnOpen(serverConn));
+ if (fd_table[serverConn->fd].closing()) {
+ bail(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw(), al));
+ return;
+ }
+
Security::SessionPointer tmp;
- if (prepareSocket() && initialize(tmp))
+ if (initialize(tmp))
negotiate();
else
mustStop("Security::PeerConnector TLS socket initialize failed");
@@ -71,34 +86,25 @@
void
Security::PeerConnector::commCloseHandler(const CommCloseCbParams ¶ms)
{
+ closeHandler = nullptr;
+
debugs(83, 5, "FD " << params.fd << ", Security::PeerConnector=" << params.data);
- connectionClosed("Security::PeerConnector::commCloseHandler");
+ const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request.getRaw(), al);
+#if USE_OPENSSL
+ err->detail = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, nullptr, nullptr);
+#endif
+ bail(err);
}
void
-Security::PeerConnector::connectionClosed(const char *reason)
+Security::PeerConnector::commTimeoutHandler(const CommTimeoutCbParams &)
{
- debugs(83, 5, reason << " socket closed/closing. this=" << (void*)this);
- mustStop(reason);
- callback = NULL;
-}
-
-bool
-Security::PeerConnector::prepareSocket()
-{
- debugs(83, 5, serverConnection() << ", this=" << (void*)this);
- if (!Comm::IsConnOpen(serverConnection()) || fd_table[serverConnection()->fd].closing()) {
- connectionClosed("Security::PeerConnector::prepareSocket");
- return false;
- }
-
- debugs(83, 5, serverConnection());
-
- // watch for external connection closures
- typedef CommCbMemFunT Dialer;
- closeHandler = JobCallback(9, 5, Dialer, this, Security::PeerConnector::commCloseHandler);
- comm_add_close_handler(serverConnection()->fd, closeHandler);
- return true;
+ debugs(83, 5, serverConnection() << " timedout. this=" << (void*)this);
+ const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scGatewayTimeout, request.getRaw(), al);
+#if USE_OPENSSL
+ err->detail = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, nullptr, nullptr);
+#endif
+ bail(err);
}
bool
@@ -163,9 +169,6 @@
void
Security::PeerConnector::negotiate()
{
- if (!Comm::IsConnOpen(serverConnection()))
- return;
-
const int fd = serverConnection()->fd;
if (fd_table[fd].closing())
return;
@@ -204,7 +207,7 @@
if (!sslFinalized())
return;
- callBack();
+ sendSuccess();
}
bool
@@ -241,7 +244,6 @@
noteNegotiationDone(anErr);
bail(anErr);
- serverConn->close();
return true;
}
}
@@ -259,9 +261,6 @@
Ssl::ErrorDetail *errDetails = NULL;
bool validatorFailed = false;
- if (!Comm::IsConnOpen(serverConnection())) {
- return;
- }
if (Debug::Enabled(83, 5)) {
Security::SessionPointer ssl(fd_table[serverConnection()->fd].ssl);
@@ -281,7 +280,7 @@
if (!errDetails && !validatorFailed) {
noteNegotiationDone(NULL);
- callBack();
+ sendSuccess();
return;
}
@@ -296,7 +295,6 @@
noteNegotiationDone(anErr);
bail(anErr);
- serverConn->close();
return;
}
#endif
@@ -473,9 +471,11 @@
#endif
// read timeout to avoid getting stuck while reading from a silent server
- AsyncCall::Pointer nil;
+ typedef CommCbMemFunT TimeoutDialer;
+ AsyncCall::Pointer timeoutCall = JobCallback(83, 5,
+ TimeoutDialer, this, Security::PeerConnector::commTimeoutHandler);
const auto timeout = Comm::MortalReadTimeout(startTime, negotiationTimeout);
- commSetConnTimeout(serverConnection(), timeout, nil);
+ commSetConnTimeout(serverConnection(), timeout, timeoutCall);
Comm::SetSelect(fd, COMM_SELECT_READ, &NegotiateSsl, new Pointer(this), 0);
}
@@ -545,13 +545,34 @@
Must(dialer);
dialer->answer().error = error;
+ if (const auto p = serverConnection()->getPeer())
+ peerConnectFailed(p);
+
+ callBack();
+ disconnect();
+
+ if (noteFwdPconnUse)
+ fwdPconnPool->noteUses(fd_table[serverConn->fd].pconn.uses);
+ serverConn->close();
+ serverConn = nullptr;
+}
+
+void
+Security::PeerConnector::sendSuccess()
+{
callBack();
- // Our job is done. The callabck recepient will probably close the failed
- // peer connection and try another peer or go direct (if possible). We
- // can close the connection ourselves (our error notification would reach
- // the recepient before the fd-closure notification), but we would rather
- // minimize the number of fd-closure notifications and let the recepient
- // manage the TCP state of the connection.
+ disconnect();
+}
+
+void
+Security::PeerConnector::disconnect()
+{
+ if (closeHandler) {
+ comm_remove_close_handler(serverConnection()->fd, closeHandler);
+ closeHandler = nullptr;
+ }
+
+ commUnsetConnTimeout(serverConnection());
}
void
@@ -563,10 +584,6 @@
// Do this now so that if we throw below, swanSong() assert that we _tried_
// to call back holds.
callback = NULL; // this should make done() true
-
- // remove close handler
- comm_remove_close_handler(serverConnection()->fd, closeHandler);
-
CbDialer *dialer = dynamic_cast(cb->getDialer());
Must(dialer);
dialer->answer().conn = serverConnection();
diff -u -r -N squid-5.0.2/src/security/PeerConnector.h squid-5.0.3/src/security/PeerConnector.h
--- squid-5.0.2/src/security/PeerConnector.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/security/PeerConnector.h 2020-06-08 03:33:42.000000000 +1200
@@ -31,34 +31,12 @@
{
/**
- * Initiates encryption on a connection to peers or servers.
- * Despite its name does not perform any connect(2) operations.
+ * Initiates encryption of a given open TCP connection to a peer or server.
+ * Despite its name does not perform any connect(2) operations. Owns the
+ * connection during TLS negotiations. The caller receives EncryptorAnswer.
*
* Contains common code and interfaces of various specialized PeerConnector's,
* including peer certificate validation code.
- \par
- * The caller receives a call back with Security::EncryptorAnswer. If answer.error
- * is not nil, then there was an error and the encryption to the peer or server
- * was not fully established. The error object is suitable for error response
- * generation.
- \par
- * The caller must monitor the connection for closure because this
- * job will not inform the caller about such events.
- \par
- * PeerConnector class currently supports a form of TLS negotiation timeout,
- * which is accounted only when sets the read timeout from encrypted peers/servers.
- * For a complete solution, the caller must monitor the overall connection
- * establishment timeout and close the connection on timeouts. This is probably
- * better than having dedicated (or none at all!) timeouts for peer selection,
- * DNS lookup, TCP handshake, SSL handshake, etc. Some steps may have their
- * own timeout, but not all steps should be forced to have theirs.
- * XXX: tunnel.cc and probably other subsystems do not have an "overall
- * connection establishment" timeout. We need to change their code so that they
- * start monitoring earlier and close on timeouts. This change may need to be
- * discussed on squid-dev.
- \par
- * This job never closes the connection, even on errors. If a 3rd-party
- * closes the connection, this job simply quits without informing the caller.
*/
class PeerConnector: virtual public AsyncJob
{
@@ -81,7 +59,10 @@
AsyncCall::Pointer &aCallback,
const AccessLogEntryPointer &alp,
const time_t timeout = 0);
- virtual ~PeerConnector();
+ virtual ~PeerConnector() = default;
+
+ /// hack: whether the connection requires fwdPconnPool->noteUses()
+ bool noteFwdPconnUse;
protected:
// AsyncJob API
@@ -90,17 +71,12 @@
virtual void swanSong();
virtual const char *status() const;
+ /// The connection read timeout callback handler.
+ void commTimeoutHandler(const CommTimeoutCbParams &);
+
/// The comm_close callback handler.
void commCloseHandler(const CommCloseCbParams ¶ms);
- /// Inform us that the connection is closed. Does the required clean-up.
- void connectionClosed(const char *reason);
-
- /// Sets up TCP socket-related notification callbacks if things go wrong.
- /// If socket already closed return false, else install the comm_close
- /// handler to monitor the socket.
- bool prepareSocket();
-
/// \returns true on successful TLS session initialization
virtual bool initialize(Security::SessionPointer &);
@@ -160,12 +136,18 @@
/// mimics FwdState to minimize changes to FwdState::initiate/negotiateSsl
Comm::ConnectionPointer const &serverConnection() const { return serverConn; }
- void bail(ErrorState *error); ///< Return an error to the PeerConnector caller
+ /// sends the given error to the initiator
+ void bail(ErrorState *error);
+
+ /// sends the encrypted connection to the initiator
+ void sendSuccess();
- /// Callback the caller class, and pass the ready to communicate secure
- /// connection or an error if PeerConnector failed.
+ /// a bail(), sendSuccess() helper: sends results to the initiator
void callBack();
+ /// a bail(), sendSuccess() helper: stops monitoring the connection
+ void disconnect();
+
/// If called the certificates validator will not used
void bypassCertValidator() {useCertValidator_ = false;}
diff -u -r -N squid-5.0.2/src/ssl/bio.cc squid-5.0.3/src/ssl/bio.cc
--- squid-5.0.2/src/ssl/bio.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ssl/bio.cc 2020-06-08 03:33:42.000000000 +1200
@@ -250,7 +250,8 @@
parsedHandshake(false),
parseError(false),
bumpMode_(bumpNone),
- rbufConsumePos(0)
+ rbufConsumePos(0),
+ parser_(Security::HandshakeParser::fromServer)
{
}
@@ -554,6 +555,13 @@
return parser_.resumingSession;
}
+bool
+Ssl::ServerBio::encryptedCertificates() const
+{
+ return parser_.details->tlsSupportedVersion &&
+ Security::Tls1p3orLater(parser_.details->tlsSupportedVersion);
+}
+
/// initializes BIO table after allocation
static int
squid_bio_create(BIO *bi)
@@ -717,6 +725,12 @@
SSL_set_options(ssl, SSL_OP_NO_COMPRESSION);
#endif
+#if defined(SSL_OP_NO_TLSv1_3)
+ // avoid "inappropriate fallback" OpenSSL error messages
+ if (details->tlsSupportedVersion && Security::Tls1p2orEarlier(details->tlsSupportedVersion))
+ SSL_set_options(ssl, SSL_OP_NO_TLSv1_3);
+#endif
+
#if defined(TLSEXT_STATUSTYPE_ocsp)
if (details->tlsStatusRequest)
SSL_set_tlsext_status_type(ssl, TLSEXT_STATUSTYPE_ocsp);
diff -u -r -N squid-5.0.2/src/ssl/bio.h squid-5.0.3/src/ssl/bio.h
--- squid-5.0.2/src/ssl/bio.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ssl/bio.h 2020-06-08 03:33:42.000000000 +1200
@@ -143,6 +143,10 @@
bool resumingSession();
+ /// whether the server encrypts its certificate (e.g., TLS v1.3)
+ /// \retval false the server uses plain certs or its intent is unknown
+ bool encryptedCertificates() const;
+
/// The write hold state
bool holdWrite() const {return holdWrite_;}
/// Enables or disables the write hold state
diff -u -r -N squid-5.0.2/src/ssl/ErrorDetail.cc squid-5.0.3/src/ssl/ErrorDetail.cc
--- squid-5.0.2/src/ssl/ErrorDetail.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ssl/ErrorDetail.cc 2020-06-08 03:33:42.000000000 +1200
@@ -233,6 +233,9 @@
"X509_V_ERR_SUBTREE_MINMAX"
},
#endif
+ { X509_V_ERR_APPLICATION_VERIFICATION, //50
+ "X509_V_ERR_APPLICATION_VERIFICATION"
+ },
#if defined(X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE)
{
X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE, //51
@@ -257,9 +260,132 @@
"X509_V_ERR_CRL_PATH_VALIDATION_ERROR"
},
#endif
- { X509_V_ERR_APPLICATION_VERIFICATION,
- "X509_V_ERR_APPLICATION_VERIFICATION"
+#if defined(X509_V_ERR_PATH_LOOP)
+ {
+ X509_V_ERR_PATH_LOOP, //55
+ "X509_V_ERR_PATH_LOOP"
+ },
+#endif
+#if defined(X509_V_ERR_SUITE_B_INVALID_VERSION)
+ {
+ X509_V_ERR_SUITE_B_INVALID_VERSION, //56
+ "X509_V_ERR_SUITE_B_INVALID_VERSION"
+ },
+#endif
+#if defined(X509_V_ERR_SUITE_B_INVALID_ALGORITHM)
+ {
+ X509_V_ERR_SUITE_B_INVALID_ALGORITHM, //57
+ "X509_V_ERR_SUITE_B_INVALID_ALGORITHM"
+ },
+#endif
+#if defined(X509_V_ERR_SUITE_B_INVALID_CURVE)
+ {
+ X509_V_ERR_SUITE_B_INVALID_CURVE, //58
+ "X509_V_ERR_SUITE_B_INVALID_CURVE"
+ },
+#endif
+#if defined(X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM)
+ {
+ X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM, //59
+ "X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM"
+ },
+#endif
+#if defined(X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED)
+ {
+ X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED, //60
+ "X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED"
+ },
+#endif
+#if defined(X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256)
+ {
+ X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256, //61
+ "X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256"
+ },
+#endif
+#if defined(X509_V_ERR_HOSTNAME_MISMATCH)
+ {
+ X509_V_ERR_HOSTNAME_MISMATCH, //62
+ "X509_V_ERR_HOSTNAME_MISMATCH"
+ },
+#endif
+#if defined(X509_V_ERR_EMAIL_MISMATCH)
+ {
+ X509_V_ERR_EMAIL_MISMATCH, //63
+ "X509_V_ERR_EMAIL_MISMATCH"
+ },
+#endif
+#if defined(X509_V_ERR_IP_ADDRESS_MISMATCH)
+ {
+ X509_V_ERR_IP_ADDRESS_MISMATCH, //64
+ "X509_V_ERR_IP_ADDRESS_MISMATCH"
+ },
+#endif
+#if defined(X509_V_ERR_DANE_NO_MATCH)
+ {
+ X509_V_ERR_DANE_NO_MATCH, //65
+ "X509_V_ERR_DANE_NO_MATCH"
},
+#endif
+#if defined(X509_V_ERR_EE_KEY_TOO_SMALL)
+ {
+ X509_V_ERR_EE_KEY_TOO_SMALL, //66
+ "X509_V_ERR_EE_KEY_TOO_SMALL"
+ },
+#endif
+#if defined(X509_V_ERR_CA_KEY_TOO_SMALL)
+ {
+ X509_V_ERR_CA_KEY_TOO_SMALL, //67
+ "X509_V_ERR_CA_KEY_TOO_SMALL"
+ },
+#endif
+#if defined(X509_V_ERR_CA_MD_TOO_WEAK)
+ {
+ X509_V_ERR_CA_MD_TOO_WEAK, //68
+ "X509_V_ERR_CA_MD_TOO_WEAK"
+ },
+#endif
+#if defined(X509_V_ERR_INVALID_CALL)
+ {
+ X509_V_ERR_INVALID_CALL, //69
+ "X509_V_ERR_INVALID_CALL"
+ },
+#endif
+#if defined(X509_V_ERR_STORE_LOOKUP)
+ {
+ X509_V_ERR_STORE_LOOKUP, //70
+ "X509_V_ERR_STORE_LOOKUP"
+ },
+#endif
+#if defined(X509_V_ERR_NO_VALID_SCTS)
+ {
+ X509_V_ERR_NO_VALID_SCTS, //71
+ "X509_V_ERR_NO_VALID_SCTS"
+ },
+#endif
+#if defined(X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION)
+ {
+ X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION, //72
+ "X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION"
+ },
+#endif
+#if defined(X509_V_ERR_OCSP_VERIFY_NEEDED)
+ {
+ X509_V_ERR_OCSP_VERIFY_NEEDED, //73
+ "X509_V_ERR_OCSP_VERIFY_NEEDED"
+ },
+#endif
+#if defined(X509_V_ERR_OCSP_VERIFY_FAILED)
+ {
+ X509_V_ERR_OCSP_VERIFY_FAILED, //74
+ "X509_V_ERR_OCSP_VERIFY_FAILED"
+ },
+#endif
+#if defined(X509_V_ERR_OCSP_CERT_UNKNOWN)
+ {
+ X509_V_ERR_OCSP_CERT_UNKNOWN, //75
+ "X509_V_ERR_OCSP_CERT_UNKNOWN"
+ },
+#endif
{ SSL_ERROR_NONE, "SSL_ERROR_NONE"},
{SSL_ERROR_NONE, NULL}
};
@@ -286,6 +412,27 @@
"X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX",
"X509_V_ERR_UNSUPPORTED_NAME_SYNTAX",
"X509_V_ERR_CRL_PATH_VALIDATION_ERROR",
+ "X509_V_ERR_PATH_LOOP",
+ "X509_V_ERR_SUITE_B_INVALID_VERSION",
+ "X509_V_ERR_SUITE_B_INVALID_ALGORITHM",
+ "X509_V_ERR_SUITE_B_INVALID_CURVE",
+ "X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM",
+ "X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED",
+ "X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256",
+ "X509_V_ERR_HOSTNAME_MISMATCH",
+ "X509_V_ERR_EMAIL_MISMATCH",
+ "X509_V_ERR_IP_ADDRESS_MISMATCH",
+ "X509_V_ERR_DANE_NO_MATCH",
+ "X509_V_ERR_EE_KEY_TOO_SMALL",
+ "X509_V_ERR_CA_KEY_TOO_SMALL",
+ "X509_V_ERR_CA_MD_TOO_WEAK",
+ "X509_V_ERR_INVALID_CALL",
+ "X509_V_ERR_STORE_LOOKUP",
+ "X509_V_ERR_NO_VALID_SCTS",
+ "X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION",
+ "X509_V_ERR_OCSP_VERIFY_NEEDED",
+ "X509_V_ERR_OCSP_VERIFY_FAILED",
+ "X509_V_ERR_OCSP_CERT_UNKNOWN",
NULL
};
@@ -390,7 +537,7 @@
return false; // not reached
}
-const char *Ssl::GetErrorName(Security::ErrorCode value)
+const char *Ssl::GetErrorName(Security::ErrorCode value, const bool prefixRawCode)
{
if (TheSslErrors.empty())
loadSslErrorMap();
@@ -399,7 +546,9 @@
if (it != TheSslErrors.end())
return it->second->name;
- return NULL;
+ static char tmpBuffer[128];
+ snprintf(tmpBuffer, sizeof(tmpBuffer), "%s%d", prefixRawCode ? "SSL_ERR=" : "", (int)value);
+ return tmpBuffer;
}
bool
@@ -529,21 +678,14 @@
*/
const char *Ssl::ErrorDetail::err_code() const
{
- static char tmpBuffer[64];
// We can use the GetErrorName but using the detailEntry is faster,
// so try it first.
- const char *err = detailEntry.name.termedBuf();
+ if (const char *err = detailEntry.name.termedBuf())
+ return err;
// error details not loaded yet or not defined in error_details.txt,
// try the GetErrorName...
- if (!err)
- err = GetErrorName(error_no);
-
- if (!err) {
- snprintf(tmpBuffer, 64, "%d", (int)error_no);
- err = tmpBuffer;
- }
- return err;
+ return GetErrorName(error_no);
}
/**
diff -u -r -N squid-5.0.2/src/ssl/ErrorDetail.h squid-5.0.3/src/ssl/ErrorDetail.h
--- squid-5.0.2/src/ssl/ErrorDetail.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ssl/ErrorDetail.h 2020-06-08 03:33:42.000000000 +1200
@@ -26,8 +26,9 @@
/// The Security::ErrorCode code of the error described by "name".
Security::ErrorCode GetErrorCode(const char *name);
-/// The string representation of the TLS error "value"
-const char *GetErrorName(Security::ErrorCode value);
+/// \return string representation of a known TLS error (or a raw error code)
+/// \param prefixRawCode whether to prefix raw codes with "SSL_ERR="
+const char *GetErrorName(Security::ErrorCode value, const bool prefixRawCode = false);
/// A short description of the TLS error "value"
const char *GetErrorDescr(Security::ErrorCode value);
diff -u -r -N squid-5.0.2/src/ssl/PeekingPeerConnector.cc squid-5.0.3/src/ssl/PeekingPeerConnector.cc
--- squid-5.0.2/src/ssl/PeekingPeerConnector.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ssl/PeekingPeerConnector.cc 2020-06-08 03:33:42.000000000 +1200
@@ -92,8 +92,9 @@
al->ssl.bumpMode = finalAction;
if (finalAction == Ssl::bumpTerminate) {
- serverConn->close();
+ bail(new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scForbidden, request.getRaw(), al));
clientConn->close();
+ clientConn = nullptr;
} else if (finalAction != Ssl::bumpSplice) {
//Allow write, proceed with the connection
srvBio->holdWrite(false);
@@ -140,11 +141,13 @@
if (!Security::PeerConnector::initialize(serverSession))
return false;
+ // client connection supplies TLS client details and is also used if we
+ // need to splice or terminate the client and server connections
+ if (!Comm::IsConnOpen(clientConn))
+ return false;
+
if (ConnStateData *csd = request->clientConnectionManager.valid()) {
- // client connection is required in the case we need to splice
- // or terminate client and server connections
- assert(clientConn != NULL);
SBuf *hostName = NULL;
//Enable Status_request TLS extension, required to bump some clients
@@ -245,6 +248,10 @@
if (!error) {
serverCertificateVerified();
if (splice) {
+ if (!Comm::IsConnOpen(clientConn)) {
+ bail(new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw(), al));
+ throw TextException("from-client connection gone", Here());
+ }
switchToTunnel(request.getRaw(), clientConn, serverConn);
tunnelInsteadOfNegotiating();
}
@@ -276,18 +283,32 @@
BIO *b = SSL_get_rbio(session.get());
Ssl::ServerBio *srvBio = static_cast(BIO_get_data(b));
- // In Peek mode, the ClientHello message sent to the server. If the
- // server resuming a previous (spliced) SSL session with the client,
- // then probably we are here because local SSL object does not know
- // anything about the session being resumed.
- //
- if (srvBio->bumpMode() == Ssl::bumpPeek && (resumingSession = srvBio->resumingSession())) {
- // we currently splice all resumed sessions unconditionally
- // if (const bool spliceResumed = true) {
- bypassCertValidator();
- checkForPeekAndSpliceMatched(Ssl::bumpSplice);
- return;
- // } // else fall through to find a matching ssl_bump action (with limited info)
+ if (srvBio->bumpMode() == Ssl::bumpPeek) {
+ auto bypassValidator = false;
+ if (srvBio->encryptedCertificates()) {
+ // it is pointless to peek at encrypted certificates
+ //
+ // we currently splice all sessions with encrypted certificates
+ // if (const auto spliceEncryptedCertificates = true) {
+ bypassValidator = true;
+ // } // else fall through to find a matching ssl_bump action (with limited info)
+ } else if (srvBio->resumingSession()) {
+ // In peek mode, the ClientHello message is forwarded to the server.
+ // If the server is resuming a previous (spliced) SSL session with
+ // the client, then probably we are here because our local SSL
+ // object does not know anything about the session being resumed.
+ //
+ // we currently splice all resumed sessions
+ // if (const auto spliceResumed = true) {
+ bypassValidator = true;
+ // } // else fall through to find a matching ssl_bump action (with limited info)
+ }
+
+ if (bypassValidator) {
+ bypassCertValidator();
+ checkForPeekAndSpliceMatched(Ssl::bumpSplice);
+ return;
+ }
}
// If we are in peek-and-splice mode and still we did not write to
diff -u -r -N squid-5.0.2/src/ssl/PeekingPeerConnector.h squid-5.0.3/src/ssl/PeekingPeerConnector.h
--- squid-5.0.2/src/ssl/PeekingPeerConnector.h 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/ssl/PeekingPeerConnector.h 2020-06-08 03:33:42.000000000 +1200
@@ -30,7 +30,6 @@
Security::PeerConnector(aServerConn, aCallback, alp, timeout),
clientConn(aClientConn),
splice(false),
- resumingSession(false),
serverCertificateHandled(false)
{
request = aRequest;
@@ -75,7 +74,6 @@
Comm::ConnectionPointer clientConn; ///< TCP connection to the client
AsyncCall::Pointer closeHandler; ///< we call this when the connection closed
bool splice; ///< whether we are going to splice or not
- bool resumingSession; ///< whether it is an SSL resuming session connection
bool serverCertificateHandled; ///< whether handleServerCertificate() succeeded
};
diff -u -r -N squid-5.0.2/src/store/id_rewriters/file/storeid_file_rewrite.8 squid-5.0.3/src/store/id_rewriters/file/storeid_file_rewrite.8
--- squid-5.0.2/src/store/id_rewriters/file/storeid_file_rewrite.8 2020-04-20 00:20:16.000000000 +1200
+++ squid-5.0.3/src/store/id_rewriters/file/storeid_file_rewrite.8 2020-06-09 19:09:21.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "STOREID_FILE_REWRITE 8"
-.TH STOREID_FILE_REWRITE 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH STOREID_FILE_REWRITE 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
diff -u -r -N squid-5.0.2/src/tests/stub_libanyp.cc squid-5.0.3/src/tests/stub_libanyp.cc
--- squid-5.0.2/src/tests/stub_libanyp.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/tests/stub_libanyp.cc 2020-06-08 03:33:42.000000000 +1200
@@ -18,6 +18,7 @@
void AnyP::Uri::host(const char *) STUB
static SBuf nil;
const SBuf &AnyP::Uri::path() const STUB_RETVAL(nil)
+void AnyP::Uri::addRelativePath(const char *) STUB
const SBuf &AnyP::Uri::SlashPath()
{
static SBuf slash("/");
@@ -33,11 +34,9 @@
void urlInitialize() STUB
const char *urlCanonicalFakeHttps(const HttpRequest *) STUB_RETVAL(nullptr)
bool urlIsRelative(const char *) STUB_RETVAL(false)
-char *urlMakeAbsolute(const HttpRequest *, const char *)STUB_RETVAL(nullptr)
char *urlRInternal(const char *, unsigned short, const char *, const char *) STUB_RETVAL(nullptr)
char *urlInternal(const char *, const char *) STUB_RETVAL(nullptr)
int matchDomainName(const char *, const char *, enum MatchDomainNameFlags) STUB_RETVAL(0)
int urlCheckRequest(const HttpRequest *) STUB_RETVAL(0)
-char *urlHostname(const char *) STUB_RETVAL(nullptr)
void urlExtMethodConfigure() STUB
diff -u -r -N squid-5.0.2/src/tests/stub_libsecurity.cc squid-5.0.3/src/tests/stub_libsecurity.cc
--- squid-5.0.2/src/tests/stub_libsecurity.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/tests/stub_libsecurity.cc 2020-06-08 03:33:42.000000000 +1200
@@ -28,7 +28,7 @@
std::ostream &Security::operator <<(std::ostream &os, const Security::EncryptorAnswer &) STUB_RETVAL(os)
#include "security/Handshake.h"
-Security::HandshakeParser::HandshakeParser() STUB
+Security::HandshakeParser::HandshakeParser(MessageSource) STUB
bool Security::HandshakeParser::parseHello(const SBuf &) STUB_RETVAL(false)
#include "security/KeyData.h"
@@ -50,14 +50,12 @@
{
PeerConnector::PeerConnector(const Comm::ConnectionPointer &, AsyncCall::Pointer &, const AccessLogEntryPointer &, const time_t) :
AsyncJob("Security::PeerConnector") {STUB}
-PeerConnector::~PeerConnector() {STUB}
void PeerConnector::start() STUB
bool PeerConnector::doneAll() const STUB_RETVAL(true)
void PeerConnector::swanSong() STUB
const char *PeerConnector::status() const STUB_RETVAL("")
void PeerConnector::commCloseHandler(const CommCloseCbParams &) STUB
-void PeerConnector::connectionClosed(const char *) STUB
-bool PeerConnector::prepareSocket() STUB_RETVAL(false)
+void PeerConnector::commTimeoutHandler(const CommTimeoutCbParams &) STUB
bool PeerConnector::initialize(Security::SessionPointer &) STUB_RETVAL(false)
void PeerConnector::negotiate() STUB
bool PeerConnector::sslFinalized() STUB_RETVAL(false)
@@ -67,7 +65,9 @@
void PeerConnector::noteNegotiationError(const int, const int, const int) STUB
// virtual Security::ContextPointer getTlsContext() = 0;
void PeerConnector::bail(ErrorState *) STUB
+void PeerConnector::sendSuccess() STUB
void PeerConnector::callBack() STUB
+void PeerConnector::disconnect() STUB
void PeerConnector::recordNegotiationDetails() STUB
}
diff -u -r -N squid-5.0.2/src/tests/stub_StatHist.cc squid-5.0.3/src/tests/stub_StatHist.cc
--- squid-5.0.2/src/tests/stub_StatHist.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/tests/stub_StatHist.cc 2020-06-08 03:33:42.000000000 +1200
@@ -16,7 +16,7 @@
void StatHist::dump(StoreEntry * sentry, StatHistBinDumper * bd) const STUB
void StatHist::enumInit(unsigned int i) STUB_NOP
-void StatHist::count(double d) STUB_NOP
+void StatHist::count(double) {/* STUB_NOP */}
double statHistDeltaMedian(const StatHist & A, const StatHist & B) STUB_RETVAL(0.0)
double statHistDeltaPctile(const StatHist & A, const StatHist & B, double pctile) STUB_RETVAL(0.0)
void StatHist::logInit(unsigned int i, double d1, double d2) STUB_NOP
diff -u -r -N squid-5.0.2/src/tunnel.cc squid-5.0.3/src/tunnel.cc
--- squid-5.0.2/src/tunnel.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/tunnel.cc 2020-06-08 03:33:42.000000000 +1200
@@ -111,9 +111,12 @@
/// starts connecting to the next hop, either for the first time or while
/// recovering from the previous connect failure
void startConnecting();
+ void closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason);
+
+ void retryOrBail();
/// called when negotiations with the peer have been successfully completed
- void notePeerReadyToShovel();
+ void notePeerReadyToShovel(const Comm::ConnectionPointer &);
class Connection
{
@@ -179,7 +182,8 @@
void copyRead(Connection &from, IOCB *completion);
/// continue to set up connection to a peer, going async for SSL peers
- void connectToPeer();
+ void connectToPeer(const Comm::ConnectionPointer &);
+ void secureConnectionToPeer(const Comm::ConnectionPointer &);
/* PeerSelectionInitiator API */
virtual void noteDestination(Comm::ConnectionPointer conn) override;
@@ -233,8 +237,15 @@
void usePinned();
- /// callback handler after connection setup (including any encryption)
- void connectedToPeer(Security::EncryptorAnswer &answer);
+ /// callback handler for the Security::PeerConnector encryptor
+ void noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &);
+
+ /// called after connection setup (including any encryption)
+ void connectedToPeer(const Comm::ConnectionPointer &);
+ void establishTunnelThruProxy(const Comm::ConnectionPointer &);
+
+ template
+ void advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep);
/// details of the "last tunneling attempt" failure (if it failed)
ErrorState *savedError = nullptr;
@@ -680,6 +691,15 @@
}
void
+TunnelStateData::closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason)
+{
+ debugs(26, 3, "because " << reason << "; " << conn);
+ assert(!server.conn);
+ if (IsConnOpen(conn))
+ conn->close();
+}
+
+void
TunnelStateData::Connection::closeIfOpen()
{
if (Comm::IsConnOpen(conn))
@@ -776,6 +796,11 @@
tunnelStartShoveling(TunnelStateData *tunnelState)
{
assert(!tunnelState->waitingForConnectExchange);
+ assert(tunnelState->server.conn);
+ AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
+ CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
+ commSetConnTimeout(tunnelState->server.conn, Config.Timeout.read, timeoutCall);
+
*tunnelState->status_ptr = Http::scOkay;
if (tunnelState->logTag_ptr)
tunnelState->logTag_ptr->update(LOG_TCP_TUNNEL);
@@ -838,35 +863,51 @@
waitingForConnectExchange = false;
- if (answer.positive()) {
+ auto sawProblem = false;
+
+ if (!answer.positive()) {
+ sawProblem = true;
+ Must(!Comm::IsConnOpen(answer.conn));
+ } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
+ sawProblem = true;
+ closePendingConnection(answer.conn, "conn was closed while waiting for tunnelEstablishmentDone");
+ }
+
+ if (!sawProblem) {
+ assert(answer.positive()); // paranoid
// copy any post-200 OK bytes to our buffer
preReadServerData = answer.leftovers;
- notePeerReadyToShovel();
+ notePeerReadyToShovel(answer.conn);
return;
}
- // TODO: Reuse to-peer connections after a CONNECT error response.
-
- // TODO: We can and, hence, should close now, but tunnelServerClosed()
- // cannot yet tell whether ErrorState is still writing an error response.
- // server.closeIfOpen();
-
if (!clientExpectsConnectResponse()) {
// closing the non-HTTP client connection is the best we can do
- debugs(50, 3, server.conn << " closing on CONNECT-to-peer error");
- server.closeIfOpen();
+ debugs(50, 3, client.conn << " closing on CONNECT-to-peer error");
+ client.closeIfOpen();
return;
}
- ErrorState *error = answer.squidError.get();
- Must(error);
- answer.squidError.clear(); // preserve error for errorSendComplete()
- sendError(error, "tunneler returns error");
+ ErrorState *error = nullptr;
+ if (answer.positive()) {
+ error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
+ } else {
+ error = answer.squidError.get();
+ Must(error);
+ answer.squidError.clear(); // preserve error for errorSendComplete()
+ }
+ assert(error);
+ saveError(error);
+ retryOrBail();
}
void
-TunnelStateData::notePeerReadyToShovel()
+TunnelStateData::notePeerReadyToShovel(const Comm::ConnectionPointer &conn)
{
+ server.conn = conn;
+ // Start monitoring for server-side connection problems
+ comm_add_close_handler(server.conn->fd, tunnelServerClosed, this);
+
if (!clientExpectsConnectResponse())
tunnelStartShoveling(this); // ssl-bumped connection, be quiet
else {
@@ -896,17 +937,42 @@
tunnelState->server.conn->close();
}
+/// reacts to the current destination failure
+void
+TunnelStateData::retryOrBail()
+{
+ // Since no TCP payload has been passed to client or server, we may
+ // TCP-connect to other destinations (including alternate IPs).
+
+ if (!FwdState::EnoughTimeToReForward(startTime))
+ return sendError(savedError, "forwarding timeout");
+
+ if (!destinations->empty())
+ return startConnecting();
+
+ if (!PeerSelectionInitiator::subscribed)
+ return sendError(savedError, "tried all destinations");
+}
+
void
TunnelStateData::noteConnection(HappyConnOpener::Answer &answer)
{
calls.connector = nullptr;
connOpener.clear();
- if (const auto error = answer.error.get()) {
+ ErrorState *error = nullptr;
+ if ((error = answer.error.get())) {
+ Must(!Comm::IsConnOpen(answer.conn));
syncHierNote(answer.conn, request->url.host());
+ answer.error.clear();
+ } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
+ error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
+ closePendingConnection(answer.conn, "conn was closed while waiting for noteConnection");
+ }
+
+ if (error) {
saveError(error);
- answer.error.clear(); // savedError has it now
- sendError(savedError, "tried all destinations");
+ retryOrBail();
return;
}
@@ -917,17 +983,12 @@
TunnelStateData::connectDone(const Comm::ConnectionPointer &conn, const char *origin, const bool reused)
{
Must(Comm::IsConnOpen(conn));
- server.conn = conn;
if (reused)
ResetMarkingsToServer(request.getRaw(), *conn);
// else Comm::ConnOpener already applied proper/current markings
- syncHierNote(server.conn, request->url.host());
-
- request->hier.resetPeerNotes(conn, origin);
- if (al)
- al->hier.resetPeerNotes(conn, origin);
+ syncHierNote(conn, origin);
#if USE_DELAY_POOLS
/* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
@@ -938,7 +999,6 @@
netdbPingSite(request->url.host());
request->peer_host = conn->getPeer() ? conn->getPeer()->host : nullptr;
- comm_add_close_handler(conn->fd, tunnelServerClosed, this);
bool toOrigin = false; // same semantics as StateFlags::toOrigin
if (const auto * const peer = conn->getPeer()) {
@@ -950,14 +1010,10 @@
}
if (!toOrigin)
- connectToPeer();
+ connectToPeer(conn);
else {
- notePeerReadyToShovel();
+ notePeerReadyToShovel(conn);
}
-
- AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
- CommTimeoutCbPtrFun(tunnelTimeout, this));
- commSetConnTimeout(conn, Config.Timeout.read, timeoutCall);
}
void
@@ -1007,38 +1063,85 @@
}
void
-TunnelStateData::connectToPeer()
+TunnelStateData::connectToPeer(const Comm::ConnectionPointer &conn)
{
- if (CachePeer *p = server.conn->getPeer()) {
- if (p->secure.encryptTransport) {
- AsyncCall::Pointer callback = asyncCall(5,4,
- "TunnelStateData::ConnectedToPeer",
- MyAnswerDialer(&TunnelStateData::connectedToPeer, this));
- auto *connector = new Security::BlindPeerConnector(request, server.conn, callback, al);
- AsyncJob::Start(connector); // will call our callback
- return;
- }
+ if (const auto p = conn->getPeer()) {
+ if (p->secure.encryptTransport)
+ return advanceDestination("secure connection to peer", conn, [this,&conn] {
+ secureConnectionToPeer(conn);
+ });
}
- Security::EncryptorAnswer nil;
- connectedToPeer(nil);
+ connectedToPeer(conn);
}
+/// encrypts an established TCP connection to peer
void
-TunnelStateData::connectedToPeer(Security::EncryptorAnswer &answer)
+TunnelStateData::secureConnectionToPeer(const Comm::ConnectionPointer &conn)
+{
+ AsyncCall::Pointer callback = asyncCall(5,4, "TunnelStateData::noteSecurityPeerConnectorAnswer",
+ MyAnswerDialer(&TunnelStateData::noteSecurityPeerConnectorAnswer, this));
+ const auto connector = new Security::BlindPeerConnector(request, conn, callback, al);
+ AsyncJob::Start(connector); // will call our callback
+}
+
+/// starts a preparation step for an established connection; retries on failures
+template
+void
+TunnelStateData::advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep)
+{
+ // TODO: Extract destination-specific handling from TunnelStateData so that
+ // all the awkward, limited-scope advanceDestination() calls can be replaced
+ // with a single simple try/catch,retry block.
+ try {
+ startStep();
+ // now wait for the step callback
+ } catch (...) {
+ debugs (26, 2, "exception while trying to " << stepDescription << ": " << CurrentException);
+ closePendingConnection(conn, "connection preparation exception");
+ if (!savedError)
+ saveError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al));
+ retryOrBail();
+ }
+}
+
+/// callback handler for the connection encryptor
+void
+TunnelStateData::noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &answer)
{
if (ErrorState *error = answer.error.get()) {
+ Must(!Comm::IsConnOpen(answer.conn));
answer.error.clear(); // sendError() will own the error
sendError(error, "TLS peer connection error");
return;
}
+ if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
+ sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al), "connecion gone");
+ closePendingConnection(answer.conn, "conn was closed while waiting for noteSecurityPeerConnectorAnswer");
+ return;
+ }
+
+ connectedToPeer(answer.conn);
+}
+
+void
+TunnelStateData::connectedToPeer(const Comm::ConnectionPointer &conn)
+{
+ advanceDestination("establish tunnel through proxy", conn, [this,&conn] {
+ establishTunnelThruProxy(conn);
+ });
+}
+
+void
+TunnelStateData::establishTunnelThruProxy(const Comm::ConnectionPointer &conn)
+{
assert(!waitingForConnectExchange);
AsyncCall::Pointer callback = asyncCall(5,4,
"TunnelStateData::tunnelEstablishmentDone",
Http::Tunneler::CbDialer(&TunnelStateData::tunnelEstablishmentDone, this));
- const auto tunneler = new Http::Tunneler(server.conn, request, callback, Config.Timeout.lifetime, al);
+ const auto tunneler = new Http::Tunneler(conn, request, callback, Config.Timeout.lifetime, al);
#if USE_DELAY_POOLS
tunneler->setDelayId(server.delayId);
#endif
@@ -1240,6 +1343,9 @@
void
switchToTunnel(HttpRequest *request, Comm::ConnectionPointer &clientConn, Comm::ConnectionPointer &srvConn)
{
+ Must(Comm::IsConnOpen(clientConn));
+ Must(Comm::IsConnOpen(srvConn));
+
debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd);
/* Create state structure. */
@@ -1277,10 +1383,6 @@
else
request->prepForDirect();
- AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
- CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
- commSetConnTimeout(srvConn, Config.Timeout.read, timeoutCall);
-
// we drain any already buffered from-server data below (rBufData)
fd_table[srvConn->fd].useDefaultIo();
diff -u -r -N squid-5.0.2/src/urn.cc squid-5.0.3/src/urn.cc
--- squid-5.0.2/src/urn.cc 2020-04-19 22:50:48.000000000 +1200
+++ squid-5.0.3/src/urn.cc 2020-06-08 03:33:42.000000000 +1200
@@ -387,7 +387,6 @@
{
char *buf = xstrdup(inbuf);
char *token;
- char *host;
url_entry *list;
url_entry *old;
int n = 32;
@@ -406,24 +405,23 @@
safe_free(old);
}
- host = urlHostname(token);
-
- if (NULL == host)
+ AnyP::Uri uri;
+ if (!uri.parse(m, SBuf(token)) || !*uri.host())
continue;
#if USE_ICMP
- list[i].rtt = netdbHostRtt(host);
+ list[i].rtt = netdbHostRtt(uri.host());
if (0 == list[i].rtt) {
- debugs(52, 3, "urnParseReply: Pinging " << host);
- netdbPingSite(host);
+ debugs(52, 3, "Pinging " << uri.host());
+ netdbPingSite(uri.host());
}
#else
list[i].rtt = 0;
#endif
- list[i].url = xstrdup(token);
- list[i].host = xstrdup(host);
+ list[i].url = xstrdup(uri.absolute().c_str());
+ list[i].host = xstrdup(uri.host());
// TODO: Use storeHas() or lock/unlock entry to avoid creating unlocked
// ones.
list[i].flags.cached = storeGetPublic(list[i].url, m) ? 1 : 0;
diff -u -r -N squid-5.0.2/tools/CharacterSet.cc squid-5.0.3/tools/CharacterSet.cc
--- squid-5.0.2/tools/CharacterSet.cc 2020-04-20 00:20:18.000000000 +1200
+++ squid-5.0.3/tools/CharacterSet.cc 2020-06-09 19:09:23.000000000 +1200
@@ -51,6 +51,13 @@
}
CharacterSet &
+CharacterSet::remove(const unsigned char c)
+{
+ chars_[static_cast(c)] = 0;
+ return *this;
+}
+
+CharacterSet &
CharacterSet::addRange(unsigned char low, unsigned char high)
{
//manual loop splitting is needed to cover case where high is 255
diff -u -r -N squid-5.0.2/tools/helper-mux/helper-mux.8 squid-5.0.3/tools/helper-mux/helper-mux.8
--- squid-5.0.2/tools/helper-mux/helper-mux.8 2020-04-20 00:20:19.000000000 +1200
+++ squid-5.0.3/tools/helper-mux/helper-mux.8 2020-06-09 19:09:23.000000000 +1200
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pod::Man 4.10 (Pod::Simple 3.35)
+.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
@@ -133,7 +133,7 @@
.\" ========================================================================
.\"
.IX Title "HELPER-MUX 8"
-.TH HELPER-MUX 8 "2020-04-19" "perl v5.28.1" "User Contributed Perl Documentation"
+.TH HELPER-MUX 8 "2020-06-09" "perl v5.30.3" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l