diff -u -r -N squid-4.0.12/aclocal.m4 squid-4.0.13/aclocal.m4 --- squid-4.0.12/aclocal.m4 2016-07-02 01:27:31.000000000 +1200 +++ squid-4.0.13/aclocal.m4 2016-08-06 00:53:48.000000000 +1200 @@ -744,7 +744,6 @@ cat <<_LT_EOF >> "$cfgfile" #! $SHELL # Generated automatically by $as_me ($PACKAGE) $VERSION -# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # Provide generalized library-building support services. diff -u -r -N squid-4.0.12/cfgaux/ltmain.sh squid-4.0.13/cfgaux/ltmain.sh --- squid-4.0.12/cfgaux/ltmain.sh 2016-07-02 01:27:42.000000000 +1200 +++ squid-4.0.13/cfgaux/ltmain.sh 2016-08-06 00:53:58.000000000 +1200 @@ -31,7 +31,7 @@ PROGRAM=libtool PACKAGE=libtool -VERSION="2.4.6 Debian-2.4.6-0.1" +VERSION="2.4.6 Debian-2.4.6-1" package_revision=2.4.6 @@ -1977,7 +1977,7 @@ # End: # Set a version string. -scriptversion='(GNU libtool) 2.4.6' +scriptversion='(GNU libtool) 2.4.6 Debian-2.4.6-1' # func_echo ARG... @@ -2068,7 +2068,7 @@ compiler: $LTCC compiler flags: $LTCFLAGS linker: $LD (gnu? $with_gnu_ld) - version: $progname (GNU libtool) 2.4.6 + version: $progname $scriptversion automake: `($AUTOMAKE --version) 2>/dev/null |$SED 1q` autoconf: `($AUTOCONF --version) 2>/dev/null |$SED 1q` diff -u -r -N squid-4.0.12/ChangeLog squid-4.0.13/ChangeLog --- squid-4.0.12/ChangeLog 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/ChangeLog 2016-08-06 00:52:55.000000000 +1200 @@ -1,3 +1,21 @@ +Changes to squid-4.0.13 (05 Aug 2016): + + - Regression Bug 4540: revert r14720 buffer update + - Bug 4555: Minor improvements to error pages CSS + - Bug 4551: fix exceptions in new chunked decoder + - Bug 4311: support collapse for internal revalidation requests (SMP-unaware caches) + - Fix Certificate Validator buffer-overflow crashes Squid + - Fix some failed transactions not being logged + - Fix segfault via Ftp::Client::readControlReply(). + - basic_db_auth: add support for unsalted SHA1 passwords + - kerberos_ldap_group: add support for SSL/TLS connection to an LDAP server + - TLS: Add missing 'tls' option for cache_peer + - TLS: Do not hang when 'connector' fails + - TLS: Add support for fetching missing certificates + - Remove XSTD_USE_LIBLTDL, which has not been needed in a long while + - ... and many code polishing updates + - ... and some documentation updates + Changes to squid-4.0.12 (01 Jul 2016): - Regression Fix: shell issues with require_smblib definition diff -u -r -N squid-4.0.12/configure squid-4.0.13/configure --- squid-4.0.12/configure 2016-07-02 01:29:18.000000000 +1200 +++ squid-4.0.13/configure 2016-08-06 00:55:28.000000000 +1200 @@ -1,7 +1,7 @@ #! /bin/sh # From configure.ac Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Squid Web Proxy 4.0.12. +# Generated by GNU Autoconf 2.69 for Squid Web Proxy 4.0.13. # # Report bugs to . # @@ -595,8 +595,8 @@ # Identity of this package. PACKAGE_NAME='Squid Web Proxy' PACKAGE_TARNAME='squid' -PACKAGE_VERSION='4.0.12' -PACKAGE_STRING='Squid Web Proxy 4.0.12' +PACKAGE_VERSION='4.0.13' +PACKAGE_STRING='Squid Web Proxy 4.0.13' PACKAGE_BUGREPORT='http://bugs.squid-cache.org/' PACKAGE_URL='' @@ -1648,7 +1648,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Squid Web Proxy 4.0.12 to adapt to many kinds of systems. +\`configure' configures Squid Web Proxy 4.0.13 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1719,7 +1719,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Squid Web Proxy 4.0.12:";; + short | recursive ) echo "Configuration of Squid Web Proxy 4.0.13:";; esac cat <<\_ACEOF @@ -2148,7 +2148,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Squid Web Proxy configure 4.0.12 +Squid Web Proxy configure 4.0.13 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -3252,7 +3252,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Squid Web Proxy $as_me 4.0.12, which was +It was created by Squid Web Proxy $as_me 4.0.13, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4119,7 +4119,7 @@ # Define the identity of the package. PACKAGE='squid' - VERSION='4.0.12' + VERSION='4.0.13' cat >>confdefs.h <<_ACEOF @@ -42599,7 +42599,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Squid Web Proxy $as_me 4.0.12, which was +This file was extended by Squid Web Proxy $as_me 4.0.13, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -42665,7 +42665,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Squid Web Proxy config.status 4.0.12 +Squid Web Proxy config.status 4.0.13 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -43996,7 +43996,6 @@ cat <<_LT_EOF >> "$cfgfile" #! $SHELL # Generated automatically by $as_me ($PACKAGE) $VERSION -# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # Provide generalized library-building support services. diff -u -r -N squid-4.0.12/configure.ac squid-4.0.13/configure.ac --- squid-4.0.12/configure.ac 2016-07-02 01:29:17.000000000 +1200 +++ squid-4.0.13/configure.ac 2016-08-06 00:55:28.000000000 +1200 @@ -5,7 +5,7 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## -AC_INIT([Squid Web Proxy],[4.0.12],[http://bugs.squid-cache.org/],[squid]) +AC_INIT([Squid Web Proxy],[4.0.13],[http://bugs.squid-cache.org/],[squid]) AC_PREREQ(2.61) AC_CONFIG_HEADERS([include/autoconf.h]) AC_CONFIG_AUX_DIR(cfgaux) diff -u -r -N squid-4.0.12/doc/release-notes/release-4.html squid-4.0.13/doc/release-notes/release-4.html --- squid-4.0.12/doc/release-notes/release-4.html 2016-07-02 02:22:44.000000000 +1200 +++ squid-4.0.13/doc/release-notes/release-4.html 2016-08-06 02:24:35.000000000 +1200 @@ -2,10 +2,10 @@ - Squid 4.0.12 release notes + Squid 4.0.13 release notes -

Squid 4.0.12 release notes

+

Squid 4.0.13 release notes

Squid Developers

@@ -61,7 +61,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-4.0.12 for testing.

+

The Squid Team are pleased to announce the release of Squid-4.0.13 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v4/ or the mirrors.

@@ -305,6 +305,12 @@

+
access_log
+

TCP accept(2) errors logged with URI error:accept-client-connection.

+

Unused connections received in http_port or https_port +or transactions terminated before reading[parsing] request headers +logged with URI error:transaction-end-before-headers.

+
acl

New -m flag for note ACL to match substrings.

@@ -315,6 +321,8 @@
cache_peer

New option auth-no-keytab to let GSSAPI implementation determine which Kerberos credentials to use, instead of specifying a keytab.

+

Replaced option ssl with tls. Use of any +tls- prefixed options implies tls is enabled.

New option tls-min-version=1.N to set minimum TLS version allowed.

New option tls-default-ca replaces sslflags=NO_DEFAULT_CA

New option tls-no-npn to disable sending TLS NPN extension.

@@ -322,7 +330,8 @@ have been removed.

Removed sslversion= option. Use tls-options= instead.

Manual squid.conf update may be required on upgrade.

-

Replaced sslcafile= with tls-cafile= which takes multiple entries.

+

Replaced option sslcafile= with tls-cafile= +which takes multiple entries.

external_acl_type

New parameter queue-size= to set the maximum number @@ -534,6 +543,8 @@

Replaced by --with-cppunit=PATH. Please prefer the default auto-detection though.

+
XSTD_USE_LIBLTDL
+

Removed. Use --with-included-ltdl instead.

diff -u -r -N squid-4.0.12/errors/errorpage.css squid-4.0.13/errors/errorpage.css --- squid-4.0.12/errors/errorpage.css 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/errors/errorpage.css 2016-08-06 00:52:55.000000000 +1200 @@ -71,12 +71,11 @@ } pre { - font-family:sans-serif; } /* special event: FTP / Gopher directory listing */ #dirmsg { - font-family: courier; + font-family: courier, monospace; color: black; font-size: 10pt; } diff -u -r -N squid-4.0.12/include/version.h squid-4.0.13/include/version.h --- squid-4.0.12/include/version.h 2016-07-02 01:29:18.000000000 +1200 +++ squid-4.0.13/include/version.h 2016-08-06 00:55:28.000000000 +1200 @@ -7,7 +7,7 @@ */ #ifndef SQUID_RELEASE_TIME -#define SQUID_RELEASE_TIME 1467379594 +#define SQUID_RELEASE_TIME 1470401568 #endif /* diff -u -r -N squid-4.0.12/libltdl/configure squid-4.0.13/libltdl/configure --- squid-4.0.12/libltdl/configure 2016-07-02 01:40:20.000000000 +1200 +++ squid-4.0.13/libltdl/configure 2016-08-06 01:07:30.000000000 +1200 @@ -14975,7 +14975,6 @@ cat <<_LT_EOF >> "$cfgfile" #! $SHELL # Generated automatically by $as_me ($PACKAGE) $VERSION -# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # Provide generalized library-building support services. diff -u -r -N squid-4.0.12/libltdl/m4/libtool.m4 squid-4.0.13/libltdl/m4/libtool.m4 --- squid-4.0.12/libltdl/m4/libtool.m4 2016-07-02 01:27:43.000000000 +1200 +++ squid-4.0.13/libltdl/m4/libtool.m4 2016-08-06 00:53:59.000000000 +1200 @@ -728,7 +728,6 @@ cat <<_LT_EOF >> "$cfgfile" #! $SHELL # Generated automatically by $as_me ($PACKAGE) $VERSION -# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # Provide generalized library-building support services. diff -u -r -N squid-4.0.12/RELEASENOTES.html squid-4.0.13/RELEASENOTES.html --- squid-4.0.12/RELEASENOTES.html 2016-07-02 02:22:44.000000000 +1200 +++ squid-4.0.13/RELEASENOTES.html 2016-08-06 02:24:35.000000000 +1200 @@ -2,10 +2,10 @@ - Squid 4.0.12 release notes + Squid 4.0.13 release notes -

Squid 4.0.12 release notes

+

Squid 4.0.13 release notes

Squid Developers

@@ -61,7 +61,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-4.0.12 for testing.

+

The Squid Team are pleased to announce the release of Squid-4.0.13 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v4/ or the mirrors.

@@ -305,6 +305,12 @@

+
access_log
+

TCP accept(2) errors logged with URI error:accept-client-connection.

+

Unused connections received in http_port or https_port +or transactions terminated before reading[parsing] request headers +logged with URI error:transaction-end-before-headers.

+
acl

New -m flag for note ACL to match substrings.

@@ -315,6 +321,8 @@
cache_peer

New option auth-no-keytab to let GSSAPI implementation determine which Kerberos credentials to use, instead of specifying a keytab.

+

Replaced option ssl with tls. Use of any +tls- prefixed options implies tls is enabled.

New option tls-min-version=1.N to set minimum TLS version allowed.

New option tls-default-ca replaces sslflags=NO_DEFAULT_CA

New option tls-no-npn to disable sending TLS NPN extension.

@@ -322,7 +330,8 @@ have been removed.

Removed sslversion= option. Use tls-options= instead.

Manual squid.conf update may be required on upgrade.

-

Replaced sslcafile= with tls-cafile= which takes multiple entries.

+

Replaced option sslcafile= with tls-cafile= +which takes multiple entries.

external_acl_type

New parameter queue-size= to set the maximum number @@ -534,6 +543,8 @@

Replaced by --with-cppunit=PATH. Please prefer the default auto-detection though.

+
XSTD_USE_LIBLTDL
+

Removed. Use --with-included-ltdl instead.

diff -u -r -N squid-4.0.12/src/acl/external/delayer/ext_delayer_acl.8 squid-4.0.13/src/acl/external/delayer/ext_delayer_acl.8 --- squid-4.0.12/src/acl/external/delayer/ext_delayer_acl.8 2016-07-02 02:23:27.000000000 +1200 +++ squid-4.0.13/src/acl/external/delayer/ext_delayer_acl.8 2016-08-06 02:26:06.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_DELAYER_ACL 8" -.TH EXT_DELAYER_ACL 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH EXT_DELAYER_ACL 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/acl/external/kerberos_ldap_group/kerberos_ldap_group.cc squid-4.0.13/src/acl/external/kerberos_ldap_group/kerberos_ldap_group.cc --- squid-4.0.12/src/acl/external/kerberos_ldap_group/kerberos_ldap_group.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/acl/external/kerberos_ldap_group/kerberos_ldap_group.cc 2016-08-06 00:52:55.000000000 +1200 @@ -214,7 +214,7 @@ margs.rc_allow = 1; break; case 's': - margs.ssl = (char *) "yes"; + margs.ssl = xstrdup("yes"); break; case 'n': margs.nokerberos = 1; diff -u -r -N squid-4.0.12/src/acl/external/kerberos_ldap_group/support_ldap.cc squid-4.0.13/src/acl/external/kerberos_ldap_group/support_ldap.cc --- squid-4.0.12/src/acl/external/kerberos_ldap_group/support_ldap.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/acl/external/kerberos_ldap_group/support_ldap.cc 2016-08-06 00:52:55.000000000 +1200 @@ -74,9 +74,13 @@ #define FILTER_AD "(samaccountname=%s)" #define ATTRIBUTE_AD "memberof" -size_t get_attributes(LDAP * ld, LDAPMessage * res, const char *attribute /* IN */ , char ***out_val /* OUT (caller frees) */ ); -size_t get_bin_attributes(LDAP * ld, LDAPMessage * res, const char *attribute /* IN */ , char ***out_val , int **out_len /* OUT (caller frees) */ ); -int search_group_tree(struct main_args *margs, LDAP * ld, char *bindp, char *ldap_group, char *group, int depth); +size_t get_attributes(LDAP * ld, LDAPMessage * res, + const char *attribute /* IN */ , char ***out_val /* OUT (caller frees) */ ); +size_t get_bin_attributes(LDAP * ld, LDAPMessage * res, + const char *attribute /* IN */ , char ***out_val, + int **out_len /* OUT (caller frees) */ ); +int search_group_tree(struct main_args *margs, LDAP * ld, char *bindp, + char *ldap_group, char *group, int depth); #if HAVE_SUN_LDAP_SDK || HAVE_MOZILLA_LDAP_SDK #if HAVE_LDAP_REBINDPROC_CALLBACK @@ -85,13 +89,8 @@ static LDAP_REBINDPROC_CALLBACK ldap_sasl_rebind; static int LDAP_CALL LDAP_CALLBACK -ldap_sasl_rebind( - LDAP * ld, - char **whop, - char **credp, - int *methodp, - int freeit, - void *params) +ldap_sasl_rebind(LDAP * ld, + char **whop, char **credp, int *methodp, int freeit, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; whop = whop; @@ -105,37 +104,29 @@ static LDAP_REBINDPROC_CALLBACK ldap_simple_rebind; static int LDAP_CALL LDAP_CALLBACK -ldap_simple_rebind( - LDAP * ld, - char **whop, - char **credp, - int *methodp, - int freeit, - void *params) +ldap_simple_rebind(LDAP * ld, + char **whop, char **credp, int *methodp, int freeit, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; struct berval cred; if (cp->pw) { - cred.bv_val=cp->pw; - cred.bv_len=strlen(cp->pw); + cred.bv_val = cp->pw; + cred.bv_len = strlen(cp->pw); } whop = whop; credp = credp; methodp = methodp; freeit = freeit; - return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, NULL); + return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, + NULL); } #elif HAVE_LDAP_REBIND_PROC #if HAVE_SASL_H || HAVE_SASL_SASL_H || HAVE_SASL_DARWIN static LDAP_REBIND_PROC ldap_sasl_rebind; static int -ldap_sasl_rebind( - LDAP * ld, - LDAP_CONST char *url, - ber_tag_t request, - ber_int_t msgid, - void *params) +ldap_sasl_rebind(LDAP * ld, + LDAP_CONST char *url, ber_tag_t request, ber_int_t msgid, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; return tool_sasl_bind(ld, cp->dn, cp->pw); @@ -145,20 +136,17 @@ static LDAP_REBIND_PROC ldap_simple_rebind; static int -ldap_simple_rebind( - LDAP * ld, - LDAP_CONST char *url, - ber_tag_t request, - ber_int_t msgid, - void *params) +ldap_simple_rebind(LDAP * ld, + LDAP_CONST char *url, ber_tag_t request, ber_int_t msgid, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; struct berval cred; if (cp->pw) { - cred.bv_val=cp->pw; - cred.bv_len=strlen(cp->pw); + cred.bv_val = cp->pw; + cred.bv_len = strlen(cp->pw); } - return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, NULL); + return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, + NULL); } #elif HAVE_LDAP_REBIND_FUNCTION @@ -169,13 +157,8 @@ static LDAP_REBIND_FUNCTION ldap_sasl_rebind; static int -ldap_sasl_rebind( - LDAP * ld, - char **whop, - char **credp, - int *methodp, - int freeit, - void *params) +ldap_sasl_rebind(LDAP * ld, + char **whop, char **credp, int *methodp, int freeit, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; whop = whop; @@ -189,25 +172,21 @@ static LDAP_REBIND_FUNCTION ldap_simple_rebind; static int -ldap_simple_rebind( - LDAP * ld, - char **whop, - char **credp, - int *methodp, - int freeit, - void *params) +ldap_simple_rebind(LDAP * ld, + char **whop, char **credp, int *methodp, int freeit, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; struct berval cred; if (cp->pw) { - cred.bv_val=cp->pw; - cred.bv_len=strlen(cp->pw); + cred.bv_val = cp->pw; + cred.bv_len = strlen(cp->pw); } whop = whop; credp = credp; methodp = methodp; freeit = freeit; - return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, NULL); + return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, + NULL); } #else #error "No rebind functione defined" @@ -217,7 +196,8 @@ static LDAP_REBIND_PROC ldap_sasl_rebind; static int -ldap_sasl_rebind(LDAP *ld, LDAP_CONST char *, ber_tag_t request, ber_int_t msgid, void *params) +ldap_sasl_rebind(LDAP * ld, LDAP_CONST char *, ber_tag_t request, + ber_int_t msgid, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; return tool_sasl_bind(ld, cp->dn, cp->pw); @@ -227,16 +207,18 @@ static LDAP_REBIND_PROC ldap_simple_rebind; static int -ldap_simple_rebind(LDAP *ld, LDAP_CONST char *, ber_tag_t request, ber_int_t msgid, void *params) +ldap_simple_rebind(LDAP * ld, LDAP_CONST char *, ber_tag_t request, + ber_int_t msgid, void *params) { struct ldap_creds *cp = (struct ldap_creds *) params; struct berval cred; if (cp->pw) { - cred.bv_val=cp->pw; - cred.bv_len=strlen(cp->pw); + cred.bv_val = cp->pw; + cred.bv_len = strlen(cp->pw); } - return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, NULL); + return ldap_sasl_bind_s(ld, cp->dn, LDAP_SASL_SIMPLE, &cred, NULL, NULL, + NULL); } #endif @@ -284,8 +266,7 @@ for (ldap_filter_esc = filter; *ldap_filter_esc; ++ldap_filter_esc) { if ((*ldap_filter_esc == '*') || (*ldap_filter_esc == '(') || - (*ldap_filter_esc == ')') || - (*ldap_filter_esc == '\\')) + (*ldap_filter_esc == ')') || (*ldap_filter_esc == '\\')) i = i + 3; } @@ -330,24 +311,34 @@ searchtime.tv_sec = SEARCH_TIMEOUT; searchtime.tv_usec = 0; - debug((char *) "%s| %s: DEBUG: Search ldap server with bind path \"\" and filter: %s\n", LogTime(), PROGRAM, FILTER_SCHEMA); - rc = ldap_search_ext_s(ld, (char *) "", LDAP_SCOPE_BASE, (char *) FILTER_SCHEMA, NULL, 0, - NULL, NULL, &searchtime, 0, &res); + debug((char *) + "%s| %s: DEBUG: Search ldap server with bind path \"\" and filter: %s\n", + LogTime(), PROGRAM, FILTER_SCHEMA); + rc = ldap_search_ext_s(ld, (char *) "", LDAP_SCOPE_BASE, + (char *) FILTER_SCHEMA, NULL, 0, NULL, NULL, &searchtime, 0, &res); if (rc == LDAP_SUCCESS) max_attr = get_attributes(ld, res, ATTRIBUTE_SCHEMA, &attr_value); if (max_attr == 1) { ldap_msgfree(res); - debug((char *) "%s| %s: DEBUG: Search ldap server with bind path %s and filter: %s\n", LogTime(), PROGRAM, attr_value[0], FILTER_SAM); - rc = ldap_search_ext_s(ld, attr_value[0], LDAP_SCOPE_SUBTREE, (char *) FILTER_SAM, NULL, 0, - NULL, NULL, &searchtime, 0, &res); - debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), PROGRAM, ldap_count_entries(ld, res), ldap_count_entries(ld, res) > 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); + debug((char *) + "%s| %s: DEBUG: Search ldap server with bind path %s and filter: %s\n", + LogTime(), PROGRAM, attr_value[0], FILTER_SAM); + rc = ldap_search_ext_s(ld, attr_value[0], LDAP_SCOPE_SUBTREE, + (char *) FILTER_SAM, NULL, 0, NULL, NULL, &searchtime, 0, &res); + debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), + PROGRAM, ldap_count_entries(ld, res), ldap_count_entries(ld, + res) > 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); if (ldap_count_entries(ld, res) > 0) margs->AD = 1; } else - debug((char *) "%s| %s: DEBUG: Did not find ldap entry for subschemasubentry\n", LogTime(), PROGRAM); - debug((char *) "%s| %s: DEBUG: Determined ldap server %sas an Active Directory server\n", LogTime(), PROGRAM, margs->AD ? "" : "not "); + debug((char *) + "%s| %s: DEBUG: Did not find ldap entry for subschemasubentry\n", + LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Determined ldap server %sas an Active Directory server\n", + LogTime(), PROGRAM, margs->AD ? "" : "not "); /* * Cleanup */ @@ -361,8 +352,10 @@ ldap_msgfree(res); return rc; } + int -search_group_tree(struct main_args *margs, LDAP * ld, char *bindp, char *ldap_group, char *group, int depth) +search_group_tree(struct main_args *margs, LDAP * ld, char *bindp, + char *ldap_group, char *group, int depth) { LDAPMessage *res = NULL; char **attr_value = NULL; @@ -395,21 +388,26 @@ xfree(ldap_filter_esc); if (depth > margs->mdepth) { - debug((char *) "%s| %s: DEBUG: Max search depth reached %d>%d\n", LogTime(), PROGRAM, depth, margs->mdepth); + debug((char *) "%s| %s: DEBUG: Max search depth reached %d>%d\n", + LogTime(), PROGRAM, depth, margs->mdepth); xfree(search_exp); return 0; } - debug((char *) "%s| %s: DEBUG: Search ldap server with bind path %s and filter : %s\n", LogTime(), PROGRAM, bindp, search_exp); - rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, - search_exp, NULL, 0, + debug((char *) + "%s| %s: DEBUG: Search ldap server with bind path %s and filter : %s\n", + LogTime(), PROGRAM, bindp, search_exp); + rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, search_exp, NULL, 0, NULL, NULL, &searchtime, 0, &res); xfree(search_exp); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error searching ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) "%s| %s: ERROR: Error searching ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); return 0; } - debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), PROGRAM, ldap_count_entries(ld, res), ldap_count_entries(ld, res) > 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); + debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), PROGRAM, + ldap_count_entries(ld, res), ldap_count_entries(ld, res) > 1 + || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); if (margs->AD) max_attr = get_attributes(ld, res, ATTRIBUTE_AD, &attr_value); @@ -435,21 +433,28 @@ } if (debug_enabled) { int n; - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" in hex UTF-8 is ", LogTime(), PROGRAM, j + 1, av); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" in hex UTF-8 is ", LogTime(), PROGRAM, j + 1, av); for (n = 0; av[n] != '\0'; ++n) fprintf(stderr, "%02x", (unsigned char) av[n]); fprintf(stderr, "\n"); } if (!strcasecmp(group, av)) { retval = 1; - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" matches group name \"%s\"\n", LogTime(), PROGRAM, j + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" matches group name \"%s\"\n", LogTime(), PROGRAM, + j + 1, av, group); break; } else - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" does not match group name \"%s\"\n", LogTime(), PROGRAM, j + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" does not match group name \"%s\"\n", LogTime(), + PROGRAM, j + 1, av, group); /* * Do recursive group search */ - debug((char *) "%s| %s: DEBUG: Perform recursive group search for group \"%s\"\n", LogTime(), PROGRAM, av); + debug((char *) + "%s| %s: DEBUG: Perform recursive group search for group \"%s\"\n", + LogTime(), PROGRAM, av); av = attr_value[j]; if (search_group_tree(margs, ld, bindp, av, group, ldepth)) { retval = 1; @@ -461,7 +466,9 @@ } } if (debug_enabled) - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" is member of group named \"%s\"\n", LogTime(), PROGRAM, j + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" is member of group named \"%s\"\n", LogTime(), + PROGRAM, j + 1, av, group); else break; @@ -492,12 +499,15 @@ val = LDAP_VERSION3; rc = ldap_set_option(ld, LDAP_OPT_PROTOCOL_VERSION, &val); if (rc != LDAP_SUCCESS) { - debug((char *) "%s| %s: DEBUG: Error while setting protocol version: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + debug((char *) + "%s| %s: DEBUG: Error while setting protocol version: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); return rc; } rc = ldap_set_option(ld, LDAP_OPT_REFERRALS, LDAP_OPT_OFF); if (rc != LDAP_SUCCESS) { - debug((char *) "%s| %s: DEBUG: Error while setting referrals off: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + debug((char *) "%s| %s: DEBUG: Error while setting referrals off: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); return rc; } #if LDAP_OPT_NETWORK_TIMEOUT @@ -505,7 +515,9 @@ tv.tv_usec = 0; rc = ldap_set_option(ld, LDAP_OPT_NETWORK_TIMEOUT, &tv); if (rc != LDAP_SUCCESS) { - debug((char *) "%s| %s: DEBUG: Error while setting network timeout: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + debug((char *) + "%s| %s: DEBUG: Error while setting network timeout: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); return rc; } #endif /* LDAP_OPT_NETWORK_TIMEOUT */ @@ -527,35 +539,75 @@ #if HAVE_OPENLDAP if (!margs->rc_allow) { char *ssl_cacertfile = NULL; - int free_path; - debug((char *) "%s| %s: DEBUG: Enable server certificate check for ldap server.\n", LogTime(), PROGRAM); + char *ssl_cacertdir = NULL; + debug((char *) + "%s| %s: DEBUG: Enable server certificate check for ldap server.\n", + LogTime(), PROGRAM); val = LDAP_OPT_X_TLS_DEMAND; rc = ldap_set_option(NULL, LDAP_OPT_X_TLS_REQUIRE_CERT, &val); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_REQUIRE_CERT DEMAND for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_REQUIRE_CERT DEMAND for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); return rc; } - ssl_cacertfile = getenv("TLS_CACERTFILE"); - free_path = 0; + ssl_cacertfile = xstrdup(getenv("TLS_CACERTFILE")); if (!ssl_cacertfile) { ssl_cacertfile = xstrdup("/etc/ssl/certs/cert.pem"); - free_path = 1; } - debug((char *) "%s| %s: DEBUG: Set certificate file for ldap server to %s.(Changeable through setting environment variable TLS_CACERTFILE)\n", LogTime(), PROGRAM, ssl_cacertfile); - rc = ldap_set_option(NULL, LDAP_OPT_X_TLS_CACERTFILE, ssl_cacertfile); - if (ssl_cacertfile && free_path) { + if (access(ssl_cacertfile, R_OK) == 0) { + debug((char *) + "%s| %s: DEBUG: Set certificate file for ldap server to %s. (Changeable through setting environment variable TLS_CACERTFILE)\n", + LogTime(), PROGRAM, ssl_cacertfile); + rc = ldap_set_option(NULL, LDAP_OPT_X_TLS_CACERTFILE, + ssl_cacertfile); xfree(ssl_cacertfile); - } - if (rc != LDAP_OPT_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_CACERTFILE for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); - return rc; + if (rc != LDAP_OPT_SUCCESS) { + error((char *) + "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_CACERTFILE for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); + return rc; + } + } else { + debug((char *) + "%s| %s: DEBUG: Set certificate file for ldap server to %s failed (%s). (Changeable through setting environment variable TLS_CACERTFILE) Trying db certificate directory\n", + LogTime(), PROGRAM, ssl_cacertfile, strerror(errno)); + xfree(ssl_cacertfile); + ssl_cacertdir = xstrdup(getenv("TLS_CACERTDIR")); + if (!ssl_cacertdir) { + ssl_cacertdir = xstrdup("/etc/ssl/certs"); + } + if (access(ssl_cacertdir, R_OK) == 0) { + debug((char *) + "%s| %s: DEBUG: Set certificate database path for ldap server to %s. (Changeable through setting environment variable TLS_CACERTDIR)\n", + LogTime(), PROGRAM, ssl_cacertdir); + rc = ldap_set_option(NULL, LDAP_OPT_X_TLS_CACERTDIR, + ssl_cacertdir); + xfree(ssl_cacertdir); + if (rc != LDAP_OPT_SUCCESS) { + error((char *) + "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_CACERTDIR for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); + return rc; + } + } else { + debug((char *) + "%s| %s: DEBUG: Set certificate database path for ldap server to %s failed (%s). (Changeable through setting environment variable TLS_CACERTDIR)\n", + LogTime(), PROGRAM, ssl_cacertdir, strerror(errno)); + xfree(ssl_cacertdir); + return errno; + } } } else { - debug((char *) "%s| %s: DEBUG: Disable server certificate check for ldap server.\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Disable server certificate check for ldap server.\n", + LogTime(), PROGRAM); val = LDAP_OPT_X_TLS_ALLOW; rc = ldap_set_option(NULL, LDAP_OPT_X_TLS_REQUIRE_CERT, &val); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_REQUIRE_CERT ALLOW for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting LDAP_OPT_X_TLS_REQUIRE_CERT ALLOW for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); return rc; } } @@ -571,26 +623,36 @@ if (!ssl_certdbpath) { ssl_certdbpath = xstrdup("/etc/certs"); } - debug((char *) "%s| %s: DEBUG: Set certificate database path for ldap server to %s.(Changeable through setting environment variable SSL_CERTDBPATH)\n", LogTime(), PROGRAM, ssl_certdbpath); + debug((char *) + "%s| %s: DEBUG: Set certificate database path for ldap server to %s. (Changeable through setting environment variable SSL_CERTDBPATH)\n", + LogTime(), PROGRAM, ssl_certdbpath); if (!margs->rc_allow) { - rc = ldapssl_advclientauth_init(ssl_certdbpath, NULL, 0, NULL, NULL, 0, NULL, 2); + rc = ldapssl_advclientauth_init(ssl_certdbpath, NULL, 0, NULL, NULL, 0, + NULL, 2); } else { - rc = ldapssl_advclientauth_init(ssl_certdbpath, NULL, 0, NULL, NULL, 0, NULL, 0); - debug((char *) "%s| %s: DEBUG: Disable server certificate check for ldap server.\n", LogTime(), PROGRAM); + rc = ldapssl_advclientauth_init(ssl_certdbpath, NULL, 0, NULL, NULL, 0, + NULL, 0); + debug((char *) + "%s| %s: DEBUG: Disable server certificate check for ldap server.\n", + LogTime(), PROGRAM); } xfree(ssl_certdbpath); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting SSL for ldap server: %s\n", LogTime(), PROGRAM, ldapssl_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting SSL for ldap server: %s\n", + LogTime(), PROGRAM, ldapssl_err2string(rc)); return rc; } #else - error((char *) "%s| %s: ERROR: SSL not supported by ldap library\n", LogTime(), PROGRAM); + error((char *) "%s| %s: ERROR: SSL not supported by ldap library\n", + LogTime(), PROGRAM); #endif return LDAP_SUCCESS; } size_t -get_attributes(LDAP * ld, LDAPMessage * res, const char *attribute, char ***ret_value) +get_attributes(LDAP * ld, LDAPMessage * res, const char *attribute, + char ***ret_value) { char **attr_value = *ret_value; @@ -599,8 +661,10 @@ /* * loop over attributes */ - debug((char *) "%s| %s: DEBUG: Search ldap entries for attribute : %s\n", LogTime(), PROGRAM, attribute); - for (LDAPMessage *msg = ldap_first_entry(ld, res); msg; msg = ldap_next_entry(ld, msg)) { + debug((char *) "%s| %s: DEBUG: Search ldap entries for attribute : %s\n", + LogTime(), PROGRAM, attribute); + for (LDAPMessage * msg = ldap_first_entry(ld, res); msg; + msg = ldap_next_entry(ld, msg)) { switch (ldap_msgtype(msg)) { @@ -611,15 +675,20 @@ if (strcasecmp(attr, attribute) == 0) { struct berval **values; - if ((values = ldap_get_values_len(ld, msg, attr)) != NULL) { + if ((values = + ldap_get_values_len(ld, msg, attr)) != NULL) { for (int il = 0; values[il] != NULL; ++il) { - attr_value = (char **) xrealloc(attr_value, (max_attr + 1) * sizeof(char *)); + attr_value = + (char **) xrealloc(attr_value, + (max_attr + 1) * sizeof(char *)); if (!attr_value) break; - attr_value[max_attr] = (char *) xmalloc(values[il]->bv_len + 1); - memcpy(attr_value[max_attr], values[il]->bv_val, values[il]->bv_len); + attr_value[max_attr] = + (char *) xmalloc(values[il]->bv_len + 1); + memcpy(attr_value[max_attr], values[il]->bv_val, + values[il]->bv_len); attr_value[max_attr][values[il]->bv_len] = 0; max_attr++; } @@ -632,24 +701,30 @@ } break; case LDAP_RES_SEARCH_REFERENCE: - debug((char *) "%s| %s: DEBUG: Received a search reference message\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Received a search reference message\n", + LogTime(), PROGRAM); break; case LDAP_RES_SEARCH_RESULT: - debug((char *) "%s| %s: DEBUG: Received a search result message\n", LogTime(), PROGRAM); + debug((char *) "%s| %s: DEBUG: Received a search result message\n", + LogTime(), PROGRAM); break; default: break; } } - debug((char *) "%s| %s: DEBUG: %" PRIuSIZE " ldap entr%s found with attribute : %s\n", LogTime(), PROGRAM, max_attr, max_attr > 1 || max_attr == 0 ? "ies" : "y", attribute); + debug((char *) "%s| %s: DEBUG: %" PRIuSIZE + " ldap entr%s found with attribute : %s\n", LogTime(), PROGRAM, + max_attr, max_attr > 1 || max_attr == 0 ? "ies" : "y", attribute); *ret_value = attr_value; return max_attr; } size_t -get_bin_attributes(LDAP * ld, LDAPMessage * res, const char *attribute, char ***ret_value, int **ret_len) +get_bin_attributes(LDAP * ld, LDAPMessage * res, const char *attribute, + char ***ret_value, int **ret_len) { char **attr_value = *ret_value; @@ -659,8 +734,10 @@ /* * loop over attributes */ - debug((char *) "%s| %s: DEBUG: Search ldap entries for attribute : %s\n", LogTime(), PROGRAM, attribute); - for ( LDAPMessage *msg = ldap_first_entry(ld, res); msg; msg = ldap_next_entry(ld, msg)) { + debug((char *) "%s| %s: DEBUG: Search ldap entries for attribute : %s\n", + LogTime(), PROGRAM, attribute); + for (LDAPMessage * msg = ldap_first_entry(ld, res); msg; + msg = ldap_next_entry(ld, msg)) { switch (ldap_msgtype(msg)) { @@ -671,21 +748,28 @@ if (strcasecmp(attr, attribute) == 0) { struct berval **values; - if ((values = ldap_get_values_len(ld, msg, attr)) != NULL) { + if ((values = + ldap_get_values_len(ld, msg, attr)) != NULL) { for (int il = 0; values[il] != NULL; ++il) { - attr_value = (char **) xrealloc(attr_value, (max_attr + 1) * sizeof(char *)); + attr_value = + (char **) xrealloc(attr_value, + (max_attr + 1) * sizeof(char *)); if (!attr_value) break; - attr_len = (int *) xrealloc(attr_len, (max_attr + 1) * sizeof(int)); + attr_len = + (int *) xrealloc(attr_len, + (max_attr + 1) * sizeof(int)); if (!attr_len) break; - attr_value[max_attr] = (char *) xmalloc(values[il]->bv_len + 1); - memcpy(attr_value[max_attr], values[il]->bv_val, values[il]->bv_len); + attr_value[max_attr] = + (char *) xmalloc(values[il]->bv_len + 1); + memcpy(attr_value[max_attr], values[il]->bv_val, + values[il]->bv_len); attr_value[max_attr][values[il]->bv_len] = 0; - attr_len[max_attr]=values[il]->bv_len; + attr_len[max_attr] = values[il]->bv_len; max_attr++; } } @@ -697,17 +781,22 @@ } break; case LDAP_RES_SEARCH_REFERENCE: - debug((char *) "%s| %s: DEBUG: Received a search reference message\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Received a search reference message\n", + LogTime(), PROGRAM); break; case LDAP_RES_SEARCH_RESULT: - debug((char *) "%s| %s: DEBUG: Received a search result message\n", LogTime(), PROGRAM); + debug((char *) "%s| %s: DEBUG: Received a search result message\n", + LogTime(), PROGRAM); break; default: break; } } - debug((char *) "%s| %s: DEBUG: %" PRIuSIZE " ldap entr%s found with attribute : %s\n", LogTime(), PROGRAM, max_attr, max_attr > 1 || max_attr == 0 ? "ies" : "y", attribute); + debug((char *) "%s| %s: DEBUG: %" PRIuSIZE + " ldap entr%s found with attribute : %s\n", LogTime(), PROGRAM, + max_attr, max_attr > 1 || max_attr == 0 ? "ies" : "y", attribute); *ret_value = attr_value; *ret_len = attr_len; @@ -752,7 +841,8 @@ #elif HAVE_LDAP_URL_PARSE rc = ldap_url_parse(ldapuri, &url); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while parsing url: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) "%s| %s: ERROR: Error while parsing url: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); xfree(ldapuri); ldap_free_urldesc(url); return NULL; @@ -764,8 +854,10 @@ rc = ldap_initialize(&ld, ldapuri); xfree(ldapuri); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while initialising connection to ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); - ldap_unbind_ext(ld,NULL,NULL); + error((char *) + "%s| %s: ERROR: Error while initialising connection to ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); + ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; } @@ -774,7 +866,9 @@ #endif rc = ldap_set_defaults(ld); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting default options for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting default options for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; @@ -786,7 +880,9 @@ debug((char *) "%s| %s: DEBUG: Set SSL defaults\n", LogTime(), PROGRAM); rc = ldap_set_ssl_defaults(margs); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting SSL default options for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting SSL default options for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; @@ -797,7 +893,9 @@ */ rc = ldap_start_tls_s(ld, NULL, NULL); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting start_tls for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + debug((char *) + "%s| %s: WARNING: Error while setting start_tls for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; url = (LDAPURLDesc *) xmalloc(sizeof(*url)); @@ -817,7 +915,8 @@ #elif HAVE_LDAP_URL_PARSE rc = ldap_url_parse(ldapuri, &url); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while parsing url: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) "%s| %s: ERROR: Error while parsing url: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); xfree(ldapuri); ldap_free_urldesc(url); return NULL; @@ -829,14 +928,18 @@ rc = ldap_initialize(&ld, ldapuri); xfree(ldapuri); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while initialising connection to ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while initialising connection to ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; } rc = ldap_set_defaults(ld); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting default options for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting default options for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; @@ -845,20 +948,25 @@ #elif HAVE_LDAPSSL_CLIENT_INIT ld = ldapssl_init(host, port, 1); if (!ld) { - error((char *) "%s| %s: ERROR: Error while setting SSL for ldap server: %s\n", LogTime(), PROGRAM, ldapssl_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting SSL for ldap server: %s\n", + LogTime(), PROGRAM, ldapssl_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; } rc = ldap_set_defaults(ld); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while setting default options for ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while setting default options for ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; return NULL; } #else - error((char *) "%s| %s: ERROR: SSL not supported by ldap library\n", LogTime(), PROGRAM); + error((char *) "%s| %s: ERROR: SSL not supported by ldap library\n", + LogTime(), PROGRAM); #endif } return ld; @@ -895,21 +1003,28 @@ * Fill Kerberos memory cache with credential from keytab for SASL/GSSAPI */ if (domain) { - debug((char *) "%s| %s: DEBUG: Setup Kerberos credential cache\n", LogTime(), PROGRAM); + debug((char *) "%s| %s: DEBUG: Setup Kerberos credential cache\n", + LogTime(), PROGRAM); #if HAVE_KRB5 if (margs->nokerberos) { kc = 1; - debug((char *) "%s| %s: DEBUG: Kerberos is disabled. Use username/password with ldap url instead\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Kerberos is disabled. Use username/password with ldap url instead\n", + LogTime(), PROGRAM); } else { kc = krb5_create_cache(domain); if (kc) { - error((char *) "%s| %s: ERROR: Error during setup of Kerberos credential cache\n", LogTime(), PROGRAM); + error((char *) + "%s| %s: ERROR: Error during setup of Kerberos credential cache\n", + LogTime(), PROGRAM); } } #else kc = 1; - debug((char *) "%s| %s: DEBUG: Kerberos is not supported. Use username/password with ldap url instead\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Kerberos is not supported. Use username/password with ldap url instead\n", + LogTime(), PROGRAM); #endif } @@ -929,13 +1044,17 @@ ldap_debug = 0; (void) ldap_set_option(NULL, LDAP_OPT_DEBUG_LEVEL, &ldap_debug); #endif - debug((char *) "%s| %s: DEBUG: Initialise ldap connection\n", LogTime(), PROGRAM); + debug((char *) "%s| %s: DEBUG: Initialise ldap connection\n", LogTime(), + PROGRAM); if (domain && !kc) { if (margs->ssl) { - debug((char *) "%s| %s: DEBUG: Enable SSL to ldap servers\n", LogTime(), PROGRAM); + debug((char *) "%s| %s: DEBUG: Enable SSL to ldap servers\n", + LogTime(), PROGRAM); } - debug((char *) "%s| %s: DEBUG: Canonicalise ldap server name for domain %s\n", LogTime(), PROGRAM, domain); + debug((char *) + "%s| %s: DEBUG: Canonicalise ldap server name for domain %s\n", + LogTime(), PROGRAM, domain); /* * Loop over list of ldap servers of users domain */ @@ -944,7 +1063,9 @@ int port = 389; if (hlist[i].port != -1) port = hlist[i].port; - debug((char *) "%s| %s: DEBUG: Setting up connection to ldap server %s:%d\n", LogTime(), PROGRAM, hlist[i].host, port); + debug((char *) + "%s| %s: DEBUG: Setting up connection to ldap server %s:%d\n", + LogTime(), PROGRAM, hlist[i].host, port); ld = tool_ldap_open(margs, hlist[i].host, port, margs->ssl); if (!ld) @@ -955,11 +1076,15 @@ */ #if HAVE_SASL_H || HAVE_SASL_SASL_H || HAVE_SASL_DARWIN - debug((char *) "%s| %s: DEBUG: Bind to ldap server with SASL/GSSAPI\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Bind to ldap server with SASL/GSSAPI\n", + LogTime(), PROGRAM); rc = tool_sasl_bind(ld, bindp, margs->ssl); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while binding to ldap server with SASL/GSSAPI: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while binding to ldap server with SASL/GSSAPI: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; continue; @@ -969,19 +1094,25 @@ lcreds->pw = margs->ssl ? xstrdup(margs->ssl) : NULL; ldap_set_rebind_proc(ld, ldap_sasl_rebind, (char *) lcreds); if (ld != NULL) { - debug((char *) "%s| %s: DEBUG: %s initialised %sconnection to ldap server %s:%d\n", LogTime(), PROGRAM, ld ? "Successfully" : "Failed to", margs->ssl ? "SSL protected " : "", hlist[i].host, port); + debug((char *) + "%s| %s: DEBUG: %s initialised %sconnection to ldap server %s:%d\n", + LogTime(), PROGRAM, ld ? "Successfully" : "Failed to", + margs->ssl ? "SSL protected " : "", hlist[i].host, port); break; } #else ldap_unbind_ext(ld, NULL, NULL); ld = NULL; - error((char *) "%s| %s: ERROR: SASL not supported on system\n", LogTime(), PROGRAM); + error((char *) "%s| %s: ERROR: SASL not supported on system\n", + LogTime(), PROGRAM); continue; #endif } nhosts = free_hostname_list(&hlist, nhosts); if (ld == NULL) { - debug((char *) "%s| %s: DEBUG: Error during initialisation of ldap connection: %s\n", LogTime(), PROGRAM, strerror(errno)); + debug((char *) + "%s| %s: DEBUG: Error during initialisation of ldap connection: %s\n", + LogTime(), PROGRAM, strerror(errno)); } bindp = convert_domain_to_bind_path(domain); } @@ -997,9 +1128,11 @@ hostname = strstr(margs->lurl, "://") + 3; ssl = strstr(margs->lurl, "ldaps://"); if (ssl) { - debug((char *) "%s| %s: DEBUG: Enable SSL to ldap servers\n", LogTime(), PROGRAM); + debug((char *) "%s| %s: DEBUG: Enable SSL to ldap servers\n", + LogTime(), PROGRAM); } - debug((char *) "%s| %s: DEBUG: Canonicalise ldap server name %s\n", LogTime(), PROGRAM, hostname); + debug((char *) "%s| %s: DEBUG: Canonicalise ldap server name %s\n", + LogTime(), PROGRAM, hostname); /* * Loop over list of ldap servers */ @@ -1015,8 +1148,8 @@ for (size_t i = 0; i < nhosts; ++i) { struct berval cred; if (margs->lpass) { - cred.bv_val=margs->lpass; - cred.bv_len=strlen(margs->lpass); + cred.bv_val = margs->lpass; + cred.bv_len = strlen(margs->lpass); } ld = tool_ldap_open(margs, hlist[i].host, port, ssl); if (!ld) @@ -1025,10 +1158,15 @@ * ldap bind with username/password authentication */ - debug((char *) "%s| %s: DEBUG: Bind to ldap server with Username/Password\n", LogTime(), PROGRAM); - rc = ldap_sasl_bind_s(ld, margs->luser, LDAP_SASL_SIMPLE, &cred, NULL, NULL, NULL); + debug((char *) + "%s| %s: DEBUG: Bind to ldap server with Username/Password\n", + LogTime(), PROGRAM); + rc = ldap_sasl_bind_s(ld, margs->luser, LDAP_SASL_SIMPLE, &cred, + NULL, NULL, NULL); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error while binding to ldap server with Username/Password: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error while binding to ldap server with Username/Password: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; continue; @@ -1037,7 +1175,10 @@ lcreds->dn = xstrdup(margs->luser); lcreds->pw = xstrdup(margs->lpass); ldap_set_rebind_proc(ld, ldap_simple_rebind, (char *) lcreds); - debug((char *) "%s| %s: DEBUG: %s set up %sconnection to ldap server %s:%d\n", LogTime(), PROGRAM, ld ? "Successfully" : "Failed to", ssl ? "SSL protected " : "", hlist[i].host, port); + debug((char *) + "%s| %s: DEBUG: %s set up %sconnection to ldap server %s:%d\n", + LogTime(), PROGRAM, ld ? "Successfully" : "Failed to", + ssl ? "SSL protected " : "", hlist[i].host, port); break; } @@ -1050,7 +1191,9 @@ } } if (ld == NULL) { - debug((char *) "%s| %s: DEBUG: Error during initialisation of ldap connection: %s\n", LogTime(), PROGRAM, strerror(errno)); + debug((char *) + "%s| %s: DEBUG: Error during initialisation of ldap connection: %s\n", + LogTime(), PROGRAM, strerror(errno)); retval = 0; goto cleanup; } @@ -1063,7 +1206,9 @@ margs->AD = 0; rc = check_AD(margs, ld); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error determining ldap server type: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) + "%s| %s: ERROR: Error determining ldap server type: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; retval = 0; @@ -1082,20 +1227,24 @@ xfree(ldap_filter_esc); - debug((char *) "%s| %s: DEBUG: Search ldap server with bind path %s and filter : %s\n", LogTime(), PROGRAM, bindp, search_exp); - rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, - search_exp, NULL, 0, + debug((char *) + "%s| %s: DEBUG: Search ldap server with bind path %s and filter : %s\n", + LogTime(), PROGRAM, bindp, search_exp); + rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, search_exp, NULL, 0, NULL, NULL, &searchtime, 0, &res); xfree(search_exp); if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error searching ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) "%s| %s: ERROR: Error searching ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); ldap_unbind_ext(ld, NULL, NULL); ld = NULL; retval = 0; goto cleanup; } - debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), PROGRAM, ldap_count_entries(ld, res), ldap_count_entries(ld, res) > 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); + debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), PROGRAM, + ldap_count_entries(ld, res), ldap_count_entries(ld, res) > 1 + || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); if (ldap_count_entries(ld, res) != 0) { @@ -1122,7 +1271,8 @@ } } if (debug_enabled) { - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" in hex UTF-8 is ", LogTime(), PROGRAM, k + 1, av); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" in hex UTF-8 is ", LogTime(), PROGRAM, k + 1, av); for (unsigned int n = 0; av[n] != '\0'; ++n) fprintf(stderr, "%02x", (unsigned char) av[n]); fprintf(stderr, "\n"); @@ -1130,18 +1280,24 @@ if (!strcasecmp(group, av)) { retval = 1; if (debug_enabled) - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" matches group name \"%s\"\n", LogTime(), PROGRAM, k + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" matches group name \"%s\"\n", LogTime(), + PROGRAM, k + 1, av, group); else break; } else - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " \"%s\" does not match group name \"%s\"\n", LogTime(), PROGRAM, k + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " \"%s\" does not match group name \"%s\"\n", LogTime(), + PROGRAM, k + 1, av, group); } /* * Do recursive group search for AD only since posixgroups can not contain other groups */ if (!retval && margs->AD) { if (debug_enabled && max_attr > 0) { - debug((char *) "%s| %s: DEBUG: Perform recursive group search\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Perform recursive group search\n", + LogTime(), PROGRAM); } for (size_t j = 0; j < max_attr; ++j) { char *av = NULL; @@ -1157,7 +1313,9 @@ } } if (debug_enabled) - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " group \"%s\" is (in)direct member of group \"%s\"\n", LogTime(), PROGRAM, j + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " group \"%s\" is (in)direct member of group \"%s\"\n", + LogTime(), PROGRAM, j + 1, av, group); else break; } @@ -1188,7 +1346,9 @@ /* * Check for primary Group membership */ - debug((char *) "%s| %s: DEBUG: Search for primary group membership: \"%s\"\n", LogTime(), PROGRAM, group); + debug((char *) + "%s| %s: DEBUG: Search for primary group membership: \"%s\"\n", + LogTime(), PROGRAM, group); if (margs->AD) filter = (char *) FILTER_AD; else @@ -1202,18 +1362,22 @@ xfree(ldap_filter_esc); - debug((char *) "%s| %s: DEBUG: Search ldap server with bind path %s and filter: %s\n", LogTime(), PROGRAM, bindp, search_exp); - rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, - search_exp, NULL, 0, - NULL, NULL, &searchtime, 0, &res); + debug((char *) + "%s| %s: DEBUG: Search ldap server with bind path %s and filter: %s\n", + LogTime(), PROGRAM, bindp, search_exp); + rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, search_exp, NULL, + 0, NULL, NULL, &searchtime, 0, &res); xfree(search_exp); - debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), PROGRAM, ldap_count_entries(ld, res), ldap_count_entries(ld, res) > 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); + debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", LogTime(), + PROGRAM, ldap_count_entries(ld, res), ldap_count_entries(ld, + res) > 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); max_attr = 0; if (!rc) { if (margs->AD) - max_attr = get_attributes(ld, res, ATTRIBUTE_GID_AD, &attr_value); + max_attr = + get_attributes(ld, res, ATTRIBUTE_GID_AD, &attr_value); else max_attr = get_attributes(ld, res, ATTRIBUTE_GID, &attr_value); } @@ -1226,48 +1390,61 @@ char **attr_value_3 = NULL; int *attr_len_3 = NULL; size_t max_attr_3 = 0; - uint32_t gid=atoi(attr_value[0]); + uint32_t gid = atoi(attr_value[0]); /* Get objectsid and search for group * with objectsid = domain(objectsid) + primarygroupid */ - debug((char *) "%s| %s: DEBUG: Got primaryGroupID %u\n", LogTime(), PROGRAM, gid); - max_attr_3 = get_bin_attributes(ld, res, ATTRIBUTE_SID, &attr_value_3, &attr_len_3); + debug((char *) "%s| %s: DEBUG: Got primaryGroupID %u\n", + LogTime(), PROGRAM, gid); + max_attr_3 = + get_bin_attributes(ld, res, ATTRIBUTE_SID, &attr_value_3, + &attr_len_3); ldap_msgfree(res); if (max_attr_3 == 1) { - int len=attr_len_3[0]; + int len = attr_len_3[0]; if (len < 4) { - debug((char *) "%s| %s: ERROR: Length %d is too short for objectSID\n", LogTime(), PROGRAM, len); + debug((char *) + "%s| %s: ERROR: Length %d is too short for objectSID\n", + LogTime(), PROGRAM, len); rc = 1; } else { - char *se=NULL; - attr_value_3[0][len-1]=((gid>>24) & 0xff); - attr_value_3[0][len-2]=((gid>>16) & 0xff); - attr_value_3[0][len-3]=((gid>>8) & 0xff); - attr_value_3[0][len-4]=((gid>>0) & 0xff); + char *se = NULL; + attr_value_3[0][len - 1] = ((gid >> 24) & 0xff); + attr_value_3[0][len - 2] = ((gid >> 16) & 0xff); + attr_value_3[0][len - 3] = ((gid >> 8) & 0xff); + attr_value_3[0][len - 4] = ((gid >> 0) & 0xff); #define FILTER_SID_1 "(objectSID=" #define FILTER_SID_2 ")" - se_len = strlen(FILTER_SID_1) + len*3 + strlen(FILTER_SID_2) + 1; + se_len = + strlen(FILTER_SID_1) + len * 3 + + strlen(FILTER_SID_2) + 1; search_exp = (char *) xmalloc(se_len); - snprintf(search_exp, se_len, "%s", FILTER_SID_1 ); + snprintf(search_exp, se_len, "%s", FILTER_SID_1); - for (int j=0; j 1 || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); + debug((char *) "%s| %s: DEBUG: Found %d ldap entr%s\n", + LogTime(), PROGRAM, ldap_count_entries(ld, res), + ldap_count_entries(ld, res) > 1 + || ldap_count_entries(ld, res) == 0 ? "ies" : "y"); } } else { @@ -1295,18 +1472,21 @@ xfree(ldap_filter_esc); - debug((char *) "%s| %s: DEBUG: Search ldap server with bind path %s and filter: %s\n", LogTime(), PROGRAM, bindp, search_exp); + debug((char *) + "%s| %s: DEBUG: Search ldap server with bind path %s and filter: %s\n", + LogTime(), PROGRAM, bindp, search_exp); rc = ldap_search_ext_s(ld, bindp, LDAP_SCOPE_SUBTREE, - search_exp, NULL, 0, - NULL, NULL, &searchtime, 0, &res); + search_exp, NULL, 0, NULL, NULL, &searchtime, 0, &res); xfree(search_exp); } if (!rc) { if (margs->AD) - max_attr_2 = get_attributes(ld, res, ATTRIBUTE_DN, &attr_value_2); + max_attr_2 = + get_attributes(ld, res, ATTRIBUTE_DN, &attr_value_2); else - max_attr_2 = get_attributes(ld, res, ATTRIBUTE, &attr_value_2); + max_attr_2 = + get_attributes(ld, res, ATTRIBUTE, &attr_value_2); ldap_msgfree(res); } else { ldap_msgfree(res); @@ -1327,9 +1507,13 @@ } if (!strcasecmp(group, av)) { retval = 1; - debug((char *) "%s| %s: DEBUG: \"%s\" matches group name \"%s\"\n", LogTime(), PROGRAM, av, group); + debug((char *) + "%s| %s: DEBUG: \"%s\" matches group name \"%s\"\n", + LogTime(), PROGRAM, av, group); } else - debug((char *) "%s| %s: DEBUG: \"%s\" does not match group name \"%s\"\n", LogTime(), PROGRAM, av, group); + debug((char *) + "%s| %s: DEBUG: \"%s\" does not match group name \"%s\"\n", + LogTime(), PROGRAM, av, group); } /* @@ -1337,7 +1521,9 @@ */ if (!retval && margs->AD) { if (debug_enabled && max_attr_2 > 0) { - debug((char *) "%s| %s: DEBUG: Perform recursive group search\n", LogTime(), PROGRAM); + debug((char *) + "%s| %s: DEBUG: Perform recursive group search\n", + LogTime(), PROGRAM); } for (size_t j = 0; j < max_attr_2; ++j) { char *av = NULL; @@ -1353,7 +1539,9 @@ } } if (debug_enabled) { - debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE " group \"%s\" is (in)direct member of group \"%s\"\n", LogTime(), PROGRAM, j + 1, av, group); + debug((char *) "%s| %s: DEBUG: Entry %" PRIuSIZE + " group \"%s\" is (in)direct member of group \"%s\"\n", + LogTime(), PROGRAM, j + 1, av, group); } else { break; } @@ -1371,11 +1559,15 @@ safe_free(attr_value_2); } - debug((char *) "%s| %s: DEBUG: Users primary group %s %s\n", LogTime(), PROGRAM, retval ? "matches" : "does not match", group); + debug((char *) "%s| %s: DEBUG: Users primary group %s %s\n", + LogTime(), PROGRAM, retval ? "matches" : "does not match", + group); } else { ldap_msgfree(res); - debug((char *) "%s| %s: DEBUG: Did not find ldap entry for group %s\n", LogTime(), PROGRAM, group); + debug((char *) + "%s| %s: DEBUG: Did not find ldap entry for group %s\n", + LogTime(), PROGRAM, group); } /* * Cleanup @@ -1390,7 +1582,8 @@ rc = ldap_unbind_ext(ld, NULL, NULL); ld = NULL; if (rc != LDAP_SUCCESS) { - error((char *) "%s| %s: ERROR: Error unbind ldap server: %s\n", LogTime(), PROGRAM, ldap_err2string(rc)); + error((char *) "%s| %s: ERROR: Error unbind ldap server: %s\n", + LogTime(), PROGRAM, ldap_err2string(rc)); } debug((char *) "%s| %s: DEBUG: Unbind ldap server\n", LogTime(), PROGRAM); cleanup: diff -u -r -N squid-4.0.12/src/acl/external/SQL_session/ext_sql_session_acl.8 squid-4.0.13/src/acl/external/SQL_session/ext_sql_session_acl.8 --- squid-4.0.12/src/acl/external/SQL_session/ext_sql_session_acl.8 2016-07-02 02:23:34.000000000 +1200 +++ squid-4.0.13/src/acl/external/SQL_session/ext_sql_session_acl.8 2016-08-06 02:26:22.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_SQL_SESSION_ACL 8" -.TH EXT_SQL_SESSION_ACL 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH EXT_SQL_SESSION_ACL 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 squid-4.0.13/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 --- squid-4.0.12/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2016-07-02 02:23:38.000000000 +1200 +++ squid-4.0.13/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2016-08-06 02:26:28.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_WBINFO_GROUP_ACL 8" -.TH EXT_WBINFO_GROUP_ACL 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH EXT_WBINFO_GROUP_ACL 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/adaptation/icap/ServiceRep.cc squid-4.0.13/src/adaptation/icap/ServiceRep.cc --- squid-4.0.12/src/adaptation/icap/ServiceRep.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/adaptation/icap/ServiceRep.cc 2016-08-06 00:52:55.000000000 +1200 @@ -34,6 +34,9 @@ Adaptation::Icap::ServiceRep::ServiceRep(const ServiceConfigPointer &svcCfg): AsyncJob("Adaptation::Icap::ServiceRep"), Adaptation::Service(svcCfg), sslContext(NULL), +#if USE_OPENSSL + sslSession(NULL), +#endif theOptions(NULL), theOptionsFetcher(0), theLastUpdate(0), theBusyConns(0), theAllWaiters(0), diff -u -r -N squid-4.0.12/src/adaptation/icap/ServiceRep.h squid-4.0.13/src/adaptation/icap/ServiceRep.h --- squid-4.0.12/src/adaptation/icap/ServiceRep.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/adaptation/icap/ServiceRep.h 2016-08-06 00:52:55.000000000 +1200 @@ -111,7 +111,9 @@ virtual void noteAdaptationAnswer(const Answer &answer); Security::ContextPtr sslContext; - Security::SessionStatePointer sslSession; +#if USE_OPENSSL + SSL_SESSION *sslSession; +#endif private: // stores Prepare() callback info diff -u -r -N squid-4.0.12/src/adaptation/icap/Xaction.cc squid-4.0.13/src/adaptation/icap/Xaction.cc --- squid-4.0.12/src/adaptation/icap/Xaction.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/adaptation/icap/Xaction.cc 2016-08-06 00:52:55.000000000 +1200 @@ -30,26 +30,26 @@ #include "icap_log.h" #include "ipcache.h" #include "pconn.h" +#include "security/PeerConnector.h" #include "SquidConfig.h" #include "SquidTime.h" -#if USE_OPENSSL -/// Gives Ssl::PeerConnector access to Answer in the PeerPoolMgr callback dialer. +/// Gives Security::PeerConnector access to Answer in the PeerPoolMgr callback dialer. class MyIcapAnswerDialer: public UnaryMemFunT, - public Ssl::PeerConnector::CbDialer + public Security::PeerConnector::CbDialer { public: MyIcapAnswerDialer(const JobPointer &aJob, Method aMethod): UnaryMemFunT(aJob, aMethod, Security::EncryptorAnswer()) {} - /* Ssl::PeerConnector::CbDialer API */ + /* Security::PeerConnector::CbDialer API */ virtual Security::EncryptorAnswer &answer() { return arg1; } }; namespace Ssl { /// A simple PeerConnector for Secure ICAP services. No SslBump capabilities. -class IcapPeerConnector: public PeerConnector { +class IcapPeerConnector: public Security::PeerConnector { CBDATA_CLASS(IcapPeerConnector); public: IcapPeerConnector( @@ -61,8 +61,8 @@ AsyncJob("Ssl::IcapPeerConnector"), PeerConnector(aServerConn, aCallback, alp, timeout), icapService(service) {} - /* PeerConnector API */ - virtual Security::SessionPtr initializeSsl(); + /* Security::PeerConnector API */ + virtual bool initializeTls(Security::SessionPointer &); virtual void noteNegotiationDone(ErrorState *error); virtual Security::ContextPtr getSslContext() {return icapService->sslContext;} @@ -72,7 +72,6 @@ } // namespace Ssl CBDATA_NAMESPACED_CLASS_INIT(Ssl, IcapPeerConnector); -#endif Adaptation::Icap::Xaction::Xaction(const char *aTypeName, Adaptation::Icap::ServiceRep::Pointer &aService): AsyncJob(aTypeName), @@ -302,7 +301,6 @@ CloseDialer(this,&Adaptation::Icap::Xaction::noteCommClosed)); comm_add_close_handler(io.conn->fd, closer); -#if USE_OPENSSL // If it is a reused connection and the SSL object is build // we should not negotiate new SSL session const auto &ssl = fd_table[io.conn->fd].ssl; @@ -311,13 +309,10 @@ securer = asyncCall(93, 4, "Adaptation::Icap::Xaction::handleSecuredPeer", MyIcapAnswerDialer(me, &Adaptation::Icap::Xaction::handleSecuredPeer)); - Ssl::PeerConnector::HttpRequestPointer tmpReq(NULL); - Ssl::IcapPeerConnector *sslConnector = - new Ssl::IcapPeerConnector(theService, io.conn, securer, masterLogEntry(), TheConfig.connect_timeout(service().cfg().bypass)); + auto *sslConnector = new Ssl::IcapPeerConnector(theService, io.conn, securer, masterLogEntry(), TheConfig.connect_timeout(service().cfg().bypass)); AsyncJob::Start(sslConnector); // will call our callback return; } -#endif // ?? fd_table[io.conn->fd].noteUse(icapPconnPool); service().noteConnectionUse(connection); @@ -709,25 +704,28 @@ return false; } -#if USE_OPENSSL -Security::SessionPtr -Ssl::IcapPeerConnector::initializeSsl() +bool +Ssl::IcapPeerConnector::initializeTls(Security::SessionPointer &serverSession) { - auto ssl = Ssl::PeerConnector::initializeSsl(); - if (!ssl) - return nullptr; + if (!Security::PeerConnector::initializeTls(serverSession)) + return false; +#if USE_OPENSSL assert(!icapService->cfg().secure.sslDomain.isEmpty()); SBuf *host = new SBuf(icapService->cfg().secure.sslDomain); - SSL_set_ex_data(ssl, ssl_ex_index_server, host); + SSL_set_ex_data(serverSession.get(), ssl_ex_index_server, host); - ACLFilledChecklist *check = (ACLFilledChecklist *)SSL_get_ex_data(ssl, ssl_ex_index_cert_error_check); + ACLFilledChecklist *check = static_cast(SSL_get_ex_data(serverSession.get(), ssl_ex_index_cert_error_check)); if (check) check->dst_peer_name = *host; - Security::GetSessionResumeData(Security::SessionPointer(ssl), icapService->sslSession); + if (icapService->sslSession) + SSL_set_session(serverSession.get(), icapService->sslSession); - return ssl; + return true; +#else + return false; +#endif } void @@ -736,8 +734,16 @@ if (error) return; +#if USE_OPENSSL const int fd = serverConnection()->fd; - Security::GetSessionResumeData(fd_table[fd].ssl, icapService->sslSession); + auto ssl = fd_table[fd].ssl.get(); + assert(ssl); + if (!SSL_session_reused(ssl)) { + if (icapService->sslSession) + SSL_SESSION_free(icapService->sslSession); + icapService->sslSession = SSL_get1_session(ssl); + } +#endif } void @@ -770,5 +776,4 @@ handleCommConnected(); } -#endif diff -u -r -N squid-4.0.12/src/adaptation/icap/Xaction.h squid-4.0.13/src/adaptation/icap/Xaction.h --- squid-4.0.12/src/adaptation/icap/Xaction.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/adaptation/icap/Xaction.h 2016-08-06 00:52:55.000000000 +1200 @@ -16,9 +16,6 @@ #include "HttpReply.h" #include "ipcache.h" #include "sbuf/SBuf.h" -#if USE_OPENSSL -#include "ssl/PeerConnector.h" -#endif class MemBuf; diff -u -r -N squid-4.0.12/src/anyp/PortCfg.cc squid-4.0.13/src/anyp/PortCfg.cc --- squid-4.0.12/src/anyp/PortCfg.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/anyp/PortCfg.cc 2016-08-06 00:52:55.000000000 +1200 @@ -143,8 +143,7 @@ } } - secure.staticContext.reset(secure.createStaticServerContext(*this)); - if (!secure.staticContext) { + if (!secure.createStaticServerContext(*this)) { char buf[128]; fatalf("%s_port %s initialization error", AnyP::ProtocolType_str[transport.protocol], s.toUrl(buf, sizeof(buf))); } diff -u -r -N squid-4.0.12/src/auth/basic/DB/basic_db_auth.8 squid-4.0.13/src/auth/basic/DB/basic_db_auth.8 --- squid-4.0.12/src/auth/basic/DB/basic_db_auth.8 2016-07-02 02:24:02.000000000 +1200 +++ squid-4.0.13/src/auth/basic/DB/basic_db_auth.8 2016-08-06 02:27:20.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_DB_AUTH 8" -.TH BASIC_DB_AUTH 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH BASIC_DB_AUTH 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -182,7 +182,10 @@ Database contains plain-text passwords .IP "\fB\-\-md5\fR" 12 .IX Item "--md5" -Database contains unsalted md5 passwords +Database contains unsalted \s-1MD5\s0 passwords +.IP "\fB\-\-sha1\fR" 12 +.IX Item "--sha1" +Database contains unsalted \s-1SHA1\s0 passwords .IP "\fB\-\-salt\fR" 12 .IX Item "--salt" Selects the correct salt to evaluate passwords diff -u -r -N squid-4.0.12/src/auth/basic/DB/basic_db_auth.pl.in squid-4.0.13/src/auth/basic/DB/basic_db_auth.pl.in --- squid-4.0.12/src/auth/basic/DB/basic_db_auth.pl.in 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/auth/basic/DB/basic_db_auth.pl.in 2016-08-06 00:52:55.000000000 +1200 @@ -61,7 +61,11 @@ =item B<--md5> -Database contains unsalted md5 passwords +Database contains unsalted MD5 passwords + +=item B<--sha1> + +Database contains unsalted SHA1 passwords =item B<--salt> @@ -127,6 +131,7 @@ use DBI; use Digest::MD5 qw(md5 md5_hex md5_base64); +use Digest::SHA qw(sha1 sha1_hex sha1_base64); my $dsn = "DBI:mysql:database=squid"; my $db_user = undef; @@ -137,6 +142,7 @@ my $db_cond = "enabled = 1"; my $plaintext = 0; my $md5 = 0; +my $sha1 = 0; my $persist = 0; my $isjoomla = 0; my $debug = 0; @@ -152,6 +158,7 @@ 'cond=s' => \$db_cond, 'plaintext' => \$plaintext, 'md5' => \$md5, + 'sha1' => \$sha1, 'persist' => \$persist, 'joomla' => \$isjoomla, 'debug' => \$debug, @@ -174,7 +181,7 @@ return $_sth if defined $_sth; $_dbh = DBI->connect($dsn, $db_user, $db_passwd); if (!defined $_dbh) { - warn ("Could not connect to $dsn\n"); + warn ("Could not connect to $dsn\n"); my @driver_names = DBI->available_drivers(); my $msg = "DSN drivers apparently installed, available:\n"; foreach my $dn (@driver_names) { @@ -203,6 +210,7 @@ return 1 if defined $hashsalt && crypt($password, $hashsalt) eq $key; return 1 if crypt($password, $key) eq $key; return 1 if $md5 && md5_hex($password) eq $key; + return 1 if $sha1 && sha1_hex($password) eq $key; return 1 if $plaintext && $password eq $key; } diff -u -r -N squid-4.0.12/src/auth/basic/POP3/basic_pop3_auth.8 squid-4.0.13/src/auth/basic/POP3/basic_pop3_auth.8 --- squid-4.0.12/src/auth/basic/POP3/basic_pop3_auth.8 2016-07-02 02:24:09.000000000 +1200 +++ squid-4.0.13/src/auth/basic/POP3/basic_pop3_auth.8 2016-08-06 02:27:35.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_POP3_AUTH 8" -.TH BASIC_POP3_AUTH 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH BASIC_POP3_AUTH 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/auth/User.cc squid-4.0.13/src/auth/User.cc --- squid-4.0.12/src/auth/User.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/auth/User.cc 2016-08-06 00:52:55.000000000 +1200 @@ -201,7 +201,7 @@ /* This ip has already been seen. */ found = 1; /* update IP ttl */ - ipdata->ip_expiretime = squid_curtime; + ipdata->ip_expiretime = squid_curtime + ::Config.authenticateIpTTL; } else if (ipdata->ip_expiretime <= squid_curtime) { /* This IP has expired - remove from the seen list */ dlinkDelete(&ipdata->node, &ip_list); diff -u -r -N squid-4.0.12/src/base/AsyncJob.cc squid-4.0.13/src/base/AsyncJob.cc --- squid-4.0.12/src/base/AsyncJob.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/base/AsyncJob.cc 2016-08-06 00:52:55.000000000 +1200 @@ -124,8 +124,9 @@ } void -AsyncJob::callException(const std::exception &) +AsyncJob::callException(const std::exception &ex) { + debugs(93, 2, ex.what()); // we must be called asynchronously and hence, the caller must lock us Must(cbdataReferenceValid(toCbdata())); diff -u -r -N squid-4.0.12/src/base/HardFun.h squid-4.0.13/src/base/HardFun.h --- squid-4.0.12/src/base/HardFun.h 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/base/HardFun.h 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,21 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#ifndef _SQUID_SRC_BASE_HARDFUN_H +#define _SQUID_SRC_BASE_HARDFUN_H + +/** + * A functor that calls a hard-coded unary function. + */ +template +struct HardFun { + ReturnType operator()(ArgType arg) { fun(arg); } +}; + +#endif /* _SQUID_SRC_BASE_HARDFUN_H */ + diff -u -r -N squid-4.0.12/src/base/Makefile.am squid-4.0.13/src/base/Makefile.am --- squid-4.0.12/src/base/Makefile.am 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/base/Makefile.am 2016-08-06 00:52:55.000000000 +1200 @@ -25,6 +25,7 @@ CharacterSet.h \ CharacterSet.cc \ EnumIterator.h \ + HardFun.h \ InstanceId.h \ Lock.h \ LookupTable.h \ @@ -38,5 +39,4 @@ Subscription.h \ TextException.cc \ TextException.h \ - TidyPointer.h \ YesNoNone.h diff -u -r -N squid-4.0.12/src/base/Makefile.in squid-4.0.13/src/base/Makefile.in --- squid-4.0.12/src/base/Makefile.in 2016-07-02 01:28:45.000000000 +1200 +++ squid-4.0.13/src/base/Makefile.in 2016-08-06 00:54:59.000000000 +1200 @@ -730,6 +730,7 @@ CharacterSet.h \ CharacterSet.cc \ EnumIterator.h \ + HardFun.h \ InstanceId.h \ Lock.h \ LookupTable.h \ @@ -743,7 +744,6 @@ Subscription.h \ TextException.cc \ TextException.h \ - TidyPointer.h \ YesNoNone.h all: all-am diff -u -r -N squid-4.0.12/src/base/TidyPointer.h squid-4.0.13/src/base/TidyPointer.h --- squid-4.0.12/src/base/TidyPointer.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/base/TidyPointer.h 1970-01-01 12:00:00.000000000 +1200 @@ -1,69 +0,0 @@ -/* - * Copyright (C) 1996-2016 The Squid Software Foundation and contributors - * - * Squid software is distributed under GPLv2+ license and includes - * contributions from numerous individuals and organizations. - * Please see the COPYING and CONTRIBUTORS files for details. - */ - -#ifndef SQUID_BASE_TIDYPOINTER_H -#define SQUID_BASE_TIDYPOINTER_H - -/** - * A pointer that deletes the object it points to when the pointer's owner or - * context is gone. Similar to std::unique_ptr but without confusing assignment - * and with a customizable cleanup method. Prevents memory leaks in - * the presence of exceptions and processing short cuts. -*/ -template class TidyPointer -{ -public: - /// Delete callback. - typedef void DCB (T *t); - TidyPointer(T *t = NULL) - : raw(t) {} -public: - bool operator !() const { return !raw; } - explicit operator bool() const { return raw; } - /// Returns raw and possibly NULL pointer - T *get() const { return raw; } - - /// Reset raw pointer - delete last one and save new one. - void reset(T *t) { - deletePointer(); - raw = t; - } - - /// Forget the raw pointer without freeing it. Become a nil pointer. - T *release() { - T *ret = raw; - raw = NULL; - return ret; - } - /// Deallocate raw pointer. - ~TidyPointer() { - deletePointer(); - } -private: - /// Forbidden copy constructor. - TidyPointer(TidyPointer const &); - /// Forbidden assigment operator. - TidyPointer & operator = (TidyPointer const &); - /// Deallocate raw pointer. Become a nil pointer. - void deletePointer() { - if (raw) { - DeAllocator(raw); - } - raw = NULL; - } - T *raw; ///< pointer to T object or NULL -}; - -/// DeAllocator for pointers that need free(3) from the std C library -template void tidyFree(T *p) -{ - xfree(p); -} - -#endif // SQUID_BASE_TIDYPOINTER_H - diff -u -r -N squid-4.0.12/src/cache_cf.cc squid-4.0.13/src/cache_cf.cc --- squid-4.0.12/src/cache_cf.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/cache_cf.cc 2016-08-06 00:52:55.000000000 +1200 @@ -946,10 +946,6 @@ } } #endif - - if (Config.readAheadGap <= 0) { - fatalf("read_ahead_gap must be greater than 0 bytes"); - } } /** Parse a line containing an obsolete directive. @@ -2208,6 +2204,8 @@ #endif } else if (strncmp(token, "tls-", 4) == 0) { p->secure.parse(token+4); + } else if (strncmp(token, "tls", 3) == 0) { + p->secure.parse(token+3); } else if (strcmp(token, "front-end-https") == 0) { p->front_end_https = 1; } else if (strcmp(token, "front-end-https=on") == 0) { diff -u -r -N squid-4.0.12/src/CacheDigest.cc squid-4.0.13/src/CacheDigest.cc --- squid-4.0.12/src/CacheDigest.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/CacheDigest.cc 2016-08-06 00:52:55.000000000 +1200 @@ -35,11 +35,11 @@ static uint32_t hashed_keys[4]; void -CacheDigest::init(int newCapacity) +CacheDigest::init(uint64_t newCapacity) { const auto newMaskSz = CacheDigest::CalcMaskSize(newCapacity, bits_per_entry); assert(newCapacity > 0 && bits_per_entry > 0); - assert(newMaskSz > 0); + assert(newMaskSz != 0); capacity = newCapacity; mask_size = newMaskSz; mask = static_cast(xcalloc(mask_size,1)); @@ -47,13 +47,13 @@ << mask_size << " bytes"); } -CacheDigest::CacheDigest(int aCapacity, int bpe) : +CacheDigest::CacheDigest(uint64_t aCapacity, uint8_t bpe) : + count(0), + del_count(0), + capacity(0), mask(nullptr), mask_size(0), - capacity(0), - bits_per_entry(bpe), - count(0), - del_count(0) + bits_per_entry(bpe) { assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely on 16 byte keys */ updateCapacity(aCapacity); @@ -83,7 +83,7 @@ } void -CacheDigest::updateCapacity(int newCapacity) +CacheDigest::updateCapacity(uint64_t newCapacity) { safe_free(mask); init(newCapacity); // will re-init mask and mask_size @@ -261,12 +261,12 @@ storeAppendPrintf(e, "%s digest: size: %d bytes\n", label ? label : "", stats.bit_count / 8 ); - storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n", + storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64 " util: %d%%\n", cd->count, cd->capacity, xpercentInt(cd->count, cd->capacity) ); - storeAppendPrintf(e, "\t deletion attempts: %d\n", + storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n", cd->del_count ); storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util: %d%%\n", @@ -280,17 +280,18 @@ ); } -size_t -CacheDigest::CalcMaskSize(int cap, int bpe) +uint32_t +CacheDigest::CalcMaskSize(uint64_t cap, uint8_t bpe) { - // XXX: might 32-bit overflow during multiply - return (size_t) (cap * bpe + 7) / 8; + uint64_t bitCount = (cap * bpe) + 7; + assert(bitCount < INT_MAX); // dont 31-bit overflow later + return static_cast(bitCount / 8); } static void cacheDigestHashKey(const CacheDigest * cd, const cache_key * key) { - const unsigned int bit_count = cd->mask_size * 8; + const uint32_t bit_count = cd->mask_size * 8; unsigned int tmp_keys[4]; /* we must memcpy to ensure alignment */ memcpy(tmp_keys, key, sizeof(tmp_keys)); diff -u -r -N squid-4.0.12/src/CacheDigest.h squid-4.0.13/src/CacheDigest.h --- squid-4.0.12/src/CacheDigest.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/CacheDigest.h 2016-08-06 00:52:55.000000000 +1200 @@ -21,7 +21,7 @@ { MEMPROXY_CLASS(CacheDigest); public: - CacheDigest(int capacity, int bpe); + CacheDigest(uint64_t capacity, uint8_t bpe); ~CacheDigest(); // NP: only used by broken unit-test @@ -32,7 +32,7 @@ void clear(); /// changes mask size to fit newCapacity, resets bits to 0 - void updateCapacity(int newCapacity); + void updateCapacity(uint64_t newCapacity); void add(const cache_key * key); void remove(const cache_key * key); @@ -45,19 +45,19 @@ /// calculate the size of mask required to digest up to /// a specified capacity and bitsize. - static size_t CalcMaskSize(int cap, int bpe); + static uint32_t CalcMaskSize(uint64_t cap, uint8_t bpe); private: - void init(int newCapacity); + void init(uint64_t newCapacity); public: /* public, read-only */ - char *mask; /* bit mask */ - int mask_size; /* mask size in bytes */ - int capacity; /* expected maximum for .count, not a hard limit */ - int bits_per_entry; /* number of bits allocated for each entry from capacity */ - int count; /* number of digested entries */ - int del_count; /* number of deletions performed so far */ + uint64_t count; /* number of digested entries */ + uint64_t del_count; /* number of deletions performed so far */ + uint64_t capacity; /* expected maximum for .count, not a hard limit */ + char *mask; /* bit mask */ + uint32_t mask_size; /* mask size in bytes */ + int8_t bits_per_entry; /* number of bits allocated for each entry from capacity */ }; void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int real_hit, int guess_hit); diff -u -r -N squid-4.0.12/src/CachePeer.cc squid-4.0.13/src/CachePeer.cc --- squid-4.0.12/src/CachePeer.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/CachePeer.cc 2016-08-06 00:52:55.000000000 +1200 @@ -42,6 +42,7 @@ domain(NULL), #if USE_OPENSSL sslContext(NULL), + sslSession(NULL), #endif front_end_https(0), connection_auth(2 /* auto */) @@ -101,6 +102,9 @@ #if USE_OPENSSL if (sslContext) SSL_CTX_free(sslContext); + + if (sslSession) + SSL_SESSION_free(sslSession); #endif } diff -u -r -N squid-4.0.12/src/CachePeer.h squid-4.0.13/src/CachePeer.h --- squid-4.0.12/src/CachePeer.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/CachePeer.h 2016-08-06 00:52:55.000000000 +1200 @@ -184,7 +184,9 @@ /// security settings for peer connection Security::PeerOptions secure; Security::ContextPtr sslContext; - Security::SessionStatePointer sslSession; +#if USE_OPENSSL + SSL_SESSION *sslSession; +#endif int front_end_https; int connection_auth; diff -u -r -N squid-4.0.12/src/cf.data.pre squid-4.0.13/src/cf.data.pre --- squid-4.0.12/src/cf.data.pre 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/cf.data.pre 2016-08-06 00:52:55.000000000 +1200 @@ -3308,7 +3308,7 @@ ==== SSL / HTTPS / TLS OPTIONS ==== - ssl Encrypt connections to this peer with SSL/TLS. + tls Encrypt connections to this peer with TLS. sslcert=/path/to/ssl/certificate A client SSL certificate to use when connecting to @@ -5650,17 +5650,6 @@ DOC_START The amount of data the cache will buffer ahead of what has been sent to the client when retrieving an object from another server. - - This also influences the maximum network read(2)/write(2) sizes in some - circumstances. Reducing the size of this buffer will decrease - per-connection memory usage at the cost of more read(2)/write(2) calls. - Conversely, increasing the size of this buffer will decrease the number of - read(2)/write(2) calls at the cost of memory usage, potentially improving - performance. - - Squid does not slow does the response delivery to the client in order to - fill the buffer. - DOC_END NAME: negative_ttl @@ -6211,13 +6200,29 @@ potentially cachable requests for the same URI before Squid knows whether the response is going to be cachable. - This feature is disabled by default: Enabling collapsed forwarding - needlessly delays forwarding requests that look cachable (when they are - collapsed) but then need to be forwarded individually anyway because - they end up being for uncachable content. However, in some cases, such - as accelleration of highly cachable content with periodic or groupped - expiration times, the gains from collapsing [large volumes of - simultenous refresh requests] outweigh losses from such delays. + When enabled, instead of forwarding each concurrent request for + the same URL, Squid just sends the first of them. The other, so + called "collapsed" requests, wait for the response to the first + request and, if it happens to be cachable, use that response. + Here, "concurrent requests" means "received after the first + request headers were parsed and before the corresponding response + headers were parsed". + + This feature is disabled by default: enabling collapsed + forwarding needlessly delays forwarding requests that look + cachable (when they are collapsed) but then need to be forwarded + individually anyway because they end up being for uncachable + content. However, in some cases, such as acceleration of highly + cachable content with periodic or grouped expiration times, the + gains from collapsing [large volumes of simultaneous refresh + requests] outweigh losses from such delays. + + Squid collapses two kinds of requests: regular client requests + received on one of the listening ports and internal "cache + revalidation" requests which are triggered by those regular + requests hitting a stale cached object. Revalidation collapsing + is currently disabled for Squid instances containing SMP-aware + disk or memory caches and for Vary-controlled cached objects. DOC_END NAME: collapsed_forwarding_shared_entries_limit diff -u -r -N squid-4.0.12/src/client_db.h squid-4.0.13/src/client_db.h --- squid-4.0.12/src/client_db.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/client_db.h 2016-08-06 00:52:55.000000000 +1200 @@ -12,7 +12,6 @@ #define SQUID_CLIENT_DB_H_ #include "anyp/ProtocolType.h" -//#include "enums.h" #include "ip/Address.h" #include "LogTags.h" diff -u -r -N squid-4.0.12/src/clients/FtpClient.cc squid-4.0.13/src/clients/FtpClient.cc --- squid-4.0.12/src/clients/FtpClient.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/clients/FtpClient.cc 2016-08-06 00:52:55.000000000 +1200 @@ -314,6 +314,11 @@ /* We've already read some reply data */ handleControlReply(); } else { + + if (!Comm::IsConnOpen(ctrl.conn)) { + debugs(9, 3, "cannot read without ctrl " << ctrl.conn); + return; + } /* * Cancel the timeout on the Data socket (if any) and * establish one on the control socket. diff -u -r -N squid-4.0.12/src/client_side.cc squid-4.0.13/src/client_side.cc --- squid-4.0.12/src/client_side.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/client_side.cc 2016-08-06 00:52:55.000000000 +1200 @@ -584,6 +584,8 @@ ConnStateData::swanSong() { debugs(33, 2, HERE << clientConnection); + checkLogging(); + flags.readMore = false; DeregisterRunner(this); clientdbEstablished(clientConnection->remote, -1); /* decrement */ @@ -1012,9 +1014,12 @@ http->uri = xstrdup(uri); setLogUri (http, uri); auto *context = new Http::Stream(clientConnection, http); + StoreIOBuffer tempBuffer; + tempBuffer.data = context->reqbuf; + tempBuffer.length = HTTP_REQBUF_SZ; clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, clientReplyStatus, new clientReplyContext(http), clientSocketRecipient, - clientSocketDetach, context, context->getClientStreamBuffer()); + clientSocketDetach, context, tempBuffer); return context; } @@ -1356,11 +1361,15 @@ http->req_sz = hp->messageHeaderSize(); Http::Stream *result = new Http::Stream(csd->clientConnection, http); + StoreIOBuffer tempBuffer; + tempBuffer.data = result->reqbuf; + tempBuffer.length = HTTP_REQBUF_SZ; + ClientStreamData newServer = new clientReplyContext(http); ClientStreamData newClient = result; clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, clientReplyStatus, newServer, clientSocketRecipient, - clientSocketDetach, newClient, result->getClientStreamBuffer()); + clientSocketDetach, newClient, tempBuffer); /* set url */ debugs(33,5, "Prepare absolute URL from " << @@ -2134,10 +2143,6 @@ // On errors, bodyPipe may become nil, but readMore will be cleared while (!inBuf.isEmpty() && !bodyPipe && flags.readMore) { - /* Don't try to parse if the buffer is empty */ - if (inBuf.isEmpty()) - break; - /* Limit the number of concurrent requests */ if (concurrentRequestQueueFilled()) break; @@ -2570,16 +2575,16 @@ #if USE_OPENSSL /** Create SSL connection structure and update fd_table */ -static Security::SessionPtr +static bool httpsCreate(const Comm::ConnectionPointer &conn, Security::ContextPtr sslContext) { - if (auto ssl = Ssl::CreateServer(sslContext, conn->fd, "client https start")) { + if (Ssl::CreateServer(sslContext, conn, "client https start")) { debugs(33, 5, "will negotate SSL on " << conn); - return ssl; + return true; } conn->close(); - return nullptr; + return false; } /** @@ -2650,7 +2655,7 @@ return; } - if (Security::SessionIsResumed(fd_table[fd].ssl)) { + if (SSL_session_reused(ssl)) { debugs(83, 2, "clientNegotiateSSL: Session " << SSL_get_session(ssl) << " reused on FD " << fd << " (" << fd_table[fd].ipaddr << ":" << (int)fd_table[fd].remote_port << ")"); } else { @@ -2725,11 +2730,10 @@ static void httpsEstablish(ConnStateData *connState, Security::ContextPtr sslContext) { - Security::SessionPtr ssl = nullptr; assert(connState); const Comm::ConnectionPointer &details = connState->clientConnection; - if (!sslContext || !(ssl = httpsCreate(details, sslContext))) + if (!sslContext || !httpsCreate(details, sslContext)) return; typedef CommCbMemFunT TimeoutDialer; @@ -4024,3 +4028,36 @@ * connection has gone away */ } +void +ConnStateData::checkLogging() +{ + // if we are parsing request body, its request is responsible for logging + if (bodyPipe) + return; + + // a request currently using this connection is responsible for logging + if (!pipeline.empty() && pipeline.back()->mayUseConnection()) + return; + + /* Either we are waiting for the very first transaction, or + * we are done with the Nth transaction and are waiting for N+1st. + * XXX: We assume that if anything was added to inBuf, then it could + * only be consumed by actions already covered by the above checks. + */ + + // do not log connections that closed after a transaction (it is normal) + // TODO: access_log needs ACLs to match received-no-bytes connections + // XXX: TLS may return here even though we got no transactions yet + // XXX: PROXY protocol may return here even though we got no + // transactions yet + if (receivedFirstByte_ && inBuf.isEmpty()) + return; + + /* Create a temporary ClientHttpRequest object. Its destructor will log. */ + ClientHttpRequest http(this); + http.req_sz = inBuf.length(); + char const *uri = "error:transaction-end-before-headers"; + http.uri = xstrdup(uri); + setLogUri(&http, uri); +} + diff -u -r -N squid-4.0.12/src/client_side.h squid-4.0.13/src/client_side.h --- squid-4.0.12/src/client_side.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/client_side.h 2016-08-06 00:52:55.000000000 +1200 @@ -329,6 +329,7 @@ private: /* ::Server API */ virtual bool connFinishedWithConn(int size); + virtual void checkLogging(); void clientAfterReadingRequests(); bool concurrentRequestQueueFilled() const; diff -u -r -N squid-4.0.12/src/client_side_reply.cc squid-4.0.13/src/client_side_reply.cc --- squid-4.0.12/src/client_side_reply.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/client_side_reply.cc 2016-08-06 00:52:55.000000000 +1200 @@ -89,7 +89,8 @@ reply(NULL), old_entry(NULL), old_sc(NULL), - deleting(false) + deleting(false), + collapsedRevalidation(crNone) { *tempbuf = 0; } @@ -264,7 +265,6 @@ clientReplyContext::processExpired() { const char *url = storeId(); - StoreEntry *entry = NULL; debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'"); assert(http->storeEntry()->lastmod >= 0); /* @@ -287,9 +287,36 @@ #endif /* Prepare to make a new temporary request */ saveState(); - entry = storeCreateEntry(url, - http->log_uri, http->request->flags, http->request->method); - /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */ + + // TODO: support collapsed revalidation for Vary-controlled entries + const bool collapsingAllowed = Config.onoff.collapsed_forwarding && + !Store::Root().smpAware() && + http->request->vary_headers.isEmpty(); + + StoreEntry *entry = nullptr; + if (collapsingAllowed) { + if ((entry = storeGetPublicByRequest(http->request, ksRevalidation))) + entry->lock("clientReplyContext::processExpired#alreadyRevalidating"); + } + + if (entry) { + debugs(88, 5, "collapsed on existing revalidation entry: " << *entry); + collapsedRevalidation = crSlave; + } else { + entry = storeCreateEntry(url, + http->log_uri, http->request->flags, http->request->method); + /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */ + + if (collapsingAllowed) { + debugs(88, 5, "allow other revalidation requests to collapse on " << *entry); + Store::Root().allowCollapsing(entry, http->request->flags, + http->request->method); + collapsedRevalidation = crInitiator; + } else { + collapsedRevalidation = crNone; + } + } + sc = storeClientListAdd(entry, this); #if USE_DELAY_POOLS /* delay_id is already set on original store client */ @@ -309,13 +336,14 @@ assert(http->out.offset == 0); assert(http->request->clientConnectionManager == http->getConn()); - /* - * A refcounted pointer so that FwdState stays around as long as - * this clientReplyContext does - */ - Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL; - FwdState::Start(conn, http->storeEntry(), http->request, http->al); - + if (collapsedRevalidation != crSlave) { + /* + * A refcounted pointer so that FwdState stays around as long as + * this clientReplyContext does + */ + Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL; + FwdState::Start(conn, http->storeEntry(), http->request, http->al); + } /* Register with storage manager to receive updates when data comes in. */ if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) @@ -427,6 +455,10 @@ // forward response from origin http->logType = LOG_TCP_REFRESH_MODIFIED; debugs(88, 3, "handleIMSReply: origin replied " << status << ", replacing existing entry and forwarding to client"); + + if (collapsedRevalidation) + http->storeEntry()->clearPublicKeyScope(); + sendClientUpstreamResponse(); } @@ -2121,21 +2153,28 @@ StoreEntry *entry = http->storeEntry(); - ConnStateData * conn = http->getConn(); + if (ConnStateData * conn = http->getConn()) { + if (!conn->isOpen()) { + debugs(33,3, "not sending more data to closing connection " << conn->clientConnection); + return; + } + if (conn->pinning.zeroReply) { + debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection); + return; + } + + if (reqofs==0 && !http->logType.isTcpHit()) { + if (Ip::Qos::TheConfig.isHitTosActive()) { + Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code); + } + if (Ip::Qos::TheConfig.isHitNfmarkActive()) { + Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code); + } + } - // too late, our conn is closing - // TODO: should we also quit? - if (conn == NULL) { - debugs(33,3, "not sending more data to a closed connection" ); - return; - } - if (!conn->isOpen()) { - debugs(33,3, "not sending more data to closing connection " << conn->clientConnection); - return; - } - if (conn->pinning.zeroReply) { - debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection); - return; + debugs(88, 5, conn->clientConnection << + " '" << entry->url() << "'" << + " out.offset=" << http->out.offset); } char *buf = next()->readBuffer.data; @@ -2146,15 +2185,6 @@ memcpy(buf, result.data, result.length); } - if (reqofs==0 && !http->logType.isTcpHit() && Comm::IsConnOpen(conn->clientConnection)) { - if (Ip::Qos::TheConfig.isHitTosActive()) { - Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code); - } - if (Ip::Qos::TheConfig.isHitNfmarkActive()) { - Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code); - } - } - /* We've got the final data to start pushing... */ flags.storelogiccomplete = 1; @@ -2173,10 +2203,6 @@ debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " << reqofs << " bytes (" << result.length << " new bytes)"); - debugs(88, 5, "clientReplyContext::sendMoreData:" - << conn->clientConnection << - " '" << entry->url() << "'" << - " out.offset=" << http->out.offset); /* update size of the request */ reqsize = reqofs; diff -u -r -N squid-4.0.12/src/client_side_reply.h squid-4.0.13/src/client_side_reply.h --- squid-4.0.12/src/client_side_reply.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/client_side_reply.h 2016-08-06 00:52:55.000000000 +1200 @@ -133,6 +133,14 @@ StoreEntry *old_entry; store_client *old_sc; /* ... for entry to be validated */ bool deleting; + + typedef enum { + crNone = 0, ///< collapsed revalidation is not allowed for this context + crInitiator, ///< we initiated collapsed revalidation request + crSlave ///< we collapsed on the existing revalidation request + } CollapsedRevalidation; + + CollapsedRevalidation collapsedRevalidation; }; #endif /* SQUID_CLIENTSIDEREPLY_H */ diff -u -r -N squid-4.0.12/src/client_side_request.cc squid-4.0.13/src/client_side_request.cc --- squid-4.0.12/src/client_side_request.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/client_side_request.cc 2016-08-06 00:52:55.000000000 +1200 @@ -177,7 +177,7 @@ #if USE_OPENSSL if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) { if (auto ssl = fd_table[aConn->clientConnection->fd].ssl.get()) - al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl)); + al->cache.sslClientCert.resetWithoutLocking(SSL_get_peer_certificate(ssl)); } #endif } @@ -885,8 +885,7 @@ if (answer == ACCESS_ALLOWED) redirectStart(http, clientRedirectDoneWrapper, context); else { - Helper::Reply nilReply; - nilReply.result = Helper::Error; + Helper::Reply const nilReply(Helper::Error); context->clientRedirectDone(nilReply); } } @@ -918,8 +917,7 @@ storeIdStart(http, clientStoreIdDoneWrapper, context); else { debugs(85, 3, "access denied expected ERR reply handling: " << answer); - Helper::Reply nilReply; - nilReply.result = Helper::Error; + Helper::Reply const nilReply(Helper::Error); context->clientStoreIdDone(nilReply); } } @@ -1409,6 +1407,11 @@ bool ClientRequestContext::sslBumpAccessCheck() { + if (!http->getConn()) { + http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log - + return false; + } + // If SSL connection tunneling or bumping decision has been made, obey it. const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode; if (bumpMode != Ssl::bumpEnd) { diff -u -r -N squid-4.0.12/src/comm/Read.cc squid-4.0.13/src/comm/Read.cc --- squid-4.0.12/src/comm/Read.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/comm/Read.cc 2016-08-06 00:52:55.000000000 +1200 @@ -20,7 +20,6 @@ #include "fde.h" #include "sbuf/SBuf.h" #include "StatCounters.h" -//#include "tools.h" // Does comm check this fd for read readiness? // Note that when comm is not monitoring, there can be a pending callback diff -u -r -N squid-4.0.12/src/comm/TcpAcceptor.cc squid-4.0.13/src/comm/TcpAcceptor.cc --- squid-4.0.12/src/comm/TcpAcceptor.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/comm/TcpAcceptor.cc 2016-08-06 00:52:55.000000000 +1200 @@ -9,6 +9,7 @@ /* DEBUG: section 05 Listener Socket Handler */ #include "squid.h" +#include "acl/FilledChecklist.h" #include "anyp/PortCfg.h" #include "base/TextException.h" #include "client_db.h" @@ -24,6 +25,7 @@ #include "globals.h" #include "ip/Intercept.h" #include "ip/QosConfig.h" +#include "log/access_log.h" #include "MasterXaction.h" #include "profiler/Profiler.h" #include "SquidConfig.h" @@ -256,6 +258,18 @@ return false; } +static void +logAcceptError(const Comm::ConnectionPointer &conn) +{ + AccessLogEntry::Pointer al = new AccessLogEntry; + al->tcpClient = conn; + al->url = "error:accept-client-connection"; + ACLFilledChecklist ch(nullptr, nullptr, nullptr); + ch.src_addr = conn->remote; + ch.my_addr = conn->local; + accessLogLog(al, &ch); +} + void Comm::TcpAcceptor::acceptOne() { @@ -281,6 +295,8 @@ // A non-recoverable error; notify the caller */ debugs(5, 5, HERE << "non-recoverable error:" << status() << " handler Subscription: " << theCallSub); + if (intendedForUserConnections()) + logAcceptError(newConnDetails); notify(flag, newConnDetails); mustStop("Listener socket closed"); return; diff -u -r -N squid-4.0.12/src/comm/TcpAcceptor.h squid-4.0.13/src/comm/TcpAcceptor.h --- squid-4.0.12/src/comm/TcpAcceptor.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/comm/TcpAcceptor.h 2016-08-06 00:52:55.000000000 +1200 @@ -104,6 +104,8 @@ Comm::Flag oldAccept(Comm::ConnectionPointer &details); void setListen(); void handleClosure(const CommCloseCbParams &io); + /// whether we are listening on one of the squid.conf *ports + bool intendedForUserConnections() const { return bool(listenPort_); } }; } // namespace Comm diff -u -r -N squid-4.0.12/src/comm.cc squid-4.0.13/src/comm.cc --- squid-4.0.12/src/comm.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/comm.cc 2016-08-06 00:52:55.000000000 +1200 @@ -838,7 +838,7 @@ comm_close_complete(const FdeCbParams ¶ms) { fde *F = &fd_table[params.fd]; - F->ssl.reset(nullptr); + F->ssl.resetWithoutLocking(nullptr); #if USE_OPENSSL if (F->dynamicSslContext) { diff -u -r -N squid-4.0.12/src/Downloader.cc squid-4.0.13/src/Downloader.cc --- squid-4.0.12/src/Downloader.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/Downloader.cc 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,267 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#include "squid.h" +#include "client_side.h" +#include "client_side_reply.h" +#include "client_side_request.h" +#include "ClientRequestContext.h" +#include "Downloader.h" +#include "fatal.h" +#include "http/one/RequestParser.h" +#include "http/Stream.h" + +CBDATA_CLASS_INIT(Downloader); + +/// Used to hold and pass the required info and buffers to the +/// clientStream callbacks +class DownloaderContext: public RefCountable +{ + MEMPROXY_CLASS(DownloaderContext); + +public: + typedef RefCount Pointer; + + DownloaderContext(Downloader *dl, ClientHttpRequest *h); + ~DownloaderContext(); + void finished(); + + CbcPointer downloader; + ClientHttpRequest *http; + char requestBuffer[HTTP_REQBUF_SZ]; +}; + +DownloaderContext::DownloaderContext(Downloader *dl, ClientHttpRequest *h): + downloader(dl), + http(h) +{ + debugs(33, 6, "DownloaderContext constructed, this=" << (void*)this); +} + +DownloaderContext::~DownloaderContext() +{ + debugs(33, 6, "DownloaderContext destructed, this=" << (void*)this); + if (http) + finished(); +} + +void +DownloaderContext::finished() +{ + delete http; + http = nullptr; +} + +void +Downloader::CbDialer::print(std::ostream &os) const +{ + os << " Http Status:" << status << Raw("body data", object.rawContent(), 64).hex(); +} + +Downloader::Downloader(SBuf &url, AsyncCall::Pointer &aCallback, unsigned int level): + AsyncJob("Downloader"), + url_(url), + callback_(aCallback), + level_(level) +{ +} + +Downloader::~Downloader() +{ +} + +bool +Downloader::doneAll() const +{ + return (!callback_ || callback_->canceled()) && AsyncJob::doneAll(); +} + +static void +downloaderRecipient(clientStreamNode * node, ClientHttpRequest * http, + HttpReply * rep, StoreIOBuffer receivedData) +{ + debugs(33, 6, MYNAME); + /* Test preconditions */ + assert(node); + + /* TODO: handle this rather than asserting + * - it should only ever happen if we cause an abort and + * the callback chain loops back to here, so we can simply return. + * However, that itself shouldn't happen, so it stays as an assert for now. + */ + assert(cbdataReferenceValid(node)); + assert(!node->node.next); + DownloaderContext::Pointer context = dynamic_cast(node->data.getRaw()); + assert(context); + + if (context->downloader.valid()) + context->downloader->handleReply(node, http, rep, receivedData); +} + +static void +downloaderDetach(clientStreamNode * node, ClientHttpRequest * http) +{ + debugs(33, 5, MYNAME); + clientStreamDetach(node, http); +} + +/// Initializes and starts the HTTP GET request to the remote server +bool +Downloader::buildRequest() +{ + const HttpRequestMethod method = Http::METHOD_GET; + + char *uri = xstrdup(url_.c_str()); + HttpRequest *const request = HttpRequest::CreateFromUrl(uri, method); + if (!request) { + debugs(33, 5, "Invalid URI: " << url_); + xfree(uri); + return false; //earlyError(...) + } + request->http_ver = Http::ProtocolVersion(); + request->header.putStr(Http::HdrType::HOST, request->url.host()); + request->header.putTime(Http::HdrType::DATE, squid_curtime); + request->flags.internalClient = true; + request->client_addr.setNoAddr(); +#if FOLLOW_X_FORWARDED_FOR + request->indirect_client_addr.setNoAddr(); +#endif /* FOLLOW_X_FORWARDED_FOR */ + request->my_addr.setNoAddr(); /* undefined for internal requests */ + request->my_addr.port(0); + request->downloader = this; + + debugs(11, 2, "HTTP Client Downloader " << this << "/" << id); + debugs(11, 2, "HTTP Client REQUEST:\n---------\n" << + request->method << " " << url_ << " " << request->http_ver << "\n" << + "\n----------"); + + ClientHttpRequest *const http = new ClientHttpRequest(nullptr); + http->request = request; + HTTPMSGLOCK(http->request); + http->req_sz = 0; + http->uri = uri; + setLogUri (http, urlCanonicalClean(request)); + + context_ = new DownloaderContext(this, http); + StoreIOBuffer tempBuffer; + tempBuffer.data = context_->requestBuffer; + tempBuffer.length = HTTP_REQBUF_SZ; + + ClientStreamData newServer = new clientReplyContext(http); + ClientStreamData newClient = context_.getRaw(); + clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, + clientReplyStatus, newServer, downloaderRecipient, + downloaderDetach, newClient, tempBuffer); + + // Build a ClientRequestContext to start doCallouts + http->calloutContext = new ClientRequestContext(http); + http->doCallouts(); + return true; +} + +void +Downloader::start() +{ + if (!buildRequest()) + callBack(Http::scInternalServerError); +} + +void +Downloader::handleReply(clientStreamNode * node, ClientHttpRequest *http, HttpReply *reply, StoreIOBuffer receivedData) +{ + DownloaderContext::Pointer callerContext = dynamic_cast(node->data.getRaw()); + // TODO: remove the following check: + assert(callerContext == context_); + + debugs(33, 4, "Received " << receivedData.length << + " object data, offset: " << receivedData.offset << + " error flag:" << receivedData.flags.error); + + const bool failed = receivedData.flags.error; + if (failed) { + callBack(Http::scInternalServerError); + return; + } + + const int64_t existingContent = reply ? reply->content_length : 0; + const size_t maxSize = MaxObjectSize > SBuf::maxSize ? SBuf::maxSize : MaxObjectSize; + const bool tooLarge = (existingContent > -1 && existingContent > static_cast(maxSize)) || + (maxSize < object_.length()) || + ((maxSize - object_.length()) < receivedData.length); + + if (tooLarge) { + callBack(Http::scInternalServerError); + return; + } + + object_.append(receivedData.data, receivedData.length); + http->out.size += receivedData.length; + http->out.offset += receivedData.length; + + switch (clientStreamStatus(node, http)) { + case STREAM_NONE: { + debugs(33, 3, "Get more data"); + StoreIOBuffer tempBuffer; + tempBuffer.offset = http->out.offset; + tempBuffer.data = context_->requestBuffer; + tempBuffer.length = HTTP_REQBUF_SZ; + clientStreamRead(node, http, tempBuffer); + } + break; + case STREAM_COMPLETE: + debugs(33, 3, "Object data transfer successfully complete"); + callBack(Http::scOkay); + break; + case STREAM_UNPLANNED_COMPLETE: + debugs(33, 3, "Object data transfer failed: STREAM_UNPLANNED_COMPLETE"); + callBack(Http::scInternalServerError); + break; + case STREAM_FAILED: + debugs(33, 3, "Object data transfer failed: STREAM_FAILED"); + callBack(Http::scInternalServerError); + break; + default: + fatal("unreachable code"); + } +} + +void +Downloader::downloadFinished() +{ + debugs(33, 7, this); + // We cannot delay http destruction until refcounting deletes + // DownloaderContext. The http object destruction will cause + // clientStream cleanup and will release the refcount to context_ + // object hold by clientStream structures. + context_->finished(); + context_ = nullptr; + Must(done()); +} + +/// Schedules for execution the "callback" with parameters the status +/// and object. +void +Downloader::callBack(Http::StatusCode const statusCode) +{ + CbDialer *dialer = dynamic_cast(callback_->getDialer()); + Must(dialer); + dialer->status = statusCode; + if (statusCode == Http::scOkay) + dialer->object = object_; + ScheduleCallHere(callback_); + callback_ = nullptr; + + // Calling deleteThis method here to finish Downloader + // may result to squid crash. + // This method called by handleReply method which maybe called + // by ClientHttpRequest::doCallouts. The doCallouts after this object + // deleted, may operate on non valid objects. + // Schedule an async call here just to force squid to delete this object. + CallJobHere(33, 7, CbcPointer(this), Downloader, downloadFinished); +} + diff -u -r -N squid-4.0.12/src/Downloader.h squid-4.0.13/src/Downloader.h --- squid-4.0.12/src/Downloader.h 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/Downloader.h 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,83 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#ifndef SQUID_DOWNLOADER_H +#define SQUID_DOWNLOADER_H + +#include "base/AsyncJob.h" +#include "defines.h" +#include "http/forward.h" +#include "http/StatusCode.h" +#include "sbuf/SBuf.h" + +class ClientHttpRequest; +class StoreIOBuffer; +class clientStreamNode; +class DownloaderContext; +typedef RefCount DownloaderContextPointer; + +/// The Downloader class fetches SBuf-storable things for other Squid +/// components/transactions using internal requests. For example, it is used +/// to fetch missing intermediate certificates when validating origin server +/// certificate chains. +class Downloader: virtual public AsyncJob +{ + CBDATA_CLASS(Downloader); +public: + + /// Callback data to use with Downloader callbacks. + class CbDialer: public CallDialer { + public: + CbDialer(): status(Http::scNone) {} + virtual ~CbDialer() {} + + /* CallDialer API */ + virtual bool canDial(AsyncCall &call) = 0; + virtual void dial(AsyncCall &call) = 0; + virtual void print(std::ostream &os) const; + + SBuf object; + Http::StatusCode status; + }; + + Downloader(SBuf &url, AsyncCall::Pointer &aCallback, unsigned int level = 0); + virtual ~Downloader(); + + /// delays destruction to protect doCallouts() + void downloadFinished(); + + /// The nested level of Downloader object (downloads inside downloads). + unsigned int nestedLevel() const {return level_;} + + void handleReply(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer); + +protected: + + /* AsyncJob API */ + virtual bool doneAll() const; + virtual void start(); + +private: + + bool buildRequest(); + void callBack(Http::StatusCode const status); + + /// The maximum allowed object size. + static const size_t MaxObjectSize = 1*1024*1024; + + SBuf url_; ///< the url to download + AsyncCall::Pointer callback_; ///< callback to call when download finishes + SBuf object_; ///< the object body data + const unsigned int level_; ///< holds the nested downloads level + + /// Pointer to an object that stores the clientStream required info + DownloaderContextPointer context_; +}; + +#endif + diff -u -r -N squid-4.0.12/src/fde.h squid-4.0.13/src/fde.h --- squid-4.0.12/src/fde.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/fde.h 2016-08-06 00:52:55.000000000 +1200 @@ -167,7 +167,7 @@ halfClosedReader = NULL; read_method = NULL; write_method = NULL; - ssl.reset(nullptr); + ssl.resetWithoutLocking(nullptr); dynamicSslContext = NULL; #if _SQUID_WINDOWS_ win32.handle = (long)NULL; diff -u -r -N squid-4.0.12/src/format/Config.h squid-4.0.13/src/format/Config.h --- squid-4.0.12/src/format/Config.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/format/Config.h 2016-08-06 00:52:55.000000000 +1200 @@ -10,8 +10,8 @@ #define SQUID_SRC_FORMAT_CONFIG_H #include "format/Format.h" -//#include "format/TokenTableEntry.h" #include "SquidString.h" + #include class StoreEntry; diff -u -r -N squid-4.0.12/src/format/Token.h squid-4.0.13/src/format/Token.h --- squid-4.0.12/src/format/Token.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/format/Token.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,7 +9,6 @@ #ifndef _SQUID_FORMAT_TOKEN_H #define _SQUID_FORMAT_TOKEN_H -//#include "format/TokenTableEntry.h" #include "format/ByteCode.h" /* diff -u -r -N squid-4.0.12/src/fs/rock/RockSwapDir.h squid-4.0.13/src/fs/rock/RockSwapDir.h --- squid-4.0.12/src/fs/rock/RockSwapDir.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/fs/rock/RockSwapDir.h 2016-08-06 00:52:55.000000000 +1200 @@ -47,6 +47,7 @@ virtual void swappedOut(const StoreEntry &e); virtual void create(); virtual void parse(int index, char *path); + virtual bool smpAware() const { return true; } // temporary path to the shared memory map of first slots of cached entries SBuf inodeMapPath() const; diff -u -r -N squid-4.0.12/src/fs/ufs/UFSStoreState.cc squid-4.0.13/src/fs/ufs/UFSStoreState.cc --- squid-4.0.12/src/fs/ufs/UFSStoreState.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/fs/ufs/UFSStoreState.cc 2016-08-06 00:52:55.000000000 +1200 @@ -15,7 +15,6 @@ #include "DiskIO/WriteRequest.h" #include "Generic.h" #include "SquidConfig.h" -#include "SquidList.h" #include "Store.h" #include "store/Disk.h" #include "UFSStoreState.h" @@ -132,8 +131,9 @@ assert (aCallback); if (!theFile->canRead()) { - debugs(79, 3, "UFSStoreState::read_: queueing read because theFile can't read"); - queueRead (buf, size, aOffset, aCallback, aCallbackData); + debugs(79, 3, "queueing read because theFile can't read"); + assert(opening); + pending_reads.emplace(buf, size, aOffset, aCallback, aCallbackData); return; } @@ -177,7 +177,8 @@ return false; } - queueWrite(buf, size, aOffset, free_func); + debugs(79, 3, (void*)this << " queueing write of size " << size); + pending_writes.emplace(buf, size, aOffset, free_func); drainWriteQueue(); return true; } @@ -191,28 +192,20 @@ void Fs::Ufs::UFSStoreState::doWrite() { - debugs(79, 3, HERE << this << " UFSStoreState::doWrite"); + debugs(79, 3, (void*)this); assert(theFile->canWrite()); - _queued_write *q = (_queued_write *)linklistShift(&pending_writes); - - if (q == NULL) { - debugs(79, 3, HERE << this << " UFSStoreState::doWrite queue is empty"); + if (pending_writes.empty()) { + debugs(79, 3, (void*)this << " write queue is empty"); return; } + auto &q = pending_writes.front(); + if (theFile->error()) { - debugs(79, DBG_IMPORTANT,HERE << "avoid write on theFile with error"); - debugs(79,3,HERE << "calling free_func for " << (void*) q->buf); - /* - * DPW 2006-05-24 - * Note "free_func" is memNodeWriteComplete(), which doesn't - * really free the memory. Instead it clears the node's - * write_pending flag. - */ - q->free_func((void*)q->buf); - delete q; + debugs(79, DBG_IMPORTANT, MYNAME << " avoid write on theFile with error"); + pending_writes.pop(); return; } @@ -226,10 +219,11 @@ * coming in. For now let's just not use the writing flag at * all. */ - debugs(79, 3, HERE << this << " calling theFile->write(" << q->size << ")"); + debugs(79, 3, (void*)this << " calling theFile->write(" << q.size << ")"); - theFile->write(new WriteRequest(q->buf, q->offset, q->size, q->free_func)); - delete q; + theFile->write(new WriteRequest(q.buf, q.offset, q.size, q.free_func)); + q.buf = nullptr; // prevent buf deletion on pop, its used by the above object + pending_writes.pop(); } void @@ -338,8 +332,6 @@ closing(false), reading(false), writing(false), - pending_reads(NULL), - pending_writes(NULL), read_buf(NULL) { // StoreIOState inherited members @@ -354,71 +346,44 @@ Fs::Ufs::UFSStoreState::~UFSStoreState() { - assert(pending_reads == NULL); - assert(pending_writes == NULL); + assert(pending_reads.empty()); + assert(pending_writes.empty()); } void Fs::Ufs::UFSStoreState::freePending() { - _queued_read *qr; - - while ((qr = (_queued_read *)linklistShift(&pending_reads))) { - cbdataReferenceDone(qr->callback_data); - delete qr; - } - - debugs(79,3,HERE << "UFSStoreState::freePending: freed pending reads"); - - _queued_write *qw; - - while ((qw = (_queued_write *)linklistShift(&pending_writes))) { - if (qw->free_func) - qw->free_func(const_cast(qw->buf)); - delete qw; - } - - debugs(79,3,HERE << "UFSStoreState::freePending: freed pending writes"); + while (!pending_reads.empty()) + pending_reads.pop(); + debugs(79, 3, "freed pending reads"); + + while (!pending_writes.empty()) + pending_writes.pop(); + debugs(79, 3, "freed pending writes"); } bool Fs::Ufs::UFSStoreState::kickReadQueue() { - _queued_read *q = (_queued_read *)linklistShift(&pending_reads); - - if (NULL == q) + if (pending_reads.empty()) return false; - debugs(79, 3, "UFSStoreState::kickReadQueue: reading queued request of " << q->size << " bytes"); + auto &q = pending_reads.front(); - void *cbdata; + debugs(79, 3, "reading queued request of " << q.size << " bytes"); - if (cbdataReferenceValidDone(q->callback_data, &cbdata)) { - read_(q->buf, q->size, q->offset, q->callback, cbdata); + bool result = true; + void *cbdata; + if (cbdataReferenceValidDone(q.callback_data, &cbdata)) { + read_(q.buf, q.size, q.offset, q.callback, cbdata); } else { - debugs(79, 2, "UFSStoreState::kickReadQueue: this: " << this << " cbdataReferenceValidDone returned false." << " closing: " << closing << " flags.try_closing: " << flags.try_closing); - delete q; - return false; + debugs(79, 2, "this=" << (void*)this << " cbdataReferenceValidDone returned false." << + " closing: " << closing << " flags.try_closing: " << flags.try_closing); + result = false; } - delete q; - - return true; -} - -void -Fs::Ufs::UFSStoreState::queueRead(char *buf, size_t size, off_t aOffset, STRCB *callback_, void *callback_data_) -{ - debugs(79, 3, "UFSStoreState::queueRead: queueing read"); - assert(opening); - assert (pending_reads == NULL); - _queued_read *q = new _queued_read; - q->buf = buf; - q->size = size; - q->offset = aOffset; - q->callback = callback_; - q->callback_data = cbdataReference(callback_data_); - linklistPush(&pending_reads, q); + pending_reads.pop(); // erase the front object + return result; } /* @@ -445,9 +410,8 @@ flags.write_draining = true; - while (pending_writes != NULL) { + while (!pending_writes.empty()) doWrite(); - } flags.write_draining = false; @@ -482,17 +446,3 @@ theFile->close(); } -void -Fs::Ufs::UFSStoreState::queueWrite(char const *buf, size_t size, off_t aOffset, FREE * free_func) -{ - debugs(79, 3, HERE << this << " UFSStoreState::queueWrite: queueing write of size " << size); - - _queued_write *q; - q = new _queued_write; - q->buf = buf; - q->size = size; - q->offset = aOffset; - q->free_func = free_func; - linklistPush(&pending_writes, q); -} - diff -u -r -N squid-4.0.12/src/fs/ufs/UFSStoreState.h squid-4.0.13/src/fs/ufs/UFSStoreState.h --- squid-4.0.12/src/fs/ufs/UFSStoreState.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/fs/ufs/UFSStoreState.h 2016-08-06 00:52:55.000000000 +1200 @@ -10,9 +10,10 @@ #define SQUID_FS_UFS_UFSSTORESTATE_H #include "DiskIO/IORequestor.h" -#include "SquidList.h" #include "StoreIOState.h" +#include + namespace Fs { namespace Ufs @@ -48,13 +49,16 @@ { MEMPROXY_CLASS(UFSStoreState::_queued_read); public: - _queued_read() : - buf(nullptr), - size(0), - offset(0), - callback(nullptr), - callback_data(nullptr) + _queued_read(char *b, size_t s, off_t o, STRCB *cb, void *data) : + buf(b), + size(s), + offset(o), + callback(cb), + callback_data(cbdataReference(data)) {} + ~_queued_read() { + cbdataReferenceDone(callback_data); + } char *buf; size_t size; @@ -62,23 +66,35 @@ STRCB *callback; void *callback_data; }; + std::queue pending_reads; class _queued_write { MEMPROXY_CLASS(UFSStoreState::_queued_write); public: - _queued_write() : - buf(nullptr), - size(0), - offset(0), - free_func(nullptr) + _queued_write(const char *b, size_t s, off_t o, FREE *f) : + buf(b), + size(s), + offset(o), + free_func(f) {} + ~_queued_write() { + /* + * DPW 2006-05-24 + * Note "free_func" is memNodeWriteComplete(), which doesn't + * really free the memory. Instead it clears the node's + * write_pending flag. + */ + if (free_func && buf) + free_func(const_cast(buf)); + } char const *buf; size_t size; off_t offset; FREE *free_func; }; + std::queue pending_writes; /** \todo These should be in the IO strategy */ @@ -98,10 +114,7 @@ */ bool try_closing; } flags; - link_list *pending_reads; - link_list *pending_writes; - void queueRead(char *, size_t, off_t, STRCB *, void *); - void queueWrite(char const *, size_t, off_t, FREE *); + bool kickReadQueue(); void drainWriteQueue(); void tryClosing(); diff -u -r -N squid-4.0.12/src/fs/ufs/UFSSwapDir.h squid-4.0.13/src/fs/ufs/UFSSwapDir.h --- squid-4.0.12/src/fs/ufs/UFSSwapDir.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/fs/ufs/UFSSwapDir.h 2016-08-06 00:52:55.000000000 +1200 @@ -71,6 +71,7 @@ virtual uint64_t currentSize() const override { return cur_size; } virtual uint64_t currentCount() const override { return n_disk_objects; } virtual ConfigOption *getOptionTree() const override; + virtual bool smpAware() const override { return false; } void unlinkFile(sfileno f); // move down when unlink is a virtual method diff -u -r -N squid-4.0.12/src/FwdState.cc squid-4.0.13/src/FwdState.cc --- squid-4.0.12/src/FwdState.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/FwdState.cc 2016-08-06 00:52:55.000000000 +1200 @@ -45,9 +45,9 @@ #include "pconn.h" #include "PeerPoolMgr.h" #include "PeerSelectState.h" +#include "security/BlindPeerConnector.h" #include "SquidConfig.h" #include "SquidTime.h" -#include "ssl/BlindPeerConnector.h" #include "ssl/PeekingPeerConnector.h" #include "Store.h" #include "StoreClient.h" @@ -78,8 +78,7 @@ static PconnPool *fwdPconnPool = new PconnPool("server-peers", NULL); CBDATA_CLASS_INIT(FwdState); -#if USE_OPENSSL -class FwdStatePeerAnswerDialer: public CallDialer, public Ssl::PeerConnector::CbDialer +class FwdStatePeerAnswerDialer: public CallDialer, public Security::PeerConnector::CbDialer { public: typedef void (FwdState::*Method)(Security::EncryptorAnswer &); @@ -94,7 +93,7 @@ os << '(' << fwd_.get() << ", " << answer_ << ')'; } - /* Ssl::PeerConnector::CbDialer API */ + /* Security::PeerConnector::CbDialer API */ virtual Security::EncryptorAnswer &answer() { return answer_; } private: @@ -102,7 +101,6 @@ CbcPointer fwd_; Security::EncryptorAnswer answer_; }; -#endif void FwdState::abort(void* d) @@ -684,7 +682,6 @@ closeHandler = comm_add_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this); -#if USE_OPENSSL if (!request->flags.pinned) { const CachePeer *p = serverConnection()->getPeer(); const bool peerWantsTls = p && p->secure.encryptTransport; @@ -700,16 +697,17 @@ FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this)); // Use positive timeout when less than one second is left. const time_t sslNegotiationTimeout = max(static_cast(1), timeLeft()); - Ssl::PeerConnector *connector = NULL; + Security::PeerConnector *connector = nullptr; +#if USE_OPENSSL if (request->flags.sslPeek) connector = new Ssl::PeekingPeerConnector(requestPointer, serverConnection(), clientConn, callback, al, sslNegotiationTimeout); else - connector = new Ssl::BlindPeerConnector(requestPointer, serverConnection(), callback, al, sslNegotiationTimeout); +#endif + connector = new Security::BlindPeerConnector(requestPointer, serverConnection(), callback, al, sslNegotiationTimeout); AsyncJob::Start(connector); // will call our callback return; } } -#endif // if not encrypting just run the post-connect actions Security::EncryptorAnswer nil; @@ -997,12 +995,10 @@ request->flags.auth_no_keytab = 0; switch (request->url.getScheme()) { -#if USE_OPENSSL case AnyP::PROTO_HTTPS: httpStart(this); break; -#endif case AnyP::PROTO_HTTP: httpStart(this); diff -u -r -N squid-4.0.12/src/helper/Reply.cc squid-4.0.13/src/helper/Reply.cc --- squid-4.0.12/src/helper/Reply.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/helper/Reply.cc 2016-08-06 00:52:55.000000000 +1200 @@ -16,29 +16,40 @@ #include "rfc1738.h" #include "SquidString.h" -Helper::Reply::Reply(char *buf, size_t len) : +Helper::Reply::Reply() : result(Helper::Unknown), whichServer(NULL) { - parse(buf,len); +} + +bool +Helper::Reply::accumulate(const char *buf, size_t len) +{ + if (other_.isNull()) + other_.init(4*1024, 1*1024*1024); + + if (other_.potentialSpaceSize() < static_cast(len)) + return false; // no space left + + other_.append(buf, len); + return true; } void -Helper::Reply::parse(char *buf, size_t len) +Helper::Reply::finalize() { debugs(84, 3, "Parsing helper buffer"); // check we have something to parse - if (!buf || len < 1) { + if (!other_.hasContent()) { // empty line response was the old URL-rewriter interface ERR response. result = Helper::Error; // for now ensure that legacy handlers are not presented with NULL strings. - debugs(84, 3, "Reply length is smaller than 1 or none at all "); - other_.init(1,1); - other_.terminate(); + debugs(84, 3, "Zero length reply"); return; } - char *p = buf; + char *p = other_.content(); + size_t len = other_.contentSize(); bool sawNA = false; // optimization: do not consider parsing result code if the response is short. @@ -67,10 +78,8 @@ // followed by an auth token char *w1 = strwordtok(NULL, &p); if (w1 != NULL) { - MemBuf authToken; - authToken.init(); - authToken.append(w1, strlen(w1)); - notes.add("token",authToken.content()); + const char *authToken = w1; + notes.add("token",authToken); } else { // token field is mandatory on this response code result = Helper::BrokenHelper; @@ -88,22 +97,16 @@ char *w2 = strwordtok(NULL, &p); if (w2 != NULL) { // Negotiate "token user" - MemBuf authToken; - authToken.init(); - authToken.append(w1, strlen(w1)); - notes.add("token",authToken.content()); - - MemBuf user; - user.init(); - user.append(w2,strlen(w2)); - notes.add("user",user.content()); + const char *authToken = w1; + notes.add("token",authToken); + + const char *user = w2; + notes.add("user",user); } else if (w1 != NULL) { // NTLM "user" - MemBuf user; - user.init(); - user.append(w1,strlen(w1)); - notes.add("user",user.content()); + const char *user = w1; + notes.add("user",user); } } else if (!strncmp(p,"NA ",3)) { // NTLM fail-closed ERR response @@ -115,21 +118,17 @@ for (; xisspace(*p); ++p); // skip whitespace } - const mb_size_t blobSize = (buf+len-p); - other_.init(blobSize+1, blobSize+1); - other_.append(p, blobSize); // remainders of the line. - - // NULL-terminate so the helper callback handlers do not buffer-overrun - other_.terminate(); + other_.consume(p - other_.content()); + other_.consumeWhitespacePrefix(); // Hack for backward-compatibility: Do not parse for kv-pairs on NA response if (!sawNA) parseResponseKeys(); // Hack for backward-compatibility: BH and NA used to be a text message... - if (other().hasContent() && (sawNA || result == Helper::BrokenHelper)) { - notes.add("message",other().content()); - modifiableOther().clean(); + if (other_.hasContent() && (sawNA || result == Helper::BrokenHelper)) { + notes.add("message", other_.content()); + other_.clean(); } } @@ -157,8 +156,9 @@ Helper::Reply::parseResponseKeys() { // parse a "key=value" pair off the 'other()' buffer. - while (other().hasContent()) { - char *p = modifiableOther().content(); + while (other_.hasContent()) { + char *p = other_.content(); + const char *key = p; while (*p && isKeyNameChar(*p)) ++p; if (*p != '=') return; // done. Not a key. @@ -171,8 +171,6 @@ *p = '\0'; ++p; - const char *key = other().content(); - // the value may be a quoted string or a token const bool urlDecode = (*p != '"'); // check before moving p. char *v = strwordtok(NULL, &p); @@ -181,11 +179,20 @@ notes.add(key, v ? v : ""); // value can be empty, but must not be NULL - modifiableOther().consume(p - other().content()); - modifiableOther().consumeWhitespacePrefix(); + other_.consume(p - other_.content()); + other_.consumeWhitespacePrefix(); } } +const MemBuf & +Helper::Reply::emptyBuf() const +{ + static MemBuf empty; + if (empty.isNull()) + empty.init(1, 1); + return empty; +} + std::ostream & operator <<(std::ostream &os, const Helper::Reply &r) { @@ -218,8 +225,9 @@ os << "}"; } - if (r.other().hasContent()) - os << ", other: \"" << r.other().content() << '\"'; + MemBuf const &o = r.other(); + if (o.hasContent()) + os << ", other: \"" << o.content() << '\"'; os << '}'; diff -u -r -N squid-4.0.12/src/helper/Reply.h squid-4.0.13/src/helper/Reply.h --- squid-4.0.12/src/helper/Reply.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/helper/Reply.h 2016-08-06 00:52:55.000000000 +1200 @@ -33,22 +33,12 @@ Reply &operator =(const Helper::Reply &r); public: - explicit Reply(Helper::ResultCode res = Helper::Unknown) : result(res), notes(), whichServer(NULL) { - other_.init(1,1); - other_.terminate(); - } - - // create/parse details from the msg buffer provided - // XXX: buf should be const but parse() needs non-const for now - Reply(char *buf, size_t len); - - const MemBuf &other() const { return other_; } - - /// backward compatibility: - /// access to modifiable blob, required by redirectHandleReply() - /// and by urlParse() in ClientRequestContext::clientRedirectDone() - /// and by token blob/arg parsing in Negotiate auth handler - MemBuf &modifiableOther() const { return *const_cast(&other_); } + explicit Reply(Helper::ResultCode res) : result(res), notes(), whichServer(NULL) {} + + /// Creates a NULL reply + Reply(); + + const MemBuf &other() const {return other_.isNull() ? emptyBuf() : other_;}; /** parse a helper response line format: * line := [ result ] *#( kv-pair ) @@ -58,7 +48,10 @@ * quoted-string are \-escape decoded and the quotes are stripped. */ // XXX: buf should be const but we may need strwordtok() and rfc1738_unescape() - void parse(char *buf, size_t len); + //void parse(char *buf, size_t len); + void finalize(); + + bool accumulate(const char *buf, size_t len); public: /// The helper response 'result' field. @@ -73,6 +66,9 @@ private: void parseResponseKeys(); + /// Return an empty MemBuf. + const MemBuf &emptyBuf() const; + /// the remainder of the line MemBuf other_; }; diff -u -r -N squid-4.0.12/src/helper.cc squid-4.0.13/src/helper.cc --- squid-4.0.12/src/helper.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/helper.cc 2016-08-06 00:52:55.000000000 +1200 @@ -37,31 +37,22 @@ /// The maximum allowed request retries. #define MAX_RETRIES 2 -/** Initial Squid input buffer size. Helper responses may exceed this, and - * Squid will grow the input buffer as needed, up to ReadBufMaxSize. - */ -const size_t ReadBufMinSize(4*1024); - -/** Maximum safe size of a helper-to-Squid response message plus one. - * Squid will warn and close the stream if a helper sends a too-big response. - * ssl_crtd helper is known to produce responses of at least 10KB in size. - * Some undocumented helpers are known to produce responses exceeding 8KB. - */ -const size_t ReadBufMaxSize(32*1024); +/// Helpers input buffer size. +const size_t ReadBufSize(32*1024); static IOCB helperHandleRead; static IOCB helperStatefulHandleRead; static void helperServerFree(helper_server *srv); static void helperStatefulServerFree(helper_stateful_server *srv); -static void Enqueue(helper * hlp, Helper::Request *); +static void Enqueue(helper * hlp, Helper::Xaction *); static helper_server *GetFirstAvailable(helper * hlp); static helper_stateful_server *StatefulGetFirstAvailable(statefulhelper * hlp); -static void helperDispatch(helper_server * srv, Helper::Request * r); -static void helperStatefulDispatch(helper_stateful_server * srv, Helper::Request * r); +static void helperDispatch(helper_server * srv, Helper::Xaction * r); +static void helperStatefulDispatch(helper_stateful_server * srv, Helper::Xaction * r); static void helperKickQueue(helper * hlp); static void helperStatefulKickQueue(statefulhelper * hlp); static void helperStatefulServerDone(helper_stateful_server * srv); -static void StatefulEnqueue(statefulhelper * hlp, Helper::Request * r); +static void StatefulEnqueue(statefulhelper * hlp, Helper::Xaction * r); CBDATA_CLASS_INIT(helper); CBDATA_CLASS_INIT(helper_server); @@ -212,10 +203,12 @@ srv->readPipe->fd = rfd; srv->writePipe = new Comm::Connection; srv->writePipe->fd = wfd; - srv->rbuf = (char *)memAllocBuf(ReadBufMinSize, &srv->rbuf_sz); + srv->rbuf = (char *)memAllocBuf(ReadBufSize, &srv->rbuf_sz); srv->wqueue = new MemBuf; srv->roffset = 0; srv->nextRequestId = 0; + srv->replyXaction = NULL; + srv->ignoreToEom = false; srv->parent = cbdataReference(hlp); dlinkAddTail(srv, &srv->link, &hlp->servers); @@ -338,7 +331,7 @@ srv->readPipe->fd = rfd; srv->writePipe = new Comm::Connection; srv->writePipe->fd = wfd; - srv->rbuf = (char *)memAllocBuf(ReadBufMinSize, &srv->rbuf_sz); + srv->rbuf = (char *)memAllocBuf(ReadBufSize, &srv->rbuf_sz); srv->roffset = 0; srv->parent = cbdataReference(hlp); @@ -377,7 +370,7 @@ } void -helper::submitRequest(Helper::Request *r) +helper::submitRequest(Helper::Xaction *r) { helper_server *srv; @@ -399,7 +392,7 @@ { if (hlp == NULL) { debugs(84, 3, "helperSubmit: hlp == NULL"); - Helper::Reply nilReply; + Helper::Reply const nilReply(Helper::Unknown); callback(data, nilReply); return; } @@ -443,7 +436,7 @@ void helper::submit(const char *buf, HLPCB * callback, void *data) { - Helper::Request *r = new Helper::Request(callback, data, buf); + Helper::Xaction *r = new Helper::Xaction(callback, data, buf); submitRequest(r); debugs(84, DBG_DATA, Raw("buf", buf, strlen(buf))); } @@ -454,7 +447,7 @@ { if (hlp == NULL) { debugs(84, 3, "helperStatefulSubmit: hlp == NULL"); - Helper::Reply nilReply; + Helper::Reply const nilReply(Helper::Unknown); callback(data, nilReply); return; } @@ -464,7 +457,7 @@ void statefulhelper::submit(const char *buf, HLPCB * callback, void *data, helper_stateful_server * lastserver) { - Helper::Request *r = new Helper::Request(callback, data, buf); + Helper::Xaction *r = new Helper::Xaction(callback, data, buf); if ((buf != NULL) && lastserver) { debugs(84, 5, "StatefulSubmit with lastserver " << lastserver); @@ -481,7 +474,7 @@ StatefulEnqueue(this, r); } - debugs(84, DBG_DATA, "placeholder: '" << r->placeholder << + debugs(84, DBG_DATA, "placeholder: '" << r->request.placeholder << "', " << Raw("buf", buf, (!buf?0:strlen(buf)))); if (!queueFull()) { @@ -548,8 +541,8 @@ for (dlink_node *link = servers.head; link; link = link->next) { HelperServerBase *srv = static_cast(link->data); assert(srv); - Helper::Request *request = srv->requests.empty() ? NULL : srv->requests.front(); - double tt = 0.001 * (request ? tvSubMsec(request->dispatch_time, current_time) : tvSubMsec(srv->dispatch_time, srv->answer_time)); + Helper::Xaction *xaction = srv->requests.empty() ? NULL : srv->requests.front(); + double tt = 0.001 * (xaction ? tvSubMsec(xaction->request.dispatch_time, current_time) : tvSubMsec(srv->dispatch_time, srv->answer_time)); p->appendf("%7u\t%7d\t%7d\t%11" PRIu64 "\t%11" PRIu64 "\t%11" PRIu64 "\t%c%c%c%c%c%c\t%7.3f\t%7d\t%s\n", srv->index.value, srv->readPipe->fd, @@ -562,10 +555,10 @@ srv->flags.closing ? 'C' : ' ', srv->flags.reserved ? 'R' : ' ', srv->flags.shutdown ? 'S' : ' ', - request && request->placeholder ? 'P' : ' ', + xaction && xaction->request.placeholder ? 'P' : ' ', tt < 0.0 ? 0.0 : tt, (int) srv->roffset, - request ? Format::QuoteMimeBlob(request->buf) : "(none)"); + xaction ? Format::QuoteMimeBlob(xaction->request.buf) : "(none)"); } p->append("\nFlags key:\n" @@ -727,13 +720,13 @@ while (!srv->requests.empty()) { // XXX: re-schedule these on another helper? - Helper::Request *r = srv->requests.front(); + Helper::Xaction *r = srv->requests.front(); srv->requests.pop_front(); void *cbdata; - if (cbdataReferenceValidDone(r->data, &cbdata)) { - Helper::Reply nilReply; - r->callback(cbdata, nilReply); + if (cbdataReferenceValidDone(r->request.data, &cbdata)) { + r->reply.result = Helper::Unknown; + r->request.callback(cbdata, r->reply); } delete r; @@ -792,13 +785,13 @@ while (!srv->requests.empty()) { // XXX: re-schedule these on another helper? - Helper::Request *r = srv->requests.front(); + Helper::Xaction *r = srv->requests.front(); srv->requests.pop_front(); void *cbdata; - if (cbdataReferenceValidDone(r->data, &cbdata)) { - Helper::Reply nilReply; - r->callback(cbdata, nilReply); + if (cbdataReferenceValidDone(r->request.data, &cbdata)) { + r->reply.result = Helper::Unknown; + r->request.callback(cbdata, r->reply); } delete r; @@ -812,39 +805,57 @@ delete srv; } -/// Calls back with a pointer to the buffer with the helper output -static void -helperReturnBuffer(int request_number, helper_server * srv, helper * hlp, char * msg, char * msg_end) +Helper::Xaction * +helper_server::popRequest(int request_number) { - Helper::Request *r = NULL; + Helper::Xaction *r = nullptr; helper_server::RequestIndex::iterator it; - if (hlp->childs.concurrency) { + if (parent->childs.concurrency) { // If concurency supported retrieve request from ID - it = srv->requestsIndex.find(request_number); - if (it != srv->requestsIndex.end()) { + it = requestsIndex.find(request_number); + if (it != requestsIndex.end()) { r = *(it->second); - srv->requests.erase(it->second); - srv->requestsIndex.erase(it); + requests.erase(it->second); + requestsIndex.erase(it); } - } else if(!srv->requests.empty()) { + } else if(!requests.empty()) { // Else get the first request from queue, if any - r = srv->requests.front(); - srv->requests.pop_front(); + r = requests.front(); + requests.pop_front(); } - if (r) { - HLPCB *callback = r->callback; - r->callback = NULL; + return r; +} + +/// Calls back with a pointer to the buffer with the helper output +static void +helperReturnBuffer(helper_server * srv, helper * hlp, char * msg, size_t msgSize, char * msgEnd) +{ + if (Helper::Xaction *r = srv->replyXaction) { + const bool hasSpace = r->reply.accumulate(msg, msgSize); + if (!hasSpace) { + debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " << + "helper that overflowed " << srv->rbuf_sz << "-byte " << + "Squid input buffer: " << hlp->id_name << " #" << srv->index); + srv->closePipesSafely(hlp->id_name); + return; + } + + if (!msgEnd) + return; // We are waiting for more data. + + HLPCB *callback = r->request.callback; + r->request.callback = nullptr; - void *cbdata = NULL; bool retry = false; - if (cbdataReferenceValidDone(r->data, &cbdata)) { - Helper::Reply response(msg, (msg_end-msg)); - if (response.result == Helper::BrokenHelper && r->retries < MAX_RETRIES) { - debugs(84, DBG_IMPORTANT, "ERROR: helper: " << response << ", attempt #" << (r->retries + 1) << " of 2"); + void *cbdata = nullptr; + if (cbdataReferenceValidDone(r->request.data, &cbdata)) { + r->reply.finalize(); + if (r->reply.result == Helper::BrokenHelper && r->request.retries < MAX_RETRIES) { + debugs(84, DBG_IMPORTANT, "ERROR: helper: " << r->reply << ", attempt #" << (r->request.retries + 1) << " of 2"); retry = true; } else - callback(cbdata, response); + callback(cbdata, r->reply); } -- srv->stats.pending; @@ -854,24 +865,20 @@ srv->answer_time = current_time; - srv->dispatch_time = r->dispatch_time; + srv->dispatch_time = r->request.dispatch_time; hlp->stats.avg_svc_time = Math::intAverage(hlp->stats.avg_svc_time, - tvSubMsec(r->dispatch_time, current_time), + tvSubMsec(r->request.dispatch_time, current_time), hlp->stats.replies, REDIRECT_AV_FACTOR); + // release or re-submit parsedRequestXaction object + srv->replyXaction = nullptr; if (retry) { - ++r->retries; + ++r->request.retries; hlp->submitRequest(r); } else delete r; - } else if (srv->stats.timedout) { - debugs(84, 3, "Timedout reply received for request-ID: " << request_number << " , ignore"); - } else { - debugs(84, DBG_IMPORTANT, "helperHandleRead: unexpected reply on channel " << - request_number << " from " << hlp->id_name << " #" << srv->index << - " '" << srv->rbuf << "'"); } if (hlp->timeout && hlp->childs.concurrency) @@ -888,7 +895,6 @@ static void helperHandleRead(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data) { - char *t = NULL; helper_server *srv = (helper_server *)data; helper *hlp = srv->parent; assert(cbdataReferenceValid(data)); @@ -922,57 +928,80 @@ srv->rbuf[0] = '\0'; } - while ((t = strchr(srv->rbuf, hlp->eom))) { - /* end of reply found */ - char *msg = srv->rbuf; - int i = 0; - int skip = 1; - debugs(84, 3, "helperHandleRead: end of reply found"); - - if (t > srv->rbuf && t[-1] == '\r' && hlp->eom == '\n') { - *t = '\0'; - // rewind to the \r octet which is the real terminal now - // and remember that we have to skip forward 2 places now. - skip = 2; - --t; + bool needsMore = false; + char *msg = srv->rbuf; + while (*msg && !needsMore) { + int skip = 0; + char *eom = strchr(msg, hlp->eom); + if (eom) { + skip = 1; + debugs(84, 3, "helperHandleRead: end of reply found"); + if (eom > msg && eom[-1] == '\r' && hlp->eom == '\n') { + *eom = '\0'; + // rewind to the \r octet which is the real terminal now + // and remember that we have to skip forward 2 places now. + skip = 2; + --eom; + } + *eom = '\0'; } - *t = '\0'; - - if (hlp->childs.concurrency) { - i = strtol(msg, &msg, 10); + if (!srv->ignoreToEom && !srv->replyXaction) { + int i = 0; + if (hlp->childs.concurrency) { + char *e = NULL; + i = strtol(msg, &e, 10); + // Do we need to check for e == msg? Means wrong response from helper. + // Will be droped as "unexpected reply on channel 0" + needsMore = !(xisspace(*e) || (eom && e == eom)); + if (!needsMore) { + msg = e; + while (*msg && xisspace(*msg)) + ++msg; + } // else not enough data to compute request number + } + if (!(srv->replyXaction = srv->popRequest(i))) { + if (srv->stats.timedout) { + debugs(84, 3, "Timedout reply received for request-ID: " << i << " , ignore"); + } else { + debugs(84, DBG_IMPORTANT, "helperHandleRead: unexpected reply on channel " << + i << " from " << hlp->id_name << " #" << srv->index << + " '" << srv->rbuf << "'"); + } + srv->ignoreToEom = true; + } + } // else we need to just append reply data to the current Xaction - while (*msg && xisspace(*msg)) - ++msg; - } + if (!needsMore) { + size_t msgSize = eom ? eom - msg : (srv->roffset - (msg - srv->rbuf)); + assert(msgSize <= srv->rbuf_sz); + helperReturnBuffer(srv, hlp, msg, msgSize, eom); + msg += msgSize + skip; + assert(static_cast(msg - srv->rbuf) <= srv->rbuf_sz); + + // The next message should not ignored. + if (eom && srv->ignoreToEom) + srv->ignoreToEom = false; + } else + assert(skip == 0 && eom == NULL); + } - helperReturnBuffer(i, srv, hlp, msg, t); - srv->roffset -= (t - srv->rbuf) + skip; - memmove(srv->rbuf, t + skip, srv->roffset); + if (needsMore) { + size_t msgSize = (srv->roffset - (msg - srv->rbuf)); + assert(msgSize <= srv->rbuf_sz); + memmove(srv->rbuf, msg, msgSize); + srv->roffset = msgSize; srv->rbuf[srv->roffset] = '\0'; + } else { + // All of the responses parsed and msg points at the end of read data + assert(static_cast(msg - srv->rbuf) == srv->roffset); + srv->roffset = 0; } if (Comm::IsConnOpen(srv->readPipe) && !fd_table[srv->readPipe->fd].closing()) { int spaceSize = srv->rbuf_sz - srv->roffset - 1; assert(spaceSize >= 0); - // grow the input buffer if needed and possible - if (!spaceSize && srv->rbuf_sz + 4096 <= ReadBufMaxSize) { - srv->rbuf = (char *)memReallocBuf(srv->rbuf, srv->rbuf_sz + 4096, &srv->rbuf_sz); - debugs(84, 3, HERE << "Grew read buffer to " << srv->rbuf_sz); - spaceSize = srv->rbuf_sz - srv->roffset - 1; - assert(spaceSize >= 0); - } - - // quit reading if there is no space left - if (!spaceSize) { - debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " << - "helper that overflowed " << srv->rbuf_sz << "-byte " << - "Squid input buffer: " << hlp->id_name << " #" << srv->index); - srv->closePipesSafely(hlp->id_name); - return; - } - AsyncCall::Pointer call = commCbCall(5,4, "helperHandleRead", CommIoCbPtrFun(helperHandleRead, srv)); comm_read(srv->readPipe, srv->rbuf + srv->roffset, spaceSize, call); @@ -1005,7 +1034,7 @@ srv->roffset += len; srv->rbuf[srv->roffset] = '\0'; - Helper::Request *r = srv->requests.front(); + Helper::Xaction *r = srv->requests.front(); debugs(84, DBG_DATA, Raw("accumulated", srv->rbuf, srv->roffset)); if (r == NULL) { @@ -1018,40 +1047,46 @@ } if ((t = strchr(srv->rbuf, hlp->eom))) { - /* end of reply found */ - srv->requests.pop_front(); // we already have it in 'r' - int called = 1; - int skip = 1; debugs(84, 3, "helperStatefulHandleRead: end of reply found"); if (t > srv->rbuf && t[-1] == '\r' && hlp->eom == '\n') { *t = '\0'; // rewind to the \r octet which is the real terminal now - // and remember that we have to skip forward 2 places now. - skip = 2; --t; } *t = '\0'; + } + + if (r && !r->reply.accumulate(srv->rbuf, t ? (t - srv->rbuf) : srv->roffset)) { + debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " << + "helper that overflowed " << srv->rbuf_sz << "-byte " << + "Squid input buffer: " << hlp->id_name << " #" << srv->index); + srv->closePipesSafely(hlp->id_name); + return; + } + /** + * BUG: the below assumes that only one response per read() was received and discards any octets remaining. + * Doing this prohibits concurrency support with multiple replies per read(). + * TODO: check that read() setup on these buffers pays attention to roffest!=0 + * TODO: check that replies bigger than the buffer are discarded and do not to affect future replies + */ + srv->roffset = 0; - if (r && cbdataReferenceValid(r->data)) { - Helper::Reply res(srv->rbuf, (t - srv->rbuf)); - res.whichServer = srv; - r->callback(r->data, res); + if (t) { + /* end of reply found */ + srv->requests.pop_front(); // we already have it in 'r' + int called = 1; + + if (r && cbdataReferenceValid(r->request.data)) { + r->reply.finalize(); + r->reply.whichServer = srv; + r->request.callback(r->request.data, r->reply); } else { debugs(84, DBG_IMPORTANT, "StatefulHandleRead: no callback data registered"); called = 0; } - // only skip off the \0's _after_ passing its location in Helper::Reply above - t += skip; - /** - * BUG: the below assumes that only one response per read() was received and discards any octets remaining. - * Doing this prohibits concurrency support with multiple replies per read(). - * TODO: check that read() setup on these buffers pays attention to roffest!=0 - * TODO: check that replies bigger than the buffer are discarded and do not to affect future replies - */ - srv->roffset = 0; delete r; -- srv->stats.pending; @@ -1071,35 +1106,17 @@ } if (Comm::IsConnOpen(srv->readPipe) && !fd_table[srv->readPipe->fd].closing()) { - int spaceSize = srv->rbuf_sz - srv->roffset - 1; - assert(spaceSize >= 0); - - // grow the input buffer if needed and possible - if (!spaceSize && srv->rbuf_sz + 4096 <= ReadBufMaxSize) { - srv->rbuf = (char *)memReallocBuf(srv->rbuf, srv->rbuf_sz + 4096, &srv->rbuf_sz); - debugs(84, 3, HERE << "Grew read buffer to " << srv->rbuf_sz); - spaceSize = srv->rbuf_sz - srv->roffset - 1; - assert(spaceSize >= 0); - } - - // quit reading if there is no space left - if (!spaceSize) { - debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " << - "helper that overflowed " << srv->rbuf_sz << "-byte " << - "Squid input buffer: " << hlp->id_name << " #" << srv->index); - srv->closePipesSafely(hlp->id_name); - return; - } + int spaceSize = srv->rbuf_sz - 1; AsyncCall::Pointer call = commCbCall(5,4, "helperStatefulHandleRead", CommIoCbPtrFun(helperStatefulHandleRead, srv)); - comm_read(srv->readPipe, srv->rbuf + srv->roffset, spaceSize, call); + comm_read(srv->readPipe, srv->rbuf, spaceSize, call); } } /// Handles a request when all running helpers, if any, are busy. static void -Enqueue(helper * hlp, Helper::Request * r) +Enqueue(helper * hlp, Helper::Xaction * r) { hlp->queue.push(r); ++ hlp->stats.queue_size; @@ -1128,7 +1145,7 @@ } static void -StatefulEnqueue(statefulhelper * hlp, Helper::Request * r) +StatefulEnqueue(statefulhelper * hlp, Helper::Xaction * r) { hlp->queue.push(r); ++ hlp->stats.queue_size; @@ -1156,7 +1173,7 @@ debugs(84, DBG_CRITICAL, "WARNING: Consider increasing the number of " << hlp->id_name << " processes in your config file."); } -Helper::Request * +Helper::Xaction * helper::nextRequest() { if (queue.empty()) @@ -1272,20 +1289,20 @@ } static void -helperDispatch(helper_server * srv, Helper::Request * r) +helperDispatch(helper_server * srv, Helper::Xaction * r) { helper *hlp = srv->parent; const uint64_t reqId = ++srv->nextRequestId; - if (!cbdataReferenceValid(r->data)) { + if (!cbdataReferenceValid(r->request.data)) { debugs(84, DBG_IMPORTANT, "helperDispatch: invalid callback data"); delete r; return; } - r->Id = reqId; + r->request.Id = reqId; helper_server::Requests::iterator it = srv->requests.insert(srv->requests.end(), r); - r->dispatch_time = current_time; + r->request.dispatch_time = current_time; if (srv->wqueue->isNull()) srv->wqueue->init(); @@ -1293,9 +1310,9 @@ if (hlp->childs.concurrency) { srv->requestsIndex.insert(helper_server::RequestIndex::value_type(reqId, it)); assert(srv->requestsIndex.size() == srv->requests.size()); - srv->wqueue->appendf("%" PRIu64 " %s", reqId, r->buf); + srv->wqueue->appendf("%" PRIu64 " %s", reqId, r->request.buf); } else - srv->wqueue->append(r->buf, strlen(r->buf)); + srv->wqueue->append(r->request.buf, strlen(r->request.buf)); if (!srv->flags.writing) { assert(NULL == srv->writebuf); @@ -1307,7 +1324,7 @@ Comm::Write(srv->writePipe, srv->writebuf->content(), srv->writebuf->contentSize(), call, NULL); } - debugs(84, 5, "helperDispatch: Request sent to " << hlp->id_name << " #" << srv->index << ", " << strlen(r->buf) << " bytes"); + debugs(84, 5, "helperDispatch: Request sent to " << hlp->id_name << " #" << srv->index << ", " << strlen(r->request.buf) << " bytes"); ++ srv->stats.uses; ++ srv->stats.pending; @@ -1319,11 +1336,11 @@ {} static void -helperStatefulDispatch(helper_stateful_server * srv, Helper::Request * r) +helperStatefulDispatch(helper_stateful_server * srv, Helper::Xaction * r) { statefulhelper *hlp = srv->parent; - if (!cbdataReferenceValid(r->data)) { + if (!cbdataReferenceValid(r->request.data)) { debugs(84, DBG_IMPORTANT, "helperStatefulDispatch: invalid callback data"); delete r; helperStatefulReleaseServer(srv); @@ -1332,13 +1349,13 @@ debugs(84, 9, "helperStatefulDispatch busying helper " << hlp->id_name << " #" << srv->index); - if (r->placeholder == 1) { + if (r->request.placeholder == 1) { /* a callback is needed before this request can _use_ a helper. */ /* we don't care about releasing this helper. The request NEVER * gets to the helper. So we throw away the return code */ - Helper::Reply nilReply; - nilReply.whichServer = srv; - r->callback(r->data, nilReply); + r->reply.result = Helper::Unknown; + r->reply.whichServer = srv; + r->request.callback(r->request.data, r->reply); /* throw away the placeholder */ delete r; /* and push the queue. Note that the callback may have submitted a new @@ -1355,10 +1372,10 @@ srv->dispatch_time = current_time; AsyncCall::Pointer call = commCbCall(5,5, "helperStatefulDispatchWriteDone", CommIoCbPtrFun(helperStatefulDispatchWriteDone, hlp)); - Comm::Write(srv->writePipe, r->buf, strlen(r->buf), call, NULL); + Comm::Write(srv->writePipe, r->request.buf, strlen(r->request.buf), call, NULL); debugs(84, 5, "helperStatefulDispatch: Request sent to " << hlp->id_name << " #" << srv->index << ", " << - (int) strlen(r->buf) << " bytes"); + (int) strlen(r->request.buf) << " bytes"); ++ srv->stats.uses; ++ srv->stats.pending; @@ -1368,7 +1385,7 @@ static void helperKickQueue(helper * hlp) { - Helper::Request *r; + Helper::Xaction *r; helper_server *srv; while ((srv = GetFirstAvailable(hlp)) && (r = hlp->nextRequest())) @@ -1378,7 +1395,7 @@ static void helperStatefulKickQueue(statefulhelper * hlp) { - Helper::Request *r; + Helper::Xaction *r; helper_stateful_server *srv; while ((srv = StatefulGetFirstAvailable(hlp)) && (r = hlp->nextRequest())) @@ -1400,29 +1417,32 @@ helper_server::checkForTimedOutRequests(bool const retry) { assert(parent->childs.concurrency); - while(!requests.empty() && requests.front()->timedOut(parent->timeout)) { - Helper::Request *r = requests.front(); + while(!requests.empty() && requests.front()->request.timedOut(parent->timeout)) { + Helper::Xaction *r = requests.front(); RequestIndex::iterator it; - it = requestsIndex.find(r->Id); + it = requestsIndex.find(r->request.Id); assert(it != requestsIndex.end()); requestsIndex.erase(it); requests.pop_front(); - debugs(84, 2, "Request " << r->Id << " timed-out, remove it from queue"); + debugs(84, 2, "Request " << r->request.Id << " timed-out, remove it from queue"); void *cbdata; bool retried = false; - if (retry && r->retries < MAX_RETRIES && cbdataReferenceValid(r->data)) { - debugs(84, 2, "Retry request " << r->Id); - ++r->retries; + if (retry && r->request.retries < MAX_RETRIES && cbdataReferenceValid(r->request.data)) { + debugs(84, 2, "Retry request " << r->request.Id); + ++r->request.retries; parent->submitRequest(r); retried = true; - } else if (cbdataReferenceValidDone(r->data, &cbdata)) { + } else if (cbdataReferenceValidDone(r->request.data, &cbdata)) { if (!parent->onTimedOutResponse.isEmpty()) { - // Helper::Reply needs a non const buffer - char *replyMsg = xstrdup(parent->onTimedOutResponse.c_str()); - r->callback(cbdata, Helper::Reply(replyMsg, strlen(replyMsg))); - xfree(replyMsg); - } else - r->callback(cbdata, Helper::Reply(Helper::TimedOut)); + if (r->reply.accumulate(parent->onTimedOutResponse.rawContent(), parent->onTimedOutResponse.length())) + r->reply.finalize(); + else + r->reply.result = Helper::TimedOut; + r->request.callback(cbdata, r->reply); + } else { + r->reply.result = Helper::TimedOut; + r->request.callback(cbdata, r->reply); + } } --stats.pending; ++stats.timedout; @@ -1447,7 +1467,7 @@ AsyncCall::Pointer timeoutCall = commCbCall(84, 4, "helper_server::requestTimeout", CommTimeoutCbPtrFun(helper_server::requestTimeout, srv)); - const int timeSpent = srv->requests.empty() ? 0 : (squid_curtime - srv->requests.front()->dispatch_time.tv_sec); + const int timeSpent = srv->requests.empty() ? 0 : (squid_curtime - srv->requests.front()->request.dispatch_time.tv_sec); const int timeLeft = max(1, (static_cast(srv->parent->timeout) - timeSpent)); commSetConnTimeout(io.conn, timeLeft, timeoutCall); diff -u -r -N squid-4.0.12/src/helper.h squid-4.0.13/src/helper.h --- squid-4.0.12/src/helper.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/helper.h 2016-08-06 00:52:55.000000000 +1200 @@ -18,6 +18,8 @@ #include "dlink.h" #include "helper/ChildConfig.h" #include "helper/forward.h" +#include "helper/Reply.h" +#include "helper/Request.h" #include "ip/Address.h" #include "sbuf/SBuf.h" @@ -28,6 +30,18 @@ class Packable; class wordlist; +namespace Helper +{ +/// Holds the required data to serve a helper request. +class Xaction { + MEMPROXY_CLASS(Helper::Xaction); +public: + Xaction(HLPCB *c, void *d, const char *b): request(c, d, b) {} + Helper::Request request; + Helper::Reply reply; +}; +} + /** * Managers a set of individual helper processes with a common queue of requests. * @@ -67,14 +81,14 @@ bool queueFull() const; /// \returns next request in the queue, or nil. - Helper::Request *nextRequest(); + Helper::Xaction *nextRequest(); ///< If not full, submit request. Otherwise, either kill Squid or return false. bool trySubmit(const char *buf, HLPCB * callback, void *data); /// Submits a request to the helper or add it to the queue if none of /// the servers is available. - void submitRequest(Helper::Request *r); + void submitRequest(Helper::Xaction *r); /// Dump some stats about the helper state to a Packable object void packStatsInto(Packable *p, const char *label = NULL) const; @@ -82,7 +96,7 @@ public: wordlist *cmdline; dlink_list servers; - std::queue queue; + std::queue queue; const char *id_name; Helper::ChildConfig childs; ///< Configuration settings for number running. int ipc_type; @@ -173,7 +187,7 @@ bool reserved; } flags; - typedef std::list Requests; + typedef std::list Requests; Requests requests; ///< requests in order of submission/expiration struct { @@ -201,10 +215,25 @@ helper *parent; + /// The helper request Xaction object for the current reply . + /// A helper reply may be distributed to more than one of the retrieved + /// packets from helper. This member stores the Xaction object as long as + /// the end-of-message for current reply is not retrieved. + Helper::Xaction *replyXaction; + + /// Whether to ignore current message, because it is timed-out or other reason + bool ignoreToEom; + // STL says storing std::list iterators is safe when changing the list typedef std::map RequestIndex; RequestIndex requestsIndex; ///< maps request IDs to requests + /// Search in queue for the request with requestId, return the related + /// Xaction object and remove it from queue. + /// If concurrency is disabled then the requestId is ignored and the + /// Xaction of the next request in queue is retrieved. + Helper::Xaction *popRequest(int requestId); + /// Run over the active requests lists and forces a retry, or timedout reply /// or the configured "on timeout response" for timedout requests. void checkForTimedOutRequests(bool const retry); diff -u -r -N squid-4.0.12/src/http/one/Parser.cc squid-4.0.13/src/http/one/Parser.cc --- squid-4.0.12/src/http/one/Parser.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/http/one/Parser.cc 2016-08-06 00:52:55.000000000 +1200 @@ -64,7 +64,11 @@ if (Config.onoff.relaxed_header_parser && tok.skipOne(CharacterSet::LF)) return true; - return false; + if (tok.atEnd() || (tok.remaining().length() == 1 && tok.remaining().at(0) == '\r')) + return false; // need more data + + throw TexcHere("garbage instead of CRLF line terminator"); + return false; // unreachable, but make naive compilers happy } /// all characters except the LF line terminator diff -u -r -N squid-4.0.12/src/http/one/Parser.h squid-4.0.13/src/http/one/Parser.h --- squid-4.0.12/src/http/one/Parser.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/http/one/Parser.h 2016-08-06 00:52:55.000000000 +1200 @@ -107,8 +107,14 @@ Http::StatusCode parseStatusCode; protected: - /// detect and skip the CRLF or (if tolerant) LF line terminator - /// consume from the tokenizer and return true only if found + /** + * detect and skip the CRLF or (if tolerant) LF line terminator + * consume from the tokenizer. + * + * throws if non-terminator is detected. + * \retval true only if line terminator found. + * \retval false incomplete or missing line terminator, need more data. + */ bool skipLineTerminator(Http1::Tokenizer &tok) const; /// the characters which are to be considered valid whitespace diff -u -r -N squid-4.0.12/src/http/one/ResponseParser.cc squid-4.0.13/src/http/one/ResponseParser.cc --- squid-4.0.12/src/http/one/ResponseParser.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/http/one/ResponseParser.cc 2016-08-06 00:52:55.000000000 +1200 @@ -75,9 +75,6 @@ // NOTE: any whitespace after the single SP is part of the reason phrase. } - if (tok.atEnd()) - return 0; // need more to be sure we have it all - /* RFC 7230 says we SHOULD ignore the reason phrase content * but it has a definite valid vs invalid character set. * We interpret the SHOULD as ignoring absence and syntax, but @@ -89,17 +86,18 @@ // if we got here we are still looking for reason-phrase bytes static const CharacterSet phraseChars = CharacterSet::WSP + CharacterSet::VCHAR + CharacterSet::OBSTEXT; (void)tok.prefix(reasonPhrase_, phraseChars); // optional, no error if missing - if (skipLineTerminator(tok)) { - debugs(74, DBG_DATA, "parse remaining buf={length=" << tok.remaining().length() << ", data='" << tok.remaining() << "'}"); - buf_ = tok.remaining(); // resume checkpoint - return 1; - } - reasonPhrase_.clear(); - - if (tok.atEnd()) + try { + if (skipLineTerminator(tok)) { + debugs(74, DBG_DATA, "parse remaining buf={length=" << tok.remaining().length() << ", data='" << tok.remaining() << "'}"); + buf_ = tok.remaining(); // resume checkpoint + return 1; + } + reasonPhrase_.clear(); return 0; // need more to be sure we have it all - debugs(74, 6, "invalid status-line. garbage in reason phrase."); + } catch (const std::exception &ex) { + debugs(74, 6, "invalid status-line: " << ex.what()); + } return -1; } diff -u -r -N squid-4.0.12/src/http/one/TeChunkedParser.cc squid-4.0.13/src/http/one/TeChunkedParser.cc --- squid-4.0.12/src/http/one/TeChunkedParser.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/http/one/TeChunkedParser.cc 2016-08-06 00:52:55.000000000 +1200 @@ -153,9 +153,6 @@ buf_ = tok.remaining(); // parse checkpoint (unless there might be more token name) } - if (tok.atEnd()) - return false; - if (skipLineTerminator(tok)) { buf_ = tok.remaining(); // checkpoint // non-0 chunk means data, 0-size means optional Trailer follows @@ -163,7 +160,6 @@ return true; } - throw TexcHere("corrupted chunk extension value"); return false; } @@ -202,9 +198,6 @@ theChunkSize = 0; // done with the current chunk parsingStage_ = Http1::HTTP_PARSE_CHUNK_SZ; return true; - - } else if (!tok.atEnd()) { - throw TexcHere("found data between chunk end and CRLF"); } return false; diff -u -r -N squid-4.0.12/src/http/Stream.cc squid-4.0.13/src/http/Stream.cc --- squid-4.0.12/src/http/Stream.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/http/Stream.cc 2016-08-06 00:52:55.000000000 +1200 @@ -11,7 +11,6 @@ #include "http/Stream.h" #include "HttpHdrContRange.h" #include "HttpHeaderTools.h" -#include "SquidConfig.h" #include "Store.h" #include "TimeOrTag.h" @@ -21,10 +20,10 @@ reply(nullptr), writtenToSocket(0), mayUseConnection_(false), - connRegistered_(false), - requestBuffer(nullptr) + connRegistered_(false) { assert(http != nullptr); + memset(reqbuf, '\0', sizeof (reqbuf)); flags.deferred = 0; flags.parsed_ok = 0; deferredparams.node = nullptr; @@ -110,10 +109,12 @@ debugs(33, 5, reply << " written " << http->out.size << " into " << clientConnection); /* More data will be coming from the stream. */ - StoreIOBuffer readBuffer = getClientStreamBuffer(); + StoreIOBuffer readBuffer; /* XXX: Next requested byte in the range sequence */ /* XXX: length = getmaximumrangelenfgth */ readBuffer.offset = getNextRangeOffset(); + readBuffer.length = HTTP_REQBUF_SZ; + readBuffer.data = reqbuf; /* we may note we have reached the end of the wanted ranges */ clientStreamRead(getTail(), http, readBuffer); } @@ -567,18 +568,6 @@ deferredparams.queuedBuffer = receivedData; } -StoreIOBuffer -Http::Stream::getClientStreamBuffer() -{ - if (!requestBuffer) { - requestBuffer = new MemBlob(Config.readAheadGap); - } - StoreIOBuffer tempBuffer; - tempBuffer.data = requestBuffer->mem; - tempBuffer.length = requestBuffer->spaceSize(); - return tempBuffer; -} - void Http::Stream::prepareReply(HttpReply *rep) { diff -u -r -N squid-4.0.12/src/http/Stream.h squid-4.0.13/src/http/Stream.h --- squid-4.0.12/src/http/Stream.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/http/Stream.h 2016-08-06 00:52:55.000000000 +1200 @@ -120,13 +120,12 @@ void deferRecipientForLater(clientStreamNode *, HttpReply *, StoreIOBuffer receivedData); - StoreIOBuffer getClientStreamBuffer(); - public: // HTTP/1.x state data Comm::ConnectionPointer clientConnection; ///< details about the client connection socket ClientHttpRequest *http; /* we pretend to own that Job */ HttpReply *reply; + char reqbuf[HTTP_REQBUF_SZ]; struct { unsigned deferred:1; ///< This is a pipelined request waiting for the current object to complete unsigned parsed_ok:1; ///< Was this parsed correctly? @@ -159,8 +158,6 @@ bool mayUseConnection_; /* This request may use the connection. Don't read anymore requests for now */ bool connRegistered_; - - MemBlob::Pointer requestBuffer; }; } // namespace Http diff -u -r -N squid-4.0.12/src/http/url_rewriters/LFS/url_lfs_rewrite.8 squid-4.0.13/src/http/url_rewriters/LFS/url_lfs_rewrite.8 --- squid-4.0.12/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2016-07-02 02:24:36.000000000 +1200 +++ squid-4.0.13/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2016-08-06 02:28:23.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "URL_LFS_REWRITE 8" -.TH URL_LFS_REWRITE 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH URL_LFS_REWRITE 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/HttpHeader.cc squid-4.0.13/src/HttpHeader.cc --- squid-4.0.12/src/HttpHeader.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/HttpHeader.cc 2016-08-06 00:52:55.000000000 +1200 @@ -24,7 +24,6 @@ #include "profiler/Profiler.h" #include "rfc1123.h" #include "SquidConfig.h" -//#include "SquidString.h" // pulled by HttpHdrCc.h #include "StatHist.h" #include "Store.h" #include "StrList.h" @@ -238,6 +237,22 @@ } } +/// check whether the fresh header has any new/changed updatable fields +bool +HttpHeader::needUpdate(HttpHeader const *fresh) const +{ + for (const auto e: fresh->entries) { + if (skipUpdateHeader(e->id)) + continue; + String value; + const char *name = e->name.termedBuf(); + if (!getByNameIfPresent(name, strlen(name), value) || + (value != fresh->getByName(name))) + return true; + } + return false; +} + void HttpHeader::updateWarnings() { @@ -258,16 +273,22 @@ return id == Http::HdrType::WARNING; } -void +bool HttpHeader::update(HttpHeader const *fresh) { - const HttpHeaderEntry *e; - HttpHeaderPos pos = HttpHeaderInitPos; assert(fresh); assert(this != fresh); + // Optimization: Finding whether a header field changed is expensive + // and probably not worth it except for collapsed revalidation needs. + if (Config.onoff.collapsed_forwarding && !needUpdate(fresh)) + return false; + updateWarnings(); + const HttpHeaderEntry *e; + HttpHeaderPos pos = HttpHeaderInitPos; + while ((e = fresh->getEntry(&pos))) { /* deny bad guys (ok to check for Http::HdrType::OTHER) here */ @@ -291,6 +312,7 @@ addEntry(e->clone()); } + return true; } int diff -u -r -N squid-4.0.12/src/HttpHeader.h squid-4.0.13/src/HttpHeader.h --- squid-4.0.12/src/HttpHeader.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/HttpHeader.h 2016-08-06 00:52:55.000000000 +1200 @@ -81,7 +81,7 @@ /* Interface functions */ void clean(); void append(const HttpHeader * src); - void update(HttpHeader const *fresh); + bool update(HttpHeader const *fresh); void compact(); int parse(const char *header_start, size_t len); void packInto(Packable * p, bool mask_sensitive_info=false) const; @@ -145,6 +145,7 @@ protected: /** \deprecated Public access replaced by removeHopByHopEntries() */ void removeConnectionHeaderEntries(); + bool needUpdate(const HttpHeader *fresh) const; bool skipUpdateHeader(const Http::HdrType id) const; void updateWarnings(); diff -u -r -N squid-4.0.12/src/HttpReply.cc squid-4.0.13/src/HttpReply.cc --- squid-4.0.12/src/HttpReply.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/HttpReply.cc 2016-08-06 00:52:55.000000000 +1200 @@ -235,19 +235,23 @@ return 1; } -void +bool HttpReply::updateOnNotModified(HttpReply const * freshRep) { assert(freshRep); + /* update raw headers */ + if (!header.update(&freshRep->header)) + return false; + /* clean cache */ hdrCacheClean(); - /* update raw headers */ - header.update(&freshRep->header); header.compact(); /* init cache */ hdrCacheInit(); + + return true; } /* internal routines */ diff -u -r -N squid-4.0.12/src/HttpReply.h squid-4.0.13/src/HttpReply.h --- squid-4.0.12/src/HttpReply.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/HttpReply.h 2016-08-06 00:52:55.000000000 +1200 @@ -72,7 +72,7 @@ virtual bool inheritProperties(const HttpMsg *aMsg); - void updateOnNotModified(HttpReply const *other); + bool updateOnNotModified(HttpReply const *other); /** set commonly used info with one call */ void setHeaders(Http::StatusCode status, diff -u -r -N squid-4.0.12/src/HttpRequest.cc squid-4.0.13/src/HttpRequest.cc --- squid-4.0.12/src/HttpRequest.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/HttpRequest.cc 2016-08-06 00:52:55.000000000 +1200 @@ -14,6 +14,7 @@ #include "acl/FilledChecklist.h" #include "client_side.h" #include "dns/LookupDetails.h" +#include "Downloader.h" #include "err_detail_type.h" #include "globals.h" #include "gopher.h" @@ -250,6 +251,8 @@ // main property is which connection the request was received on (if any) clientConnectionManager = aReq->clientConnectionManager; + downloader = aReq->downloader; + notes = aReq->notes; sources = aReq->sources; diff -u -r -N squid-4.0.12/src/HttpRequest.h squid-4.0.13/src/HttpRequest.h --- squid-4.0.12/src/HttpRequest.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/HttpRequest.h 2016-08-06 00:52:55.000000000 +1200 @@ -34,6 +34,7 @@ #endif class ConnStateData; +class Downloader; /* Http Request */ void httpRequestPack(void *obj, Packable *p); @@ -212,6 +213,9 @@ */ CbcPointer clientConnectionManager; + /// The Downloader object which initiated the HTTP request if any + CbcPointer downloader; + /// forgets about the cached Range header (for a reason) void ignoreRange(const char *reason); int64_t getRangeOffsetLimit(); /* the result of this function gets cached in rangeOffsetLimit */ diff -u -r -N squid-4.0.12/src/LoadableModule.cc squid-4.0.13/src/LoadableModule.cc --- squid-4.0.12/src/LoadableModule.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/LoadableModule.cc 2016-08-06 00:52:55.000000000 +1200 @@ -7,20 +7,8 @@ */ #include "squid.h" - -/* The original code has this constant ./configure-able. - * The "#else" branches use raw dlopen interface and have not been tested. - * We can remove that code if we are going to rely on libtool's ltdl in - * all environments. */ -#define XSTD_USE_LIBLTDL 1 - -#if XSTD_USE_LIBLTDL -#include "libltdl/ltdl.h" /* generated file */ -#else -#include -#endif - #include "base/TextException.h" +#include "libltdl/ltdl.h" /* generated file */ #include "LoadableModule.h" // Note: We must use preprocessor instead of C ifs because if dlopen() @@ -28,29 +16,27 @@ LoadableModule::LoadableModule(const String &aName): theName(aName), theHandle(0) { -# if XSTD_USE_LIBLTDL // Initialise preloaded symbol lookup table. LTDL_SET_PRELOADED_SYMBOLS(); if (lt_dlinit() != 0) throw TexcHere("internal error: cannot initialize libtool module loader"); -# endif } LoadableModule::~LoadableModule() { if (loaded()) unload(); -# if XSTD_USE_LIBLTDL assert(lt_dlexit() == 0); // XXX: replace with a warning -# endif } -bool LoadableModule::loaded() const +bool +LoadableModule::loaded() const { return theHandle != 0; } -void LoadableModule::load(int mode) +void +LoadableModule::load(int mode) { if (loaded()) throw TexcHere("internal error: reusing LoadableModule object"); @@ -61,7 +47,8 @@ throw TexcHere(errorMsg()); } -void LoadableModule::unload() +void +LoadableModule::unload() { if (!loaded()) throw TexcHere("internal error: unloading not loaded module"); @@ -72,32 +59,22 @@ theHandle = 0; } -void *LoadableModule::openModule(int mode) +void * +LoadableModule::openModule(int mode) { -# if XSTD_USE_LIBLTDL return lt_dlopen(theName.termedBuf()); -# else - return dlopen(theName.termedBuf(), - mode == lmNow ? RTLD_NOW : RTLD_LAZY); -# endif } -bool LoadableModule::closeModule() +bool +LoadableModule::closeModule() { -# if XSTD_USE_LIBLTDL // we cast to avoid including ltdl.h in LoadableModule.h return lt_dlclose(static_cast(theHandle)) == 0; -# else - return dlclose(theHandle) == 0; -# endif } -const char *LoadableModule::errorMsg() +const char * +LoadableModule::errorMsg() { -# if XSTD_USE_LIBLTDL return lt_dlerror(); -# else - return dlerror(); -# endif } diff -u -r -N squid-4.0.12/src/log/CustomLog.h squid-4.0.13/src/log/CustomLog.h --- squid-4.0.12/src/log/CustomLog.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/log/CustomLog.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,7 +9,6 @@ #ifndef SQUID_CUSTOMLOG_H_ #define SQUID_CUSTOMLOG_H_ -//#include "format/Format.h" #include "acl/forward.h" #include "log/Formats.h" diff -u -r -N squid-4.0.12/src/log/DB/log_db_daemon.8 squid-4.0.13/src/log/DB/log_db_daemon.8 --- squid-4.0.12/src/log/DB/log_db_daemon.8 2016-07-02 02:24:43.000000000 +1200 +++ squid-4.0.13/src/log/DB/log_db_daemon.8 2016-08-06 02:28:37.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "LOG_DB_DAEMON 8" -.TH LOG_DB_DAEMON 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH LOG_DB_DAEMON 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/Makefile.am squid-4.0.13/src/Makefile.am --- squid-4.0.12/src/Makefile.am 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/Makefile.am 2016-08-06 00:52:55.000000000 +1200 @@ -272,6 +272,8 @@ dlink.h \ dlink.cc \ $(DNSSOURCE) \ + Downloader.cc \ + Downloader.h \ enums.h \ err_type.h \ err_detail_type.h \ @@ -1433,6 +1435,8 @@ tests_testCacheManager_LDFLAGS = $(LIBADD_DL) tests_testDiskIO_SOURCES = \ + AccessLogEntry.cc \ + AccessLogEntry.h \ CacheDigest.h \ tests/stub_CacheDigest.cc \ cbdata.cc \ @@ -1538,6 +1542,7 @@ tests/stub_libeui.cc \ tests/stub_libformat.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_MemStore.cc \ mime.h \ tests/stub_mime.cc \ @@ -1581,6 +1586,7 @@ fs/libfs.la \ ipc/libipc.la \ $(REPL_OBJS) \ + $(ADAPTATION_LIBS) \ DiskIO/libdiskio.la \ acl/libapi.la \ anyp/libanyp.la \ @@ -2896,6 +2902,8 @@ $(REPL_OBJS) tests_testUfs_SOURCES = \ + AccessLogEntry.cc \ + AccessLogEntry.h \ tests/testUfs.cc \ tests/testUfs.h \ tests/stub_cache_manager.cc \ @@ -2907,6 +2915,7 @@ tests/stub_ipcache.cc \ tests/stub_libeui.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_MemStore.cc \ tests/stub_neighbors.cc \ tests/stub_pconn.cc \ @@ -3053,6 +3062,7 @@ ip/libip.la \ mem/libmem.la \ store/libstore.la \ + $(ADAPTATION_LIBS) \ sbuf/libsbuf.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ @@ -3083,6 +3093,8 @@ $(XTRA_LIBS) tests_testRock_SOURCES = \ + AccessLogEntry.cc \ + AccessLogEntry.h \ cbdata.cc \ CacheDigest.h \ CollapsedForwarding.h \ @@ -3181,6 +3193,7 @@ tests/stub_libeui.cc \ tests/stub_libformat.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_libmgr.cc \ tests/stub_libsecurity.cc \ tests/stub_MemStore.cc \ @@ -3226,6 +3239,7 @@ base/libbase.la \ mem/libmem.la \ store/libstore.la \ + $(ADAPTATION_LIBS) \ sbuf/libsbuf.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ diff -u -r -N squid-4.0.12/src/Makefile.in squid-4.0.13/src/Makefile.in --- squid-4.0.12/src/Makefile.in 2016-07-02 01:28:23.000000000 +1200 +++ squid-4.0.13/src/Makefile.in 2016-08-06 00:54:37.000000000 +1200 @@ -256,15 +256,15 @@ DelayUser.cc DelayUser.h DelayVector.cc DelayVector.h \ NullDelayId.cc NullDelayId.h ClientDelayConfig.cc \ ClientDelayConfig.h fs_io.h fs_io.cc dlink.h dlink.cc \ - dns_internal.cc enums.h err_type.h err_detail_type.h \ - errorpage.cc errorpage.h ETag.cc ETag.h event.cc event.h \ - EventLoop.h EventLoop.cc external_acl.cc ExternalACL.h \ - ExternalACLEntry.cc ExternalACLEntry.h FadingCounter.h \ - FadingCounter.cc fatal.h fatal.cc fd.h fd.cc fde.cc fde.h \ - FileMap.h filemap.cc fqdncache.h fqdncache.cc FwdState.cc \ - FwdState.h Generic.h globals.h gopher.h gopher.cc helper.cc \ - helper.h hier_code.h HierarchyLogEntry.h htcp.cc htcp.h \ - HttpStateFlags.h http.cc http.h HttpHeaderFieldStat.h \ + dns_internal.cc Downloader.cc Downloader.h enums.h err_type.h \ + err_detail_type.h errorpage.cc errorpage.h ETag.cc ETag.h \ + event.cc event.h EventLoop.h EventLoop.cc external_acl.cc \ + ExternalACL.h ExternalACLEntry.cc ExternalACLEntry.h \ + FadingCounter.h FadingCounter.cc fatal.h fatal.cc fd.h fd.cc \ + fde.cc fde.h FileMap.h filemap.cc fqdncache.h fqdncache.cc \ + FwdState.cc FwdState.h Generic.h globals.h gopher.h gopher.cc \ + helper.cc helper.h hier_code.h HierarchyLogEntry.h htcp.cc \ + htcp.h HttpStateFlags.h http.cc http.h HttpHeaderFieldStat.h \ HttpHdrCc.h HttpHdrCc.cc HttpHdrCc.cci HttpHdrRange.cc \ HttpHdrSc.cc HttpHdrSc.h HttpHdrScTarget.cc HttpHdrScTarget.h \ HttpHdrContRange.cc HttpHdrContRange.h HttpHeaderStat.h \ @@ -345,8 +345,8 @@ CpuAffinity.$(OBJEXT) CpuAffinityMap.$(OBJEXT) \ CpuAffinitySet.$(OBJEXT) debug.$(OBJEXT) $(am__objects_3) \ fs_io.$(OBJEXT) dlink.$(OBJEXT) $(am__objects_4) \ - errorpage.$(OBJEXT) ETag.$(OBJEXT) event.$(OBJEXT) \ - EventLoop.$(OBJEXT) external_acl.$(OBJEXT) \ + Downloader.$(OBJEXT) errorpage.$(OBJEXT) ETag.$(OBJEXT) \ + event.$(OBJEXT) EventLoop.$(OBJEXT) external_acl.$(OBJEXT) \ ExternalACLEntry.$(OBJEXT) FadingCounter.$(OBJEXT) \ fatal.$(OBJEXT) fd.$(OBJEXT) fde.$(OBJEXT) filemap.$(OBJEXT) \ fqdncache.$(OBJEXT) FwdState.$(OBJEXT) gopher.$(OBJEXT) \ @@ -687,9 +687,9 @@ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ $(AM_CXXFLAGS) $(CXXFLAGS) $(tests_testConfigParser_LDFLAGS) \ $(LDFLAGS) -o $@ -am__tests_testDiskIO_SOURCES_DIST = CacheDigest.h \ - tests/stub_CacheDigest.cc cbdata.cc client_db.h ClientInfo.h \ - tests/stub_CollapsedForwarding.cc ConfigOption.cc \ +am__tests_testDiskIO_SOURCES_DIST = AccessLogEntry.cc AccessLogEntry.h \ + CacheDigest.h tests/stub_CacheDigest.cc cbdata.cc client_db.h \ + ClientInfo.h tests/stub_CollapsedForwarding.cc ConfigOption.cc \ ConfigParser.cc CommonPool.h CompositePoolNode.h \ delay_pools.cc DelayId.cc DelayId.h DelayIdComposite.h \ DelayBucket.cc DelayBucket.h DelayConfig.cc DelayConfig.h \ @@ -726,17 +726,19 @@ tests/stub_internal.cc tests/stub_ipc.cc tests/stub_ipcache.cc \ tests/stub_libauth_acls.cc tests/stub_libauth.cc \ tests/stub_libeui.cc tests/stub_libformat.cc \ - tests/stub_libicmp.cc tests/stub_MemStore.cc mime.h \ - tests/stub_mime.cc tests/stub_neighbors.cc tests/stub_pconn.cc \ - tests/stub_Port.cc tests/stub_stat.cc \ - tests/stub_store_client.cc tests/stub_store_stats.cc \ - store_rebuild.h tests/stub_store_rebuild.cc \ - tests/stub_UdsOp.cc tests/testDiskIO.cc tests/testDiskIO.h \ + tests/stub_libicmp.cc tests/stub_liblog.cc \ + tests/stub_MemStore.cc mime.h tests/stub_mime.cc \ + tests/stub_neighbors.cc tests/stub_pconn.cc tests/stub_Port.cc \ + tests/stub_stat.cc tests/stub_store_client.cc \ + tests/stub_store_stats.cc store_rebuild.h \ + tests/stub_store_rebuild.cc tests/stub_UdsOp.cc \ + tests/testDiskIO.cc tests/testDiskIO.h \ tests/testStoreSupport.cc tests/testStoreSupport.h \ tests/stub_time.cc unlinkd.h unlinkd.cc url.cc win32.cc \ wordlist.h wordlist.cc tools.h tests/stub_tools.cc -am_tests_testDiskIO_OBJECTS = tests/stub_CacheDigest.$(OBJEXT) \ - cbdata.$(OBJEXT) tests/stub_CollapsedForwarding.$(OBJEXT) \ +am_tests_testDiskIO_OBJECTS = AccessLogEntry.$(OBJEXT) \ + tests/stub_CacheDigest.$(OBJEXT) cbdata.$(OBJEXT) \ + tests/stub_CollapsedForwarding.$(OBJEXT) \ ConfigOption.$(OBJEXT) ConfigParser.$(OBJEXT) $(am__objects_3) \ fs_io.$(OBJEXT) tests/stub_ETag.$(OBJEXT) EventLoop.$(OBJEXT) \ event.$(OBJEXT) tests/stub_fatal.$(OBJEXT) fd.$(OBJEXT) \ @@ -771,10 +773,11 @@ tests/stub_ipc.$(OBJEXT) tests/stub_ipcache.$(OBJEXT) \ tests/stub_libauth_acls.$(OBJEXT) tests/stub_libauth.$(OBJEXT) \ tests/stub_libeui.$(OBJEXT) tests/stub_libformat.$(OBJEXT) \ - tests/stub_libicmp.$(OBJEXT) tests/stub_MemStore.$(OBJEXT) \ - tests/stub_mime.$(OBJEXT) tests/stub_neighbors.$(OBJEXT) \ - tests/stub_pconn.$(OBJEXT) tests/stub_Port.$(OBJEXT) \ - tests/stub_stat.$(OBJEXT) tests/stub_store_client.$(OBJEXT) \ + tests/stub_libicmp.$(OBJEXT) tests/stub_liblog.$(OBJEXT) \ + tests/stub_MemStore.$(OBJEXT) tests/stub_mime.$(OBJEXT) \ + tests/stub_neighbors.$(OBJEXT) tests/stub_pconn.$(OBJEXT) \ + tests/stub_Port.$(OBJEXT) tests/stub_stat.$(OBJEXT) \ + tests/stub_store_client.$(OBJEXT) \ tests/stub_store_stats.$(OBJEXT) \ tests/stub_store_rebuild.$(OBJEXT) tests/stub_UdsOp.$(OBJEXT) \ tests/testDiskIO.$(OBJEXT) tests/testStoreSupport.$(OBJEXT) \ @@ -1337,14 +1340,14 @@ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ $(AM_CXXFLAGS) $(CXXFLAGS) $(tests_testLookupTable_LDFLAGS) \ $(LDFLAGS) -o $@ -am__tests_testRock_SOURCES_DIST = cbdata.cc CacheDigest.h \ - CollapsedForwarding.h CollapsedForwarding.cc \ - tests/stub_CacheDigest.cc ConfigOption.cc ConfigParser.cc \ - fs_io.h fs_io.cc ETag.cc EventLoop.cc event.cc fatal.h \ - fatal.cc fd.h fd.cc fde.h fde.cc FileMap.h filemap.cc \ - HttpHeaderFieldStat.h HttpBody.h HttpBody.cc HttpHdrCc.cc \ - HttpHdrContRange.cc HttpHdrRange.cc HttpHdrSc.cc \ - HttpHdrScTarget.cc HttpHeader.h HttpHeader.cc \ +am__tests_testRock_SOURCES_DIST = AccessLogEntry.cc AccessLogEntry.h \ + cbdata.cc CacheDigest.h CollapsedForwarding.h \ + CollapsedForwarding.cc tests/stub_CacheDigest.cc \ + ConfigOption.cc ConfigParser.cc fs_io.h fs_io.cc ETag.cc \ + EventLoop.cc event.cc fatal.h fatal.cc fd.h fd.cc fde.h fde.cc \ + FileMap.h filemap.cc HttpHeaderFieldStat.h HttpBody.h \ + HttpBody.cc HttpHdrCc.cc HttpHdrContRange.cc HttpHdrRange.cc \ + HttpHdrSc.cc HttpHdrScTarget.cc HttpHeader.h HttpHeader.cc \ HttpHeaderFieldInfo.h HttpHeaderTools.h HttpHeaderTools.cc \ HttpMsg.cc HttpReply.cc int.h int.cc SquidList.h SquidList.cc \ MasterXaction.cc MasterXaction.h MemBuf.cc MemObject.cc \ @@ -1370,10 +1373,11 @@ tests/stub_HttpRequest.cc tests/stub_libauth.cc \ tests/stub_icp.cc tests/stub_ipc.cc tests/stub_ipcache.cc \ tests/stub_libeui.cc tests/stub_libformat.cc \ - tests/stub_libicmp.cc tests/stub_libmgr.cc \ - tests/stub_libsecurity.cc tests/stub_MemStore.cc mime.h \ - tests/stub_mime.cc tests/stub_neighbors.cc tests/stub_Port.cc \ - tests/stub_pconn.cc tests/stub_store_client.cc store_rebuild.h \ + tests/stub_libicmp.cc tests/stub_liblog.cc \ + tests/stub_libmgr.cc tests/stub_libsecurity.cc \ + tests/stub_MemStore.cc mime.h tests/stub_mime.cc \ + tests/stub_neighbors.cc tests/stub_Port.cc tests/stub_pconn.cc \ + tests/stub_store_client.cc store_rebuild.h \ tests/stub_store_rebuild.cc tests/stub_store_stats.cc tools.h \ tests/stub_tools.cc time.cc url.cc wordlist.h wordlist.cc \ CommonPool.h CompositePoolNode.h delay_pools.cc DelayId.cc \ @@ -1383,7 +1387,7 @@ DelayTagged.h DelayUser.cc DelayUser.h DelayVector.cc \ DelayVector.h NullDelayId.cc NullDelayId.h \ ClientDelayConfig.cc ClientDelayConfig.h unlinkd.h unlinkd.cc -am_tests_testRock_OBJECTS = cbdata.$(OBJEXT) \ +am_tests_testRock_OBJECTS = AccessLogEntry.$(OBJEXT) cbdata.$(OBJEXT) \ CollapsedForwarding.$(OBJEXT) tests/stub_CacheDigest.$(OBJEXT) \ ConfigOption.$(OBJEXT) ConfigParser.$(OBJEXT) fs_io.$(OBJEXT) \ ETag.$(OBJEXT) EventLoop.$(OBJEXT) event.$(OBJEXT) \ @@ -1416,11 +1420,11 @@ tests/stub_libauth.$(OBJEXT) tests/stub_icp.$(OBJEXT) \ tests/stub_ipc.$(OBJEXT) tests/stub_ipcache.$(OBJEXT) \ tests/stub_libeui.$(OBJEXT) tests/stub_libformat.$(OBJEXT) \ - tests/stub_libicmp.$(OBJEXT) tests/stub_libmgr.$(OBJEXT) \ - tests/stub_libsecurity.$(OBJEXT) tests/stub_MemStore.$(OBJEXT) \ - tests/stub_mime.$(OBJEXT) tests/stub_neighbors.$(OBJEXT) \ - tests/stub_Port.$(OBJEXT) tests/stub_pconn.$(OBJEXT) \ - tests/stub_store_client.$(OBJEXT) \ + tests/stub_libicmp.$(OBJEXT) tests/stub_liblog.$(OBJEXT) \ + tests/stub_libmgr.$(OBJEXT) tests/stub_libsecurity.$(OBJEXT) \ + tests/stub_MemStore.$(OBJEXT) tests/stub_mime.$(OBJEXT) \ + tests/stub_neighbors.$(OBJEXT) tests/stub_Port.$(OBJEXT) \ + tests/stub_pconn.$(OBJEXT) tests/stub_store_client.$(OBJEXT) \ tests/stub_store_rebuild.$(OBJEXT) \ tests/stub_store_stats.$(OBJEXT) tests/stub_tools.$(OBJEXT) \ time.$(OBJEXT) url.$(OBJEXT) wordlist.$(OBJEXT) \ @@ -1774,34 +1778,35 @@ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ $(AM_CXXFLAGS) $(CXXFLAGS) $(tests_testURL_LDFLAGS) $(LDFLAGS) \ -o $@ -am__tests_testUfs_SOURCES_DIST = tests/testUfs.cc tests/testUfs.h \ - tests/stub_cache_manager.cc tests/stub_client_db.cc \ - tests/stub_CollapsedForwarding.cc \ +am__tests_testUfs_SOURCES_DIST = AccessLogEntry.cc AccessLogEntry.h \ + tests/testUfs.cc tests/testUfs.h tests/stub_cache_manager.cc \ + tests/stub_client_db.cc tests/stub_CollapsedForwarding.cc \ tests/stub_HelperChildConfig.cc tests/stub_icp.cc \ tests/stub_ipc.cc tests/stub_ipcache.cc tests/stub_libeui.cc \ - tests/stub_libicmp.cc tests/stub_MemStore.cc \ - tests/stub_neighbors.cc tests/stub_pconn.cc tests/stub_Port.cc \ - tests/stub_UdsOp.cc internal.h tests/stub_internal.cc \ - tests/stub_libformat.cc tests/stub_libsecurity.cc \ - tests/stub_stat.cc store_rebuild.h tests/stub_store_rebuild.cc \ - tests/stub_store_stats.cc fatal.h tests/stub_fatal.cc fd.h \ - fd.cc fde.h fde.cc client_db.h fs_io.h fs_io.cc FileMap.h \ - filemap.cc HttpBody.h HttpBody.cc HttpReply.cc int.h int.cc \ - RequestFlags.h RequestFlags.cc SquidList.h SquidList.cc \ - Transients.cc MasterXaction.cc MasterXaction.h MemObject.cc \ - Notes.h Notes.cc StoreSwapLogData.cc StoreIOState.cc \ - StoreMetaUnpacker.cc StoreMeta.cc StoreMeta.h StoreMetaMD5.cc \ - StoreMetaMD5.h StoreMetaSTD.cc StoreMetaSTD.h \ - StoreMetaSTDLFS.cc StoreMetaSTDLFS.h StoreMetaObjSize.h \ - StoreMetaURL.cc StoreMetaURL.h StoreMetaVary.cc \ - StoreMetaVary.h StoreFileSystem.cc store_io.cc \ - store_swapout.cc store_swapmeta.cc unlinkd.h unlinkd.cc \ - win32.cc event.cc CommonPool.h CompositePoolNode.h \ - delay_pools.cc DelayId.cc DelayId.h DelayIdComposite.h \ - DelayBucket.cc DelayBucket.h DelayConfig.cc DelayConfig.h \ - DelayPool.cc DelayPool.h DelayPools.h DelaySpec.cc DelaySpec.h \ - DelayTagged.cc DelayTagged.h DelayUser.cc DelayUser.h \ - DelayVector.cc DelayVector.h NullDelayId.cc NullDelayId.h \ + tests/stub_libicmp.cc tests/stub_liblog.cc \ + tests/stub_MemStore.cc tests/stub_neighbors.cc \ + tests/stub_pconn.cc tests/stub_Port.cc tests/stub_UdsOp.cc \ + internal.h tests/stub_internal.cc tests/stub_libformat.cc \ + tests/stub_libsecurity.cc tests/stub_stat.cc store_rebuild.h \ + tests/stub_store_rebuild.cc tests/stub_store_stats.cc fatal.h \ + tests/stub_fatal.cc fd.h fd.cc fde.h fde.cc client_db.h \ + fs_io.h fs_io.cc FileMap.h filemap.cc HttpBody.h HttpBody.cc \ + HttpReply.cc int.h int.cc RequestFlags.h RequestFlags.cc \ + SquidList.h SquidList.cc Transients.cc MasterXaction.cc \ + MasterXaction.h MemObject.cc Notes.h Notes.cc \ + StoreSwapLogData.cc StoreIOState.cc StoreMetaUnpacker.cc \ + StoreMeta.cc StoreMeta.h StoreMetaMD5.cc StoreMetaMD5.h \ + StoreMetaSTD.cc StoreMetaSTD.h StoreMetaSTDLFS.cc \ + StoreMetaSTDLFS.h StoreMetaObjSize.h StoreMetaURL.cc \ + StoreMetaURL.h StoreMetaVary.cc StoreMetaVary.h \ + StoreFileSystem.cc store_io.cc store_swapout.cc \ + store_swapmeta.cc unlinkd.h unlinkd.cc win32.cc event.cc \ + CommonPool.h CompositePoolNode.h delay_pools.cc DelayId.cc \ + DelayId.h DelayIdComposite.h DelayBucket.cc DelayBucket.h \ + DelayConfig.cc DelayConfig.h DelayPool.cc DelayPool.h \ + DelayPools.h DelaySpec.cc DelaySpec.h DelayTagged.cc \ + DelayTagged.h DelayUser.cc DelayUser.h DelayVector.cc \ + DelayVector.h NullDelayId.cc NullDelayId.h \ ClientDelayConfig.cc ClientDelayConfig.h CacheDigest.h \ tests/stub_CacheDigest.cc ConfigParser.cc EventLoop.cc \ HttpMsg.cc RemovalPolicy.cc repl_modules.h store.cc \ @@ -1822,17 +1827,18 @@ tests/stub_store_client.cc tools.h tests/stub_tools.cc \ tests/testStoreSupport.cc tests/testStoreSupport.h time.cc \ wordlist.h wordlist.cc -am_tests_testUfs_OBJECTS = tests/testUfs.$(OBJEXT) \ - tests/stub_cache_manager.$(OBJEXT) \ +am_tests_testUfs_OBJECTS = AccessLogEntry.$(OBJEXT) \ + tests/testUfs.$(OBJEXT) tests/stub_cache_manager.$(OBJEXT) \ tests/stub_client_db.$(OBJEXT) \ tests/stub_CollapsedForwarding.$(OBJEXT) \ tests/stub_HelperChildConfig.$(OBJEXT) \ tests/stub_icp.$(OBJEXT) tests/stub_ipc.$(OBJEXT) \ tests/stub_ipcache.$(OBJEXT) tests/stub_libeui.$(OBJEXT) \ - tests/stub_libicmp.$(OBJEXT) tests/stub_MemStore.$(OBJEXT) \ - tests/stub_neighbors.$(OBJEXT) tests/stub_pconn.$(OBJEXT) \ - tests/stub_Port.$(OBJEXT) tests/stub_UdsOp.$(OBJEXT) \ - tests/stub_internal.$(OBJEXT) tests/stub_libformat.$(OBJEXT) \ + tests/stub_libicmp.$(OBJEXT) tests/stub_liblog.$(OBJEXT) \ + tests/stub_MemStore.$(OBJEXT) tests/stub_neighbors.$(OBJEXT) \ + tests/stub_pconn.$(OBJEXT) tests/stub_Port.$(OBJEXT) \ + tests/stub_UdsOp.$(OBJEXT) tests/stub_internal.$(OBJEXT) \ + tests/stub_libformat.$(OBJEXT) \ tests/stub_libsecurity.$(OBJEXT) tests/stub_stat.$(OBJEXT) \ tests/stub_store_rebuild.$(OBJEXT) \ tests/stub_store_stats.$(OBJEXT) tests/stub_fatal.$(OBJEXT) \ @@ -2848,47 +2854,47 @@ CpuAffinityMap.cc CpuAffinityMap.h CpuAffinitySet.cc \ CpuAffinitySet.h debug.cc Debug.h defines.h \ $(DELAY_POOL_SOURCE) fs_io.h fs_io.cc dlink.h dlink.cc \ - $(DNSSOURCE) enums.h err_type.h err_detail_type.h errorpage.cc \ - errorpage.h ETag.cc ETag.h event.cc event.h EventLoop.h \ - EventLoop.cc external_acl.cc ExternalACL.h ExternalACLEntry.cc \ - ExternalACLEntry.h FadingCounter.h FadingCounter.cc fatal.h \ - fatal.cc fd.h fd.cc fde.cc fde.h FileMap.h filemap.cc \ - fqdncache.h fqdncache.cc FwdState.cc FwdState.h Generic.h \ - globals.h gopher.h gopher.cc helper.cc helper.h hier_code.h \ - HierarchyLogEntry.h $(HTCPSOURCE) HttpStateFlags.h http.cc \ - http.h HttpHeaderFieldStat.h HttpHdrCc.h HttpHdrCc.cc \ - HttpHdrCc.cci HttpHdrRange.cc HttpHdrSc.cc HttpHdrSc.h \ - HttpHdrScTarget.cc HttpHdrScTarget.h HttpHdrContRange.cc \ - HttpHdrContRange.h HttpHeaderStat.h HttpHeader.h HttpHeader.cc \ - HttpHeaderMask.h HttpHeaderRange.h HttpHeaderFieldInfo.h \ - HttpHeaderTools.h HttpHeaderTools.cc HttpBody.h HttpBody.cc \ - HttpControlMsg.cc HttpControlMsg.h HttpMsg.cc HttpMsg.h \ - HttpReply.cc HttpReply.h RequestFlags.h RequestFlags.cc \ - HttpRequest.cc HttpRequest.h ICP.h icp_opcode.h icp_v2.cc \ - icp_v3.cc int.h int.cc internal.h internal.cc $(IPC_SOURCE) \ - ipcache.cc ipcache.h $(LEAKFINDERSOURCE) SquidList.h \ - SquidList.cc LogTags.cc LogTags.h lookup_t.h main.cc \ - MasterXaction.cc MasterXaction.h mem_node.cc mem_node.h \ - MemBuf.cc MemObject.cc MemObject.h MessageSizes.h mime.h \ - mime.cc mime_header.h mime_header.cc multicast.h multicast.cc \ - neighbors.h neighbors.cc Notes.h Notes.cc Parsing.cc Parsing.h \ - $(XPROF_STATS_SOURCE) pconn.cc pconn.h PeerDigest.h \ - peer_digest.cc peer_proxy_negotiate_auth.h \ - peer_proxy_negotiate_auth.cc peer_select.cc peer_sourcehash.h \ - peer_sourcehash.cc peer_userhash.h peer_userhash.cc \ - PeerPoolMgr.h PeerPoolMgr.cc PeerSelectState.h PingData.h \ - Pipeline.cc Pipeline.h protos.h redirect.h redirect.cc \ - refresh.h refresh.cc RemovalPolicy.cc RemovalPolicy.h \ - send-announce.h send-announce.cc SBufStatsAction.h \ - SBufStatsAction.cc sbuf/StringConvert.h $(SNMP_SOURCE) \ - SquidMath.h SquidMath.cc SquidNew.cc IoStats.h stat.h stat.cc \ - StatCounters.h StatCounters.cc StatHist.h StatHist.cc \ - String.cc StrList.h StrList.cc stmem.cc stmem.h repl_modules.h \ - store.cc Store.h StoreFileSystem.cc StoreFileSystem.h \ - store_io.cc StoreIOBuffer.h StoreIOState.cc StoreIOState.h \ - store_client.cc StoreClient.h store_digest.h store_digest.cc \ - store_key_md5.h store_key_md5.cc store_log.h store_log.cc \ - store_rebuild.h store_rebuild.cc store_swapin.h \ + $(DNSSOURCE) Downloader.cc Downloader.h enums.h err_type.h \ + err_detail_type.h errorpage.cc errorpage.h ETag.cc ETag.h \ + event.cc event.h EventLoop.h EventLoop.cc external_acl.cc \ + ExternalACL.h ExternalACLEntry.cc ExternalACLEntry.h \ + FadingCounter.h FadingCounter.cc fatal.h fatal.cc fd.h fd.cc \ + fde.cc fde.h FileMap.h filemap.cc fqdncache.h fqdncache.cc \ + FwdState.cc FwdState.h Generic.h globals.h gopher.h gopher.cc \ + helper.cc helper.h hier_code.h HierarchyLogEntry.h \ + $(HTCPSOURCE) HttpStateFlags.h http.cc http.h \ + HttpHeaderFieldStat.h HttpHdrCc.h HttpHdrCc.cc HttpHdrCc.cci \ + HttpHdrRange.cc HttpHdrSc.cc HttpHdrSc.h HttpHdrScTarget.cc \ + HttpHdrScTarget.h HttpHdrContRange.cc HttpHdrContRange.h \ + HttpHeaderStat.h HttpHeader.h HttpHeader.cc HttpHeaderMask.h \ + HttpHeaderRange.h HttpHeaderFieldInfo.h HttpHeaderTools.h \ + HttpHeaderTools.cc HttpBody.h HttpBody.cc HttpControlMsg.cc \ + HttpControlMsg.h HttpMsg.cc HttpMsg.h HttpReply.cc HttpReply.h \ + RequestFlags.h RequestFlags.cc HttpRequest.cc HttpRequest.h \ + ICP.h icp_opcode.h icp_v2.cc icp_v3.cc int.h int.cc internal.h \ + internal.cc $(IPC_SOURCE) ipcache.cc ipcache.h \ + $(LEAKFINDERSOURCE) SquidList.h SquidList.cc LogTags.cc \ + LogTags.h lookup_t.h main.cc MasterXaction.cc MasterXaction.h \ + mem_node.cc mem_node.h MemBuf.cc MemObject.cc MemObject.h \ + MessageSizes.h mime.h mime.cc mime_header.h mime_header.cc \ + multicast.h multicast.cc neighbors.h neighbors.cc Notes.h \ + Notes.cc Parsing.cc Parsing.h $(XPROF_STATS_SOURCE) pconn.cc \ + pconn.h PeerDigest.h peer_digest.cc \ + peer_proxy_negotiate_auth.h peer_proxy_negotiate_auth.cc \ + peer_select.cc peer_sourcehash.h peer_sourcehash.cc \ + peer_userhash.h peer_userhash.cc PeerPoolMgr.h PeerPoolMgr.cc \ + PeerSelectState.h PingData.h Pipeline.cc Pipeline.h protos.h \ + redirect.h redirect.cc refresh.h refresh.cc RemovalPolicy.cc \ + RemovalPolicy.h send-announce.h send-announce.cc \ + SBufStatsAction.h SBufStatsAction.cc sbuf/StringConvert.h \ + $(SNMP_SOURCE) SquidMath.h SquidMath.cc SquidNew.cc IoStats.h \ + stat.h stat.cc StatCounters.h StatCounters.cc StatHist.h \ + StatHist.cc String.cc StrList.h StrList.cc stmem.cc stmem.h \ + repl_modules.h store.cc Store.h StoreFileSystem.cc \ + StoreFileSystem.h store_io.cc StoreIOBuffer.h StoreIOState.cc \ + StoreIOState.h store_client.cc StoreClient.h store_digest.h \ + store_digest.cc store_key_md5.h store_key_md5.cc store_log.h \ + store_log.cc store_rebuild.h store_rebuild.cc store_swapin.h \ store_swapin.cc store_swapmeta.cc store_swapout.cc \ StoreMetaUnpacker.cc StoreMetaUnpacker.h $(STOREMETA_SOURCE) \ StoreSearch.h StoreStats.cc StoreStats.h StoreSwapLogData.cc \ @@ -3097,6 +3103,7 @@ tests/stub_libeui.cc \ tests/stub_libformat.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_libmem.cc \ tests/stub_libmgr.cc \ tests/stub_libsecurity.cc \ @@ -3692,6 +3699,8 @@ tests_testCacheManager_LDFLAGS = $(LIBADD_DL) tests_testDiskIO_SOURCES = \ + AccessLogEntry.cc \ + AccessLogEntry.h \ CacheDigest.h \ tests/stub_CacheDigest.cc \ cbdata.cc \ @@ -3797,6 +3806,7 @@ tests/stub_libeui.cc \ tests/stub_libformat.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_MemStore.cc \ mime.h \ tests/stub_mime.cc \ @@ -3842,6 +3852,7 @@ fs/libfs.la \ ipc/libipc.la \ $(REPL_OBJS) \ + $(ADAPTATION_LIBS) \ DiskIO/libdiskio.la \ acl/libapi.la \ anyp/libanyp.la \ @@ -5169,6 +5180,8 @@ $(REPL_OBJS) tests_testUfs_SOURCES = \ + AccessLogEntry.cc \ + AccessLogEntry.h \ tests/testUfs.cc \ tests/testUfs.h \ tests/stub_cache_manager.cc \ @@ -5180,6 +5193,7 @@ tests/stub_ipcache.cc \ tests/stub_libeui.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_MemStore.cc \ tests/stub_neighbors.cc \ tests/stub_pconn.cc \ @@ -5327,6 +5341,7 @@ ip/libip.la \ mem/libmem.la \ store/libstore.la \ + $(ADAPTATION_LIBS) \ sbuf/libsbuf.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ @@ -5358,6 +5373,8 @@ $(XTRA_LIBS) tests_testRock_SOURCES = \ + AccessLogEntry.cc \ + AccessLogEntry.h \ cbdata.cc \ CacheDigest.h \ CollapsedForwarding.h \ @@ -5456,6 +5473,7 @@ tests/stub_libeui.cc \ tests/stub_libformat.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_libmgr.cc \ tests/stub_libsecurity.cc \ tests/stub_MemStore.cc \ @@ -5503,6 +5521,7 @@ base/libbase.la \ mem/libmem.la \ store/libstore.la \ + $(ADAPTATION_LIBS) \ sbuf/libsbuf.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ @@ -6318,6 +6337,8 @@ tests/$(DEPDIR)/$(am__dirstamp) tests/stub_libicmp.$(OBJEXT): tests/$(am__dirstamp) \ tests/$(DEPDIR)/$(am__dirstamp) +tests/stub_liblog.$(OBJEXT): tests/$(am__dirstamp) \ + tests/$(DEPDIR)/$(am__dirstamp) tests/stub_neighbors.$(OBJEXT): tests/$(am__dirstamp) \ tests/$(DEPDIR)/$(am__dirstamp) tests/stub_stat.$(OBJEXT): tests/$(am__dirstamp) \ @@ -6523,6 +6544,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DelayUser.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DelayVector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DescriptorSet.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Downloader.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ETag.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/EventLoop.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExternalACLEntry.Po@am__quote@ @@ -6717,6 +6739,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_libeui.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_libformat.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_libicmp.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_liblog.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_libmem.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_libmgr.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/stub_libsecurity.Po@am__quote@ diff -u -r -N squid-4.0.12/src/MemBuf.cc squid-4.0.13/src/MemBuf.cc --- squid-4.0.12/src/MemBuf.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/MemBuf.cc 2016-08-06 00:52:55.000000000 +1200 @@ -154,7 +154,7 @@ * Unfortunate hack to test if the buffer has been Init()ialized */ int -MemBuf::isNull() +MemBuf::isNull() const { if (!buf && !max_capacity && !capacity && !size) return 1; /* is null (not initialized) */ diff -u -r -N squid-4.0.12/src/MemBuf.h squid-4.0.13/src/MemBuf.h --- squid-4.0.12/src/MemBuf.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/MemBuf.h 2016-08-06 00:52:55.000000000 +1200 @@ -99,7 +99,7 @@ void reset(); /** unfirtunate hack to test if the buffer has been Init()ialized */ - int isNull(); + int isNull() const; /** * freezes the object! and returns function to clear it up. diff -u -r -N squid-4.0.12/src/MemStore.h squid-4.0.13/src/MemStore.h --- squid-4.0.12/src/MemStore.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/MemStore.h 2016-08-06 00:52:55.000000000 +1200 @@ -63,6 +63,7 @@ virtual bool updateCollapsed(StoreEntry &e) override; virtual void markForUnlink(StoreEntry &) override; virtual void unlink(StoreEntry &e) override; + virtual bool smpAware() const override { return true; } static int64_t EntryLimit(); diff -u -r -N squid-4.0.12/src/peer_digest.cc squid-4.0.13/src/peer_digest.cc --- squid-4.0.12/src/peer_digest.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/peer_digest.cc 2016-08-06 00:52:55.000000000 +1200 @@ -746,7 +746,7 @@ if (!reason && !size) { if (!pd->cd) reason = "null digest?!"; - else if (fetch->mask_offset != (int)pd->cd->mask_size) + else if (fetch->mask_offset != pd->cd->mask_size) reason = "premature end of digest?!"; else if (!peerDigestUseful(pd)) reason = "useless digest"; diff -u -r -N squid-4.0.12/src/PeerDigest.h squid-4.0.13/src/PeerDigest.h --- squid-4.0.12/src/PeerDigest.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/PeerDigest.h 2016-08-06 00:52:55.000000000 +1200 @@ -56,7 +56,7 @@ store_client *old_sc; HttpRequest *request; int offset; - int mask_offset; + uint32_t mask_offset; time_t start_time; time_t resp_time; time_t expires; diff -u -r -N squid-4.0.12/src/PeerPoolMgr.cc squid-4.0.13/src/PeerPoolMgr.cc --- squid-4.0.12/src/PeerPoolMgr.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/PeerPoolMgr.cc 2016-08-06 00:52:55.000000000 +1200 @@ -7,6 +7,7 @@ */ #include "squid.h" +#include "AccessLogEntry.h" #include "base/AsyncJobCalls.h" #include "base/RunnersRegistry.h" #include "CachePeer.h" @@ -20,25 +21,23 @@ #include "neighbors.h" #include "pconn.h" #include "PeerPoolMgr.h" +#include "security/BlindPeerConnector.h" #include "SquidConfig.h" #include "SquidTime.h" -#include "ssl/BlindPeerConnector.h" CBDATA_CLASS_INIT(PeerPoolMgr); -#if USE_OPENSSL -/// Gives Ssl::PeerConnector access to Answer in the PeerPoolMgr callback dialer. +/// Gives Security::PeerConnector access to Answer in the PeerPoolMgr callback dialer. class MyAnswerDialer: public UnaryMemFunT, - public Ssl::PeerConnector::CbDialer + public Security::PeerConnector::CbDialer { public: MyAnswerDialer(const JobPointer &aJob, Method aMethod): UnaryMemFunT(aJob, aMethod, Security::EncryptorAnswer()) {} - /* Ssl::PeerConnector::CbDialer API */ + /* Security::PeerConnector::CbDialer API */ virtual Security::EncryptorAnswer &answer() { return arg1; } }; -#endif PeerPoolMgr::PeerPoolMgr(CachePeer *aPeer): AsyncJob("PeerPoolMgr"), peer(cbdataReference(aPeer)), @@ -109,8 +108,7 @@ Must(params.conn != NULL); -#if USE_OPENSSL - // Handle SSL peers. + // Handle TLS peers. if (peer->secure.encryptTransport) { typedef CommCbMemFunT CloserDialer; closer = JobCallback(48, 3, CloserDialer, this, @@ -125,12 +123,10 @@ const int timeUsed = squid_curtime - params.conn->startTime(); // Use positive timeout when less than one second is left for conn. const int timeLeft = max(1, (peerTimeout - timeUsed)); - Ssl::BlindPeerConnector *connector = - new Ssl::BlindPeerConnector(request, params.conn, securer, NULL, timeLeft); + auto *connector = new Security::BlindPeerConnector(request, params.conn, securer, nullptr, timeLeft); AsyncJob::Start(connector); // will call our callback return; } -#endif pushNewConnection(params.conn); } diff -u -r -N squid-4.0.12/src/PeerPoolMgr.h squid-4.0.13/src/PeerPoolMgr.h --- squid-4.0.12/src/PeerPoolMgr.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/PeerPoolMgr.h 2016-08-06 00:52:55.000000000 +1200 @@ -51,7 +51,7 @@ /// Comm::ConnOpener calls this when done opening a connection for us void handleOpenedConnection(const CommConnectCbParams ¶ms); - /// Ssl::PeerConnector callback + /// Security::PeerConnector callback void handleSecuredPeer(Security::EncryptorAnswer &answer); /// called when the connection we are trying to secure is closed by a 3rd party diff -u -r -N squid-4.0.12/src/Pipeline.cc squid-4.0.13/src/Pipeline.cc --- squid-4.0.12/src/Pipeline.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/Pipeline.cc 2016-08-06 00:52:55.000000000 +1200 @@ -36,6 +36,18 @@ return requests.front(); } +Http::StreamPointer +Pipeline::back() const +{ + if (requests.empty()) { + debugs(33, 3, "Pipeline " << (void*)this << " empty"); + return Http::StreamPointer(); + } + + debugs(33, 3, "Pipeline " << (void*)this << " back " << requests.back()); + return requests.back(); +} + void Pipeline::terminateAll(int xerrno) { diff -u -r -N squid-4.0.12/src/Pipeline.h squid-4.0.13/src/Pipeline.h --- squid-4.0.12/src/Pipeline.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/Pipeline.h 2016-08-06 00:52:55.000000000 +1200 @@ -46,6 +46,9 @@ /// get the first request context in the pipeline Http::StreamPointer front() const; + /// get the last request context in the pipeline + Http::StreamPointer back() const; + /// how many requests are currently pipelined size_t count() const {return requests.size();} diff -u -r -N squid-4.0.12/src/redirect.cc squid-4.0.13/src/redirect.cc --- squid-4.0.12/src/redirect.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/redirect.cc 2016-08-06 00:52:55.000000000 +1200 @@ -94,25 +94,28 @@ // * trim all but the first word off the response. // * warn once every 50 responses that this will stop being fixed-up soon. // - if (const char * res = reply.other().content()) { + if (reply.other().hasContent()) { + const char * res = reply.other().content(); + size_t replySize = 0; if (const char *t = strchr(res, ' ')) { static int warn = 0; debugs(61, (!(warn++%50)? DBG_CRITICAL:2), "UPGRADE WARNING: URL rewriter reponded with garbage '" << t << "'. Future Squid will treat this as part of the URL."); - const mb_size_t garbageLength = reply.other().contentSize() - (t-res); - reply.modifiableOther().truncate(garbageLength); - } - if (reply.other().hasContent() && *res == '\0') - reply.modifiableOther().clean(); // drop the whole buffer of garbage. + replySize = t - res; + } else + replySize = reply.other().contentSize(); // if we still have anything in other() after all that // parse it into status=, url= and rewrite-url= keys - if (reply.other().hasContent()) { + if (replySize) { /* 2012-06-28: This cast is due to urlParse() truncating too-long URLs itself. * At this point altering the helper buffer in that way is not harmful, but annoying. * When Bug 1961 is resolved and urlParse has a const API, this needs to die. */ - char * result = reply.modifiableOther().content(); + MemBuf replyBuffer; + replyBuffer.init(replySize, replySize); + replyBuffer.append(reply.other().content(), reply.other().contentSize()); + char * result = replyBuffer.content(); Helper::Reply newReply; // BACKWARD COMPATIBILITY 2012-06-15: diff -u -r -N squid-4.0.12/src/security/BlindPeerConnector.cc squid-4.0.13/src/security/BlindPeerConnector.cc --- squid-4.0.12/src/security/BlindPeerConnector.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/security/BlindPeerConnector.cc 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,84 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#include "squid.h" +#include "CachePeer.h" +#include "comm/Connection.h" +#include "fde.h" +#include "HttpRequest.h" +#include "neighbors.h" +#include "security/BlindPeerConnector.h" +#include "security/NegotiationHistory.h" +#include "SquidConfig.h" + +CBDATA_NAMESPACED_CLASS_INIT(Security, BlindPeerConnector); + +Security::ContextPtr +Security::BlindPeerConnector::getSslContext() +{ + if (const CachePeer *peer = serverConnection()->getPeer()) { + assert(peer->secure.encryptTransport); + Security::ContextPtr sslContext(peer->sslContext); + return sslContext; + } + return ::Config.ssl_client.sslContext; +} + +bool +Security::BlindPeerConnector::initializeTls(Security::SessionPointer &serverSession) +{ + if (!Security::PeerConnector::initializeTls(serverSession)) + return false; + + if (const CachePeer *peer = serverConnection()->getPeer()) { + assert(peer); + + // NP: domain may be a raw-IP but it is now always set + assert(!peer->secure.sslDomain.isEmpty()); + +#if USE_OPENSSL + // const loss is okay here, ssl_ex_index_server is only read and not assigned a destructor + SBuf *host = new SBuf(peer->secure.sslDomain); + SSL_set_ex_data(serverSession.get(), ssl_ex_index_server, host); + + if (peer->sslSession) + SSL_set_session(serverSession.get(), peer->sslSession); + } else { + SBuf *hostName = new SBuf(request->url.host()); + SSL_set_ex_data(serverSession.get(), ssl_ex_index_server, (void*)hostName); +#endif + } + return true; +} + +void +Security::BlindPeerConnector::noteNegotiationDone(ErrorState *error) +{ + if (error) { + // XXX: forward.cc calls peerConnectSucceeded() after an OK TCP connect but + // we call peerConnectFailed() if SSL failed afterwards. Is that OK? + // It is not clear whether we should call peerConnectSucceeded/Failed() + // based on TCP results, SSL results, or both. And the code is probably not + // consistent in this aspect across tunnelling and forwarding modules. + if (CachePeer *p = serverConnection()->getPeer()) + peerConnectFailed(p); + return; + } + +#if USE_OPENSSL + const int fd = serverConnection()->fd; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + if (serverConnection()->getPeer() && !SSL_session_reused(ssl)) { + if (serverConnection()->getPeer()->sslSession) + SSL_SESSION_free(serverConnection()->getPeer()->sslSession); + + serverConnection()->getPeer()->sslSession = SSL_get1_session(ssl); + } +#endif +} + diff -u -r -N squid-4.0.12/src/security/BlindPeerConnector.h squid-4.0.13/src/security/BlindPeerConnector.h --- squid-4.0.12/src/security/BlindPeerConnector.h 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/security/BlindPeerConnector.h 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,50 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#ifndef SQUID_SRC_SSL_BLINDPEERCONNECTOR_H +#define SQUID_SRC_SSL_BLINDPEERCONNECTOR_H + +#include "security/PeerConnector.h" + +namespace Security +{ + +/// A simple PeerConnector for SSL/TLS cache_peers. No SslBump capabilities. +class BlindPeerConnector: public Security::PeerConnector { + CBDATA_CLASS(BlindPeerConnector); +public: + BlindPeerConnector(HttpRequestPointer &aRequest, + const Comm::ConnectionPointer &aServerConn, + AsyncCall::Pointer &aCallback, + const AccessLogEntryPointer &alp, + const time_t timeout = 0) : + AsyncJob("Security::BlindPeerConnector"), + Security::PeerConnector(aServerConn, aCallback, alp, timeout) + { + request = aRequest; + } + + /* Security::PeerConnector API */ + + /// Calls parent initializeTls(), configure the created TLS session object to + /// try reuse TLS session and sets the hostname to use for certificates validation + /// \returns true on successful initialization + virtual bool initializeTls(Security::SessionPointer &); + + /// Return the configured Security::ContextPtr object + virtual Security::ContextPtr getSslContext(); + + /// On error calls peerConnectFailed function, on success store the used SSL session + /// for later use + virtual void noteNegotiationDone(ErrorState *error); +}; + +} // namespace Security + +#endif /* SQUID_SRC_SSL_BLINDPEERCONNECTOR_H */ + diff -u -r -N squid-4.0.12/src/security/cert_generators/file/certificate_db.cc squid-4.0.13/src/security/cert_generators/file/certificate_db.cc --- squid-4.0.12/src/security/cert_generators/file/certificate_db.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/cert_generators/file/certificate_db.cc 2016-08-06 00:52:55.000000000 +1200 @@ -7,10 +7,12 @@ */ #include "squid.h" +#include "base/HardFun.h" #include "security/cert_generators/file/certificate_db.h" #include #include +#include #include #if HAVE_SYS_STAT_H #include @@ -252,7 +254,6 @@ db_full(aDb_path + "/" + db_file), cert_full(aDb_path + "/" + cert_dir), size_full(aDb_path + "/" + size_file), - db(NULL), max_db_size(aMax_db_size), fs_block_size((aFs_block_size ? aFs_block_size : 2048)), dbLock(db_full), @@ -287,12 +288,16 @@ load(); if (!db || !cert || !pkey) return false; + + // Functor to wrap xfree() for std::unique_ptr + typedef HardFun CharDeleter; + Row row; ASN1_INTEGER * ai = X509_get_serialNumber(cert.get()); std::string serial_string; Ssl::BIGNUM_Pointer serial(ASN1_INTEGER_to_BN(ai, NULL)); { - TidyPointer hex_bn(BN_bn2hex(serial.get())); + std::unique_ptr hex_bn(BN_bn2hex(serial.get())); serial_string = std::string(hex_bn.get()); } row.setValue(cnlSerial, serial_string.c_str()); @@ -305,13 +310,13 @@ } { - TidyPointer subject(X509_NAME_oneline(X509_get_subject_name(cert.get()), NULL, 0)); + std::unique_ptr subject(X509_NAME_oneline(X509_get_subject_name(cert.get()), nullptr, 0)); Security::CertPointer findCert; Ssl::EVP_PKEY_Pointer findPkey; if (pure_find(useName.empty() ? subject.get() : useName, findCert, findPkey)) { // Replace with database certificate - cert.reset(findCert.release()); - pkey.reset(findPkey.release()); + cert = std::move(findCert); + pkey = std::move(findPkey); return true; } // pure_find may fail because the entry is expired, or because the @@ -348,7 +353,7 @@ if (!useName.empty()) row.setValue(cnlName, useName.c_str()); else { - TidyPointer subject(X509_NAME_oneline(X509_get_subject_name(cert.get()), NULL, 0)); + std::unique_ptr subject(X509_NAME_oneline(X509_get_subject_name(cert.get()), nullptr, 0)); row.setValue(cnlName, subject.get()); } diff -u -r -N squid-4.0.12/src/security/cert_generators/file/security_file_certgen.cc squid-4.0.13/src/security/cert_generators/file/security_file_certgen.cc --- squid-4.0.12/src/security/cert_generators/file/security_file_certgen.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/cert_generators/file/security_file_certgen.cc 2016-08-06 00:52:55.000000000 +1200 @@ -204,8 +204,8 @@ if (!Ssl::certificateMatchesProperties(cert.get(), certProperties)) { // The certificate changed (renewed or other reason). // Generete a new one with the updated fields. - cert.reset(NULL); - pkey.reset(NULL); + cert.resetWithoutLocking(nullptr); + pkey.resetWithoutLocking(nullptr); db.purgeCert(cert_subject); } } diff -u -r -N squid-4.0.12/src/security/cert_validators/fake/security_fake_certverify.8 squid-4.0.13/src/security/cert_validators/fake/security_fake_certverify.8 --- squid-4.0.12/src/security/cert_validators/fake/security_fake_certverify.8 2016-07-02 02:25:00.000000000 +1200 +++ squid-4.0.13/src/security/cert_validators/fake/security_fake_certverify.8 2016-08-06 02:29:07.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "SECURITY_FAKE_CERTVERIFY 8" -.TH SECURITY_FAKE_CERTVERIFY 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH SECURITY_FAKE_CERTVERIFY 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/security/forward.h squid-4.0.13/src/security/forward.h --- squid-4.0.12/src/security/forward.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/forward.h 2016-08-06 00:52:55.000000000 +1200 @@ -19,6 +19,16 @@ #endif #include +#if USE_OPENSSL +// Macro to be used to define the C++ wrapper functor of the sk_*_pop_free +// OpenSSL family of functions. The C++ functor is suffixed with the _free_wrapper +// extension +#define sk_dtor_wrapper(sk_object, argument_type, freefunction) \ + struct sk_object ## _free_wrapper { \ + void operator()(argument_type a) { sk_object ## _pop_free(a, freefunction); } \ + } +#endif /* USE_OPENSSL */ + /* flags a SSL connection can be configured with */ #define SSL_FLAG_NO_DEFAULT_CA (1<<0) #define SSL_FLAG_DELAYED_AUTH (1<<1) @@ -32,10 +42,6 @@ namespace Security { -class EncryptorAnswer; -class PeerOptions; -class ServerOptions; - #if USE_OPENSSL CtoCpp1(X509_free, X509 *) typedef Security::LockingPointer CertPointer; @@ -56,6 +62,8 @@ typedef void *CrlPointer; #endif +typedef std::list CertList; + typedef std::list CertRevokeList; #if USE_OPENSSL @@ -65,7 +73,11 @@ typedef void *DhePointer; #endif +class EncryptorAnswer; class KeyData; +class PeerConnector; +class PeerOptions; +class ServerOptions; } // namespace Security diff -u -r -N squid-4.0.12/src/security/Handshake.cc squid-4.0.13/src/security/Handshake.cc --- squid-4.0.12/src/security/Handshake.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/Handshake.cc 2016-08-06 00:52:55.000000000 +1200 @@ -326,6 +326,7 @@ return; case HandshakeType::hskCertificate: Must(state < atCertificatesReceived); + parseServerCertificates(message.msg_body); state = atCertificatesReceived; return; case HandshakeType::hskServerHelloDone: @@ -534,13 +535,43 @@ return false; // unreached } +void +Security::HandshakeParser::ParseCertificate(const SBuf &raw, Security::CertPointer &pCert) +{ #if USE_OPENSSL + auto x509Start = reinterpret_cast(raw.rawContent()); + auto x509Pos = x509Start; + X509 *x509 = d2i_X509(nullptr, &x509Pos, raw.length()); + Must(x509); // successfully parsed + Must(x509Pos == x509Start + raw.length()); // no leftovers + pCert.resetAndLock(x509); +#endif +} + +void +Security::HandshakeParser::parseServerCertificates(const SBuf &raw) +{ + Parser::BinaryTokenizer tkList(raw); + const SBuf clist = tkList.pstring24("CertificateList"); + Must(tkList.atEnd()); // no leftovers after all certificates + + Parser::BinaryTokenizer tkItems(clist); + while (!tkItems.atEnd()) { + Security::CertPointer cert; + ParseCertificate(tkItems.pstring24("Certificate"), cert); + serverCertificates.push_back(cert); + debugs(83, 7, "parsed " << serverCertificates.size() << " certificates so far"); + } + +} /// A helper function to create a set of all supported TLS extensions static Security::Extensions Security::SupportedExtensions() { +#if USE_OPENSSL + // optimize lookup speed by reserving the number of values x3, approximately Security::Extensions extensions(64); @@ -624,15 +655,9 @@ #endif return extensions; // might be empty -} - #else -static -Security::Extensions -Security::SupportedExtensions() -{ return Extensions(); // no extensions are supported without OpenSSL -} #endif +} diff -u -r -N squid-4.0.12/src/security/Handshake.h squid-4.0.13/src/security/Handshake.h --- squid-4.0.12/src/security/Handshake.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/Handshake.h 2016-08-06 00:52:55.000000000 +1200 @@ -12,6 +12,7 @@ #include "anyp/ProtocolVersion.h" #include "base/YesNoNone.h" #include "parser/BinaryTokenizer.h" +#include "security/forward.h" #include @@ -68,6 +69,8 @@ TlsDetails::Pointer details; ///< TLS handshake meta info or nil. + Security::CertList serverCertificates; ///< parsed certificates chain + ParserState state; ///< current parsing state. bool resumingSession; ///< True if this is a resuming session @@ -97,6 +100,9 @@ void parseCiphers(const SBuf &raw); void parseV23Ciphers(const SBuf &raw); + void parseServerCertificates(const SBuf &raw); + static void ParseCertificate(const SBuf &raw, CertPointer &cert); + unsigned int currentContentType; ///< The current TLS/SSL record content type const char *done; ///< not nil if we got what we were looking for diff -u -r -N squid-4.0.12/src/security/LockingPointer.h squid-4.0.13/src/security/LockingPointer.h --- squid-4.0.12/src/security/LockingPointer.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/LockingPointer.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,8 +9,6 @@ #ifndef SQUID_SRC_SECURITY_LOCKINGPOINTER_H #define SQUID_SRC_SECURITY_LOCKINGPOINTER_H -#include "base/TidyPointer.h" - #if USE_OPENSSL #if HAVE_OPENSSL_CRYPTO_H #include @@ -24,7 +22,7 @@ sk_object ## _pop_free(a, freefunction); \ } -#endif +#endif /* USE_OPENSSL */ // Macro to be used to define the C++ equivalent function of an extern "C" // function. The C++ function suffixed with the _cpp extension @@ -37,50 +35,117 @@ { /** - * Add SSL locking (a.k.a. reference counting) and assignment to TidyPointer + * A shared pointer to a reference-counting Object with library-specific + * absorption, locking, and unlocking implementations. The API largely + * follows std::shared_ptr. + * + * The constructor and the resetWithoutLocking() method import a raw Object pointer. + * Normally, reset() would lock(), but libraries like OpenSSL + * pre-lock objects before they are fed to LockingPointer, necessitating + * this resetWithoutLocking() customization hook. */ -template -class LockingPointer: public TidyPointer +template +class LockingPointer { public: - typedef TidyPointer Parent; - typedef LockingPointer SelfType; + /// a helper label to simplify this objects API definitions below + typedef Security::LockingPointer SelfType; - explicit LockingPointer(T *t = nullptr): Parent(t) {} + /** + * Construct directly from a raw pointer. + * This action requires that the producer of that pointer has already + * created one reference lock for the object pointed to. + * Our destructor will do the matching unlock. + */ + explicit LockingPointer(T *t = nullptr): raw(nullptr) { + // de-optimized for clarity about non-locking + resetWithoutLocking(t); + } - explicit LockingPointer(const SelfType &o): Parent() { + /// use the custom UnLocker to unlock any value still stored. + ~LockingPointer() { unlock(); } + + // copy semantics are okay only when adding a lock reference + explicit LockingPointer(const SelfType &o) : raw(nullptr) { resetAndLock(o.get()); } - - SelfType &operator =(const SelfType & o) { + const SelfType &operator =(const SelfType &o) { resetAndLock(o.get()); return *this; } -#if __cplusplus >= 201103L - explicit LockingPointer(LockingPointer &&o): Parent(o.release()) { + // move semantics are definitely okay, when possible + explicit LockingPointer(SelfType &&) = default; + SelfType &operator =(SelfType &&o) { + if (o.get() != raw) + resetWithoutLocking(o.release()); + return *this; } - LockingPointer &operator =(LockingPointer &&o) { - if (o.get() != this->get()) - this->reset(o.release()); - return *this; + bool operator !() const { return !raw; } + explicit operator bool() const { return raw; } + + /// Returns raw and possibly nullptr pointer + T *get() const { return raw; } + + /// Reset raw pointer - unlock any previous one and save new one without locking. + void resetWithoutLocking(T *t) { + unlock(); + raw = t; } -#endif void resetAndLock(T *t) { - if (t != this->get()) { - this->reset(t); + if (t != get()) { + resetWithoutLocking(t); + lock(t); + } + } + + /// Forget the raw pointer - unlock if any value was set. Become a nil pointer. + void reset() { unlock(); } + + /// Forget the raw pointer without unlocking it. Become a nil pointer. + T *release() { + T *ret = raw; + raw = nullptr; + return ret; + } + +private: + /// The lock() method increments Object's reference counter. + void lock(T *t) { #if USE_OPENSSL - if (t) - CRYPTO_add(&t->references, 1, lock); + if (t) + CRYPTO_add(&t->references, 1, lockId); #elif USE_GNUTLS - // XXX: GnuTLS does not provide locking ? + // XXX: GnuTLS does not provide locking ? #else - assert(false); + assert(false); #endif + } + + /// Become a nil pointer. Decrements any pointed-to Object's reference counter + /// using UnLocker which ideally destroys the object when the counter reaches zero. + void unlock() { + if (raw) { + UnLocker(raw); + raw = nullptr; } } + + /** + * Normally, no other code will have this raw pointer. + * + * However, OpenSSL does some strange and not always consistent things. + * OpenSSL library may keep its own internal raw pointers and manage + * their reference counts independently, or it may not. This varies between + * API functions, though it is usually documented. + * + * This means the caller code needs to be carefuly written to use the correct + * reset method and avoid the raw-pointer constructor unless OpenSSL function + * producing the pointer is clearly documented as incrementing a lock for it. + */ + T *raw; }; } // namespace Security diff -u -r -N squid-4.0.12/src/security/Makefile.am squid-4.0.13/src/security/Makefile.am --- squid-4.0.12/src/security/Makefile.am 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/Makefile.am 2016-08-06 00:52:55.000000000 +1200 @@ -13,6 +13,8 @@ noinst_LTLIBRARIES = libsecurity.la libsecurity_la_SOURCES= \ + BlindPeerConnector.cc \ + BlindPeerConnector.h \ Context.h \ EncryptorAnswer.cc \ EncryptorAnswer.h \ @@ -23,6 +25,8 @@ LockingPointer.h \ NegotiationHistory.cc \ NegotiationHistory.h \ + PeerConnector.cc \ + PeerConnector.h \ PeerOptions.cc \ PeerOptions.h \ ServerOptions.cc \ diff -u -r -N squid-4.0.12/src/security/Makefile.in squid-4.0.13/src/security/Makefile.in --- squid-4.0.12/src/security/Makefile.in 2016-07-02 01:29:00.000000000 +1200 +++ squid-4.0.13/src/security/Makefile.in 2016-08-06 00:55:12.000000000 +1200 @@ -163,9 +163,9 @@ CONFIG_CLEAN_VPATH_FILES = LTLIBRARIES = $(noinst_LTLIBRARIES) libsecurity_la_LIBADD = -am_libsecurity_la_OBJECTS = EncryptorAnswer.lo Handshake.lo \ - NegotiationHistory.lo PeerOptions.lo ServerOptions.lo \ - Session.lo +am_libsecurity_la_OBJECTS = BlindPeerConnector.lo EncryptorAnswer.lo \ + Handshake.lo NegotiationHistory.lo PeerConnector.lo \ + PeerOptions.lo ServerOptions.lo Session.lo libsecurity_la_OBJECTS = $(am_libsecurity_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) @@ -758,6 +758,8 @@ SUBDIRS = cert_generators cert_validators noinst_LTLIBRARIES = libsecurity.la libsecurity_la_SOURCES = \ + BlindPeerConnector.cc \ + BlindPeerConnector.h \ Context.h \ EncryptorAnswer.cc \ EncryptorAnswer.h \ @@ -768,6 +770,8 @@ LockingPointer.h \ NegotiationHistory.cc \ NegotiationHistory.h \ + PeerConnector.cc \ + PeerConnector.h \ PeerOptions.cc \ PeerOptions.h \ ServerOptions.cc \ @@ -839,9 +843,11 @@ distclean-compile: -rm -f *.tab.c +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BlindPeerConnector.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/EncryptorAnswer.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Handshake.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NegotiationHistory.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerConnector.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerOptions.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerOptions.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Session.Plo@am__quote@ diff -u -r -N squid-4.0.12/src/security/PeerConnector.cc squid-4.0.13/src/security/PeerConnector.cc --- squid-4.0.12/src/security/PeerConnector.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/security/PeerConnector.cc 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,645 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +/* DEBUG: section 83 TLS Server/Peer negotiation */ + +#include "squid.h" +#include "acl/FilledChecklist.h" +#include "comm/Loops.h" +#include "Downloader.h" +#include "errorpage.h" +#include "fde.h" +#include "http/Stream.h" +#include "HttpRequest.h" +#include "security/NegotiationHistory.h" +#include "security/PeerConnector.h" +#include "SquidConfig.h" +#if USE_OPENSSL +#include "ssl/bio.h" +#include "ssl/cert_validate_message.h" +#include "ssl/Config.h" +#include "ssl/helper.h" +#endif + +CBDATA_NAMESPACED_CLASS_INIT(Security, PeerConnector); + +Security::PeerConnector::PeerConnector(const Comm::ConnectionPointer &aServerConn, AsyncCall::Pointer &aCallback, const AccessLogEntryPointer &alp, const time_t timeout) : + AsyncJob("Security::PeerConnector"), + serverConn(aServerConn), + al(alp), + callback(aCallback), + negotiationTimeout(timeout), + startTime(squid_curtime), + useCertValidator_(true), + certsDownloads(0) +{ + debugs(83, 5, "Security::PeerConnector constructed, this=" << (void*)this); + // if this throws, the caller's cb dialer is not our CbDialer + Must(dynamic_cast(callback->getDialer())); +} + +Security::PeerConnector::~PeerConnector() +{ + debugs(83, 5, "Security::PeerConnector destructed, this=" << (void*)this); +} + +bool Security::PeerConnector::doneAll() const +{ + return (!callback || callback->canceled()) && AsyncJob::doneAll(); +} + +/// Preps connection and SSL state. Calls negotiate(). +void +Security::PeerConnector::start() +{ + AsyncJob::start(); + + Security::SessionPointer tmp; + if (prepareSocket() && initializeTls(tmp)) + negotiateSsl(); + else + mustStop("Security::PeerConnector TLS socket initialize failed"); +} + +void +Security::PeerConnector::commCloseHandler(const CommCloseCbParams ¶ms) +{ + debugs(83, 5, "FD " << params.fd << ", Security::PeerConnector=" << params.data); + connectionClosed("Security::PeerConnector::commCloseHandler"); +} + +void +Security::PeerConnector::connectionClosed(const char *reason) +{ + mustStop(reason); + callback = NULL; +} + +bool +Security::PeerConnector::prepareSocket() +{ + const int fd = serverConnection()->fd; + if (!Comm::IsConnOpen(serverConn) || fd_table[serverConn->fd].closing()) { + connectionClosed("Security::PeerConnector::prepareSocket"); + return false; + } + + // watch for external connection closures + typedef CommCbMemFunT Dialer; + closeHandler = JobCallback(9, 5, Dialer, this, Security::PeerConnector::commCloseHandler); + comm_add_close_handler(fd, closeHandler); + return true; +} + +bool +Security::PeerConnector::initializeTls(Security::SessionPointer &serverSession) +{ +#if USE_OPENSSL + Security::ContextPtr sslContext(getSslContext()); + assert(sslContext); + + if (!Ssl::CreateClient(sslContext, serverConnection(), "server https start")) { + ErrorState *anErr = new ErrorState(ERR_SOCKET_FAILURE, Http::scInternalServerError, request.getRaw()); + anErr->xerrno = errno; + debugs(83, DBG_IMPORTANT, "Error allocating SSL handle: " << ERR_error_string(ERR_get_error(), NULL)); + noteNegotiationDone(anErr); + bail(anErr); + return false; + } + + // A TLS/SSL session has now been created for the connection and stored in fd_table + serverSession = fd_table[serverConnection()->fd].ssl; + + // If CertValidation Helper used do not lookup checklist for errors, + // but keep a list of errors to send it to CertValidator + if (!Ssl::TheConfig.ssl_crt_validator) { + // Create the ACL check list now, while we have access to more info. + // The list is used in ssl_verify_cb() and is freed in ssl_free(). + if (acl_access *acl = ::Config.ssl_client.cert_error) { + ACLFilledChecklist *check = new ACLFilledChecklist(acl, request.getRaw(), dash_str); + check->al = al; + // check->fd(fd); XXX: need client FD here + SSL_set_ex_data(serverSession.get(), ssl_ex_index_cert_error_check, check); + } + } + + return true; +#else + return false; +#endif +} + +void +Security::PeerConnector::setReadTimeout() +{ + int timeToRead; + if (negotiationTimeout) { + const int timeUsed = squid_curtime - startTime; + const int timeLeft = max(0, static_cast(negotiationTimeout - timeUsed)); + timeToRead = min(static_cast(::Config.Timeout.read), timeLeft); + } else + timeToRead = ::Config.Timeout.read; + AsyncCall::Pointer nil; + commSetConnTimeout(serverConnection(), timeToRead, nil); +} + +void +Security::PeerConnector::recordNegotiationDetails() +{ +#if USE_OPENSSL + const int fd = serverConnection()->fd; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + + // retrieve TLS server negotiated information if any + serverConnection()->tlsNegotiations()->retrieveNegotiatedInfo(ssl); + // retrieve TLS parsed extra info + BIO *b = SSL_get_rbio(ssl); + Ssl::ServerBio *bio = static_cast(b->ptr); + if (const Security::TlsDetails::Pointer &details = bio->receivedHelloDetails()) + serverConnection()->tlsNegotiations()->retrieveParsedInfo(details); +#endif +} + +void +Security::PeerConnector::negotiateSsl() +{ + if (!Comm::IsConnOpen(serverConnection()) || fd_table[serverConnection()->fd].closing()) + return; + +#if USE_OPENSSL + const int fd = serverConnection()->fd; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + const int result = SSL_connect(ssl); +#else + const int result = -1; +#endif + if (result <= 0) { + handleNegotiateError(result); + return; // we might be gone by now + } + + recordNegotiationDetails(); + + if (!sslFinalized()) + return; + + callBack(); +} + +bool +Security::PeerConnector::sslFinalized() +{ +#if USE_OPENSSL + if (Ssl::TheConfig.ssl_crt_validator && useCertValidator_) { + const int fd = serverConnection()->fd; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + + Ssl::CertValidationRequest validationRequest; + // WARNING: Currently we do not use any locking for any of the + // members of the Ssl::CertValidationRequest class. In this code the + // Ssl::CertValidationRequest object used only to pass data to + // Ssl::CertValidationHelper::submit method. + validationRequest.ssl = ssl; + validationRequest.domainName = request->url.host(); + if (Ssl::CertErrors *errs = static_cast(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors))) + // validationRequest disappears on return so no need to cbdataReference + validationRequest.errors = errs; + else + validationRequest.errors = NULL; + try { + debugs(83, 5, "Sending SSL certificate for validation to ssl_crtvd."); + AsyncCall::Pointer call = asyncCall(83,5, "Security::PeerConnector::sslCrtvdHandleReply", Ssl::CertValidationHelper::CbDialer(this, &Security::PeerConnector::sslCrtvdHandleReply, nullptr)); + Ssl::CertValidationHelper::GetInstance()->sslSubmit(validationRequest, call); + return false; + } catch (const std::exception &e) { + debugs(83, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtvd " << + "request for " << validationRequest.domainName << + " certificate: " << e.what() << "; will now block to " << + "validate that certificate."); + // fall through to do blocking in-process generation. + ErrorState *anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw()); + + noteNegotiationDone(anErr); + bail(anErr); + serverConn->close(); + return true; + } + } +#endif + + noteNegotiationDone(NULL); + return true; +} + +#if USE_OPENSSL +void +Security::PeerConnector::sslCrtvdHandleReply(Ssl::CertValidationResponse::Pointer validationResponse) +{ + Must(validationResponse != NULL); + + Ssl::ErrorDetail *errDetails = NULL; + bool validatorFailed = false; + if (!Comm::IsConnOpen(serverConnection())) { + return; + } + + debugs(83,5, request->url.host() << " cert validation result: " << validationResponse->resultCode); + + if (validationResponse->resultCode == ::Helper::Error) { + if (Ssl::CertErrors *errs = sslCrtvdCheckForErrors(*validationResponse, errDetails)) { + Security::SessionPtr ssl = fd_table[serverConnection()->fd].ssl.get(); + Ssl::CertErrors *oldErrs = static_cast(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors)); + SSL_set_ex_data(ssl, ssl_ex_index_ssl_errors, (void *)errs); + delete oldErrs; + } + } else if (validationResponse->resultCode != ::Helper::Okay) + validatorFailed = true; + + if (!errDetails && !validatorFailed) { + noteNegotiationDone(NULL); + callBack(); + return; + } + + ErrorState *anErr = NULL; + if (validatorFailed) { + anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw()); + } else { + anErr = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request.getRaw()); + anErr->detail = errDetails; + /*anErr->xerrno= Should preserved*/ + } + + noteNegotiationDone(anErr); + bail(anErr); + serverConn->close(); + return; +} +#endif + +#if USE_OPENSSL +/// Checks errors in the cert. validator response against sslproxy_cert_error. +/// The first honored error, if any, is returned via errDetails parameter. +/// The method returns all seen errors except SSL_ERROR_NONE as Ssl::CertErrors. +Ssl::CertErrors * +Security::PeerConnector::sslCrtvdCheckForErrors(Ssl::CertValidationResponse const &resp, Ssl::ErrorDetail *& errDetails) +{ + Ssl::CertErrors *errs = NULL; + + ACLFilledChecklist *check = NULL; + if (acl_access *acl = ::Config.ssl_client.cert_error) { + check = new ACLFilledChecklist(acl, request.getRaw(), dash_str); + check->al = al; + } + + Security::SessionPtr ssl = fd_table[serverConnection()->fd].ssl.get(); + typedef Ssl::CertValidationResponse::RecvdErrors::const_iterator SVCRECI; + for (SVCRECI i = resp.errors.begin(); i != resp.errors.end(); ++i) { + debugs(83, 7, "Error item: " << i->error_no << " " << i->error_reason); + + assert(i->error_no != SSL_ERROR_NONE); + + if (!errDetails) { + bool allowed = false; + if (check) { + check->sslErrors = new Ssl::CertErrors(Ssl::CertError(i->error_no, i->cert.get(), i->error_depth)); + if (check->fastCheck() == ACCESS_ALLOWED) + allowed = true; + } + // else the Config.ssl_client.cert_error access list is not defined + // and the first error will cause the error page + + if (allowed) { + debugs(83, 3, "bypassing SSL error " << i->error_no << " in " << "buffer"); + } else { + debugs(83, 5, "confirming SSL error " << i->error_no); + X509 *brokenCert = i->cert.get(); + Security::CertPointer peerCert(SSL_get_peer_certificate(ssl)); + const char *aReason = i->error_reason.empty() ? NULL : i->error_reason.c_str(); + errDetails = new Ssl::ErrorDetail(i->error_no, peerCert.get(), brokenCert, aReason); + } + if (check) { + delete check->sslErrors; + check->sslErrors = NULL; + } + } + + if (!errs) + errs = new Ssl::CertErrors(Ssl::CertError(i->error_no, i->cert.get(), i->error_depth)); + else + errs->push_back_unique(Ssl::CertError(i->error_no, i->cert.get(), i->error_depth)); + } + if (check) + delete check; + + return errs; +} +#endif + +/// A wrapper for Comm::SetSelect() notifications. +void +Security::PeerConnector::NegotiateSsl(int, void *data) +{ + PeerConnector *pc = static_cast(data); + // Use job calls to add done() checks and other job logic/protections. + CallJobHere(83, 7, pc, Security::PeerConnector, negotiateSsl); +} + +void +Security::PeerConnector::handleNegotiateError(const int ret) +{ +#if USE_OPENSSL + const int fd = serverConnection()->fd; + unsigned long ssl_lib_error = SSL_ERROR_NONE; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + const int ssl_error = SSL_get_error(ssl, ret); + + switch (ssl_error) { + case SSL_ERROR_WANT_READ: + noteWantRead(); + return; + + case SSL_ERROR_WANT_WRITE: + noteWantWrite(); + return; + + case SSL_ERROR_SSL: + case SSL_ERROR_SYSCALL: + ssl_lib_error = ERR_get_error(); + // proceed to the general error handling code + break; + default: + // no special error handling for all other errors + break; + } + + // Log connection details, if any + recordNegotiationDetails(); + noteSslNegotiationError(ret, ssl_error, ssl_lib_error); +#endif +} + +void +Security::PeerConnector::noteWantRead() +{ + const int fd = serverConnection()->fd; +#if USE_OPENSSL + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + BIO *b = SSL_get_rbio(ssl); + Ssl::ServerBio *srvBio = static_cast(b->ptr); + if (srvBio->holdRead()) { + if (srvBio->gotHello()) { + if (checkForMissingCertificates()) + return; // Wait to download certificates before proceed. + + srvBio->holdRead(false); + // schedule a negotiateSSl to allow openSSL parse received data + Security::PeerConnector::NegotiateSsl(fd, this); + return; + } else if (srvBio->gotHelloFailed()) { + srvBio->holdRead(false); + debugs(83, DBG_IMPORTANT, "Error parsing SSL Server Hello Message on FD " << fd); + // schedule a negotiateSSl to allow openSSL parse received data + Security::PeerConnector::NegotiateSsl(fd, this); + return; + } + } +#endif + setReadTimeout(); + Comm::SetSelect(fd, COMM_SELECT_READ, &NegotiateSsl, this, 0); +} + +void +Security::PeerConnector::noteWantWrite() +{ + const int fd = serverConnection()->fd; + Comm::SetSelect(fd, COMM_SELECT_WRITE, &NegotiateSsl, this, 0); + return; +} + +void +Security::PeerConnector::noteSslNegotiationError(const int ret, const int ssl_error, const int ssl_lib_error) +{ +#if USE_OPENSSL // not used unless OpenSSL enabled. +#if defined(EPROTO) + int sysErrNo = EPROTO; +#else + int sysErrNo = EACCES; +#endif + + // store/report errno when ssl_error is SSL_ERROR_SYSCALL, ssl_lib_error is 0, and ret is -1 + if (ssl_error == SSL_ERROR_SYSCALL && ret == -1 && ssl_lib_error == 0) + sysErrNo = errno; + + const int fd = serverConnection()->fd; + debugs(83, DBG_IMPORTANT, "Error negotiating SSL on FD " << fd << + ": " << ERR_error_string(ssl_lib_error, NULL) << " (" << + ssl_error << "/" << ret << "/" << errno << ")"); + + ErrorState *anErr = NULL; + if (request != NULL) + anErr = ErrorState::NewForwarding(ERR_SECURE_CONNECT_FAIL, request.getRaw()); + else + anErr = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, NULL); + anErr->xerrno = sysErrNo; + + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + Ssl::ErrorDetail *errFromFailure = (Ssl::ErrorDetail *)SSL_get_ex_data(ssl, ssl_ex_index_ssl_error_detail); + if (errFromFailure != NULL) { + // The errFromFailure is attached to the ssl object + // and will be released when ssl object destroyed. + // Copy errFromFailure to a new Ssl::ErrorDetail object + anErr->detail = new Ssl::ErrorDetail(*errFromFailure); + } else { + // server_cert can be NULL here + X509 *server_cert = SSL_get_peer_certificate(ssl); + anErr->detail = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, server_cert, NULL); + X509_free(server_cert); + } + + if (ssl_lib_error != SSL_ERROR_NONE) + anErr->detail->setLibError(ssl_lib_error); + + noteNegotiationDone(anErr); + bail(anErr); +#endif +} + +void +Security::PeerConnector::bail(ErrorState *error) +{ + Must(error); // or the recepient will not know there was a problem + Must(callback != NULL); + CbDialer *dialer = dynamic_cast(callback->getDialer()); + Must(dialer); + dialer->answer().error = error; + + callBack(); + // Our job is done. The callabck recepient will probably close the failed + // peer connection and try another peer or go direct (if possible). We + // can close the connection ourselves (our error notification would reach + // the recepient before the fd-closure notification), but we would rather + // minimize the number of fd-closure notifications and let the recepient + // manage the TCP state of the connection. +} + +void +Security::PeerConnector::callBack() +{ + AsyncCall::Pointer cb = callback; + // Do this now so that if we throw below, swanSong() assert that we _tried_ + // to call back holds. + callback = NULL; // this should make done() true + + // remove close handler + comm_remove_close_handler(serverConnection()->fd, closeHandler); + + CbDialer *dialer = dynamic_cast(cb->getDialer()); + Must(dialer); + dialer->answer().conn = serverConnection(); + ScheduleCallHere(cb); +} + +void +Security::PeerConnector::swanSong() +{ + // XXX: unregister fd-closure monitoring and CommSetSelect interest, if any + AsyncJob::swanSong(); + if (callback != NULL) { // paranoid: we have left the caller waiting + debugs(83, DBG_IMPORTANT, "BUG: Unexpected state while connecting to a cache_peer or origin server"); + ErrorState *anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw()); + bail(anErr); + assert(!callback); + return; + } +} + +const char * +Security::PeerConnector::status() const +{ + static MemBuf buf; + buf.reset(); + + // TODO: redesign AsyncJob::status() API to avoid this + // id and stop reason reporting duplication. + buf.append(" [", 2); + if (stopReason != NULL) { + buf.append("Stopped, reason:", 16); + buf.appendf("%s",stopReason); + } + if (serverConn != NULL) + buf.appendf(" FD %d", serverConn->fd); + buf.appendf(" %s%u]", id.Prefix, id.value); + buf.terminate(); + + return buf.content(); +} + +#if USE_OPENSSL +/// CallDialer to allow use Downloader objects within PeerConnector class. +class PeerConnectorCertDownloaderDialer: public Downloader::CbDialer +{ +public: + typedef void (Security::PeerConnector::*Method)(SBuf &object, int status); + + PeerConnectorCertDownloaderDialer(Method method, Security::PeerConnector *pc): + method_(method), + peerConnector_(pc) {} + + /* CallDialer API */ + virtual bool canDial(AsyncCall &call) { return peerConnector_.valid(); } + virtual void dial(AsyncCall &call) { ((&(*peerConnector_))->*method_)(object, status); } + Method method_; ///< The Security::PeerConnector method to dial + CbcPointer peerConnector_; ///< The Security::PeerConnector object +}; + +void +Security::PeerConnector::startCertDownloading(SBuf &url) +{ + AsyncCall::Pointer certCallback = asyncCall(81, 4, + "Security::PeerConnector::certDownloadingDone", + PeerConnectorCertDownloaderDialer(&Security::PeerConnector::certDownloadingDone, this)); + + const Downloader *csd = dynamic_cast(request->downloader.valid()); + Downloader *dl = new Downloader(url, certCallback, csd ? csd->nestedLevel() + 1 : 1); + AsyncJob::Start(dl); +} + +void +Security::PeerConnector::certDownloadingDone(SBuf &obj, int downloadStatus) +{ + ++certsDownloads; + debugs(81, 5, "Certificate downloading status: " << downloadStatus << " certificate size: " << obj.length()); + + // get ServerBio from SSL object + const int fd = serverConnection()->fd; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + BIO *b = SSL_get_rbio(ssl); + Ssl::ServerBio *srvBio = static_cast(b->ptr); + + // Parse Certificate. Assume that it is in DER format. + // According to RFC 4325: + // The server must provide a DER encoded certificate or a collection + // collection of certificates in a "certs-only" CMS message. + // The applications MUST accept DER encoded certificates and SHOULD + // be able to accept collection of certificates. + // TODO: support collection of certificates + const unsigned char *raw = (const unsigned char*)obj.rawContent(); + if (X509 *cert = d2i_X509(NULL, &raw, obj.length())) { + char buffer[1024]; + debugs(81, 5, "Retrieved certificate: " << X509_NAME_oneline(X509_get_subject_name(cert), buffer, 1024)); + const Security::CertList &certsList = srvBio->serverCertificatesIfAny(); + if (const char *issuerUri = Ssl::uriOfIssuerIfMissing(cert, certsList)) { + urlsOfMissingCerts.push(SBuf(issuerUri)); + } + Ssl::SSL_add_untrusted_cert(ssl, cert); + } + + // Check if there are URIs to download from and if yes start downloading + // the first in queue. + if (urlsOfMissingCerts.size() && certsDownloads <= MaxCertsDownloads) { + startCertDownloading(urlsOfMissingCerts.front()); + urlsOfMissingCerts.pop(); + return; + } + + srvBio->holdRead(false); + Security::PeerConnector::NegotiateSsl(serverConnection()->fd, this); +} + +bool +Security::PeerConnector::checkForMissingCertificates() +{ + // Check for nested SSL certificates downloads. For example when the + // certificate located in an SSL site which requires to download a + // a missing certificate (... from an SSL site which requires to ...). + + const Downloader *csd = request->downloader.get(); + if (csd && csd->nestedLevel() >= MaxNestedDownloads) + return false; + + const int fd = serverConnection()->fd; + Security::SessionPtr ssl = fd_table[fd].ssl.get(); + BIO *b = SSL_get_rbio(ssl); + Ssl::ServerBio *srvBio = static_cast(b->ptr); + const Security::CertList &certs = srvBio->serverCertificatesIfAny(); + + if (certs.size()) { + debugs(83, 5, "SSL server sent " << certs.size() << " certificates"); + Ssl::missingChainCertificatesUrls(urlsOfMissingCerts, certs); + if (urlsOfMissingCerts.size()) { + startCertDownloading(urlsOfMissingCerts.front()); + urlsOfMissingCerts.pop(); + return true; + } + } + + return false; +} +#endif //USE_OPENSSL + diff -u -r -N squid-4.0.12/src/security/PeerConnector.h squid-4.0.13/src/security/PeerConnector.h --- squid-4.0.12/src/security/PeerConnector.h 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/security/PeerConnector.h 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,214 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#ifndef SQUID_SRC_SSL_PEERCONNECTOR_H +#define SQUID_SRC_SSL_PEERCONNECTOR_H + +#include "acl/Acl.h" +#include "base/AsyncCbdataCalls.h" +#include "base/AsyncJob.h" +#include "CommCalls.h" +#include "security/EncryptorAnswer.h" +#include "security/forward.h" +#if USE_OPENSSL +#include "ssl/support.h" +#endif + +#include +#include + +class HttpRequest; +class ErrorState; +class AccessLogEntry; +typedef RefCount AccessLogEntryPointer; + +namespace Security +{ + +/** + * Connects Squid to SSL/TLS-capable peers or services. + * Contains common code and interfaces of various specialized PeerConnectors, + * including peer certificate validation code. + \par + * The caller receives a call back with Security::EncryptorAnswer. If answer.error + * is not nil, then there was an error and the SSL connection to the SSL peer + * was not fully established. The error object is suitable for error response + * generation. + \par + * The caller must monitor the connection for closure because this + * job will not inform the caller about such events. + \par + * PeerConnector class curently supports a form of SSL negotiation timeout, + * which accounted only when sets the read timeout from SSL peer. + * For a complete solution, the caller must monitor the overall connection + * establishment timeout and close the connection on timeouts. This is probably + * better than having dedicated (or none at all!) timeouts for peer selection, + * DNS lookup, TCP handshake, SSL handshake, etc. Some steps may have their + * own timeout, but not all steps should be forced to have theirs. + * XXX: tunnel.cc and probably other subsystems does not have an "overall + * connection establishment" timeout. We need to change their code so that they + * start monitoring earlier and close on timeouts. This change may need to be + * discussed on squid-dev. + \par + * This job never closes the connection, even on errors. If a 3rd-party + * closes the connection, this job simply quits without informing the caller. + */ +class PeerConnector: virtual public AsyncJob +{ + CBDATA_CLASS(PeerConnector); + +public: + /// Callback dialier API to allow PeerConnector to set the answer. + class CbDialer + { + public: + virtual ~CbDialer() {} + /// gives PeerConnector access to the in-dialer answer + virtual Security::EncryptorAnswer &answer() = 0; + }; + + typedef RefCount HttpRequestPointer; + +public: + PeerConnector(const Comm::ConnectionPointer &aServerConn, + AsyncCall::Pointer &aCallback, + const AccessLogEntryPointer &alp, + const time_t timeout = 0); + virtual ~PeerConnector(); + +protected: + // AsyncJob API + virtual void start(); + virtual bool doneAll() const; + virtual void swanSong(); + virtual const char *status() const; + + /// The comm_close callback handler. + void commCloseHandler(const CommCloseCbParams ¶ms); + + /// Inform us that the connection is closed. Does the required clean-up. + void connectionClosed(const char *reason); + + /// Sets up TCP socket-related notification callbacks if things go wrong. + /// If socket already closed return false, else install the comm_close + /// handler to monitor the socket. + bool prepareSocket(); + + /// Sets the read timeout to avoid getting stuck while reading from a + /// silent server + void setReadTimeout(); + + /// \returns true on successful TLS session initialization + virtual bool initializeTls(Security::SessionPointer &); + + /// Performs a single secure connection negotiation step. + /// It is called multiple times untill the negotiation finish or aborted. + void negotiateSsl(); + + /// Called after SSL negotiations have finished. Cleans up SSL state. + /// Returns false if we are now waiting for the certs validation job. + /// Otherwise, returns true, regardless of negotiation success/failure. + bool sslFinalized(); + + /// Called when the SSL negotiation step aborted because data needs to + /// be transferred to/from SSL server or on error. In the first case + /// setups the appropriate Comm::SetSelect handler. In second case + /// fill an error and report to the PeerConnector caller. + void handleNegotiateError(const int result); + + /// Called when the openSSL SSL_connect fnction request more data from + /// the remote SSL server. Sets the read timeout and sets the + /// Squid COMM_SELECT_READ handler. + void noteWantRead(); + +#if USE_OPENSSL + /// Run the certificates list sent by the SSL server and check if there + /// are missing certificates. Adds to the urlOfMissingCerts list the + /// URLS of missing certificates if this information provided by the + /// issued certificates with Authority Info Access extension. + bool checkForMissingCertificates(); + + /// Start downloading procedure for the given URL. + void startCertDownloading(SBuf &url); + + /// Called by Downloader after a certificate object downloaded. + void certDownloadingDone(SBuf &object, int status); +#endif + + /// Called when the openSSL SSL_connect function needs to write data to + /// the remote SSL server. Sets the Squid COMM_SELECT_WRITE handler. + virtual void noteWantWrite(); + + /// Called when the SSL_connect function aborts with an SSL negotiation error + /// \param result the SSL_connect return code + /// \param ssl_error the error code returned from the SSL_get_error function + /// \param ssl_lib_error the error returned from the ERR_Get_Error function + virtual void noteSslNegotiationError(const int result, const int ssl_error, const int ssl_lib_error); + + /// Called when the SSL negotiation to the server completed and the certificates + /// validated using the cert validator. + /// \param error if not NULL the SSL negotiation was aborted with an error + virtual void noteNegotiationDone(ErrorState *error) {} + + /// Must implemented by the kid classes to return the Security::ContextPtr object to use + /// for building the SSL objects. + virtual Security::ContextPtr getSslContext() = 0; + + /// mimics FwdState to minimize changes to FwdState::initiate/negotiateSsl + Comm::ConnectionPointer const &serverConnection() const { return serverConn; } + + void bail(ErrorState *error); ///< Return an error to the PeerConnector caller + + /// Callback the caller class, and pass the ready to communicate secure + /// connection or an error if PeerConnector failed. + void callBack(); + + /// If called the certificates validator will not used + void bypassCertValidator() {useCertValidator_ = false;} + + /// Called after negotiation finishes to record connection details for + /// logging + void recordNegotiationDetails(); + + HttpRequestPointer request; ///< peer connection trigger or cause + Comm::ConnectionPointer serverConn; ///< TCP connection to the peer + AccessLogEntryPointer al; ///< info for the future access.log entry + AsyncCall::Pointer callback; ///< we call this with the results +private: + PeerConnector(const PeerConnector &); // not implemented + PeerConnector &operator =(const PeerConnector &); // not implemented + +#if USE_OPENSSL + /// Process response from cert validator helper + void sslCrtvdHandleReply(Ssl::CertValidationResponsePointer); + + /// Check SSL errors returned from cert validator against sslproxy_cert_error access list + Ssl::CertErrors *sslCrtvdCheckForErrors(Ssl::CertValidationResponse const &, Ssl::ErrorDetail *&); +#endif + + /// A wrapper function for negotiateSsl for use with Comm::SetSelect + static void NegotiateSsl(int fd, void *data); + + /// The maximum allowed missing certificates downloads. + static const unsigned int MaxCertsDownloads = 10; + /// The maximum allowed nested certificates downloads. + static const unsigned int MaxNestedDownloads = 3; + + AsyncCall::Pointer closeHandler; ///< we call this when the connection closed + time_t negotiationTimeout; ///< the SSL connection timeout to use + time_t startTime; ///< when the peer connector negotiation started + bool useCertValidator_; ///< whether the certificate validator should bypassed + /// The list of URLs where missing certificates should be downloaded. + std::queue urlsOfMissingCerts; + unsigned int certsDownloads; ///< the number of downloaded missing certificates +}; + +} // namespace Security + +#endif /* SQUID_SRC_SSL_PEERCONNECTOR_H */ + diff -u -r -N squid-4.0.12/src/security/ServerOptions.cc squid-4.0.13/src/security/ServerOptions.cc --- squid-4.0.12/src/security/ServerOptions.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/ServerOptions.cc 2016-08-06 00:52:55.000000000 +1200 @@ -117,19 +117,21 @@ return t; } -Security::ContextPtr +bool Security::ServerOptions::createStaticServerContext(AnyP::PortCfg &port) { updateTlsVersionLimits(); - Security::ContextPtr t = createBlankContext(); + Security::ContextPointer t(createBlankContext()); if (t) { #if USE_OPENSSL - Ssl::InitServerContext(t, port); + if (!Ssl::InitServerContext(t, port)) + return false; #endif } - return t; + staticContext = std::move(t); + return bool(staticContext); } void @@ -159,7 +161,7 @@ } } - parsedDhParams.reset(dhp); + parsedDhParams.resetWithoutLocking(dhp); #endif } diff -u -r -N squid-4.0.12/src/security/ServerOptions.h squid-4.0.13/src/security/ServerOptions.h --- squid-4.0.12/src/security/ServerOptions.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/ServerOptions.h 2016-08-06 00:52:55.000000000 +1200 @@ -33,7 +33,9 @@ virtual void dumpCfg(Packable *, const char *pfx) const; /// generate a security server-context from these configured options - Security::ContextPtr createStaticServerContext(AnyP::PortCfg &); + /// the resulting context is stored in staticContext + /// \returns true if a context could be created + bool createStaticServerContext(AnyP::PortCfg &); /// update the context with DH, EDH, EECDH settings void updateContextEecdh(Security::ContextPtr &); diff -u -r -N squid-4.0.12/src/security/Session.cc squid-4.0.13/src/security/Session.cc --- squid-4.0.12/src/security/Session.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/Session.cc 2016-08-06 00:52:55.000000000 +1200 @@ -16,45 +16,6 @@ #define SSL_SESSION_ID_SIZE 32 #define SSL_SESSION_MAX_SIZE 10*1024 -bool -Security::SessionIsResumed(const Security::SessionPointer &s) -{ - return -#if USE_OPENSSL - SSL_session_reused(s.get()) == 1; -#elif USE_GNUTLS - gnutls_session_is_resumed(s.get()) != 0; -#else - false; -#endif -} - -void -Security::GetSessionResumeData(const Security::SessionPointer &s, Security::SessionStatePointer &data) -{ - if (!SessionIsResumed(s)) { -#if USE_OPENSSL - data.reset(SSL_get1_session(s.get())); -#elif USE_GNUTLS - gnutls_datum_t *tmp = nullptr; - (void)gnutls_session_get_data2(s.get(), tmp); - data.reset(tmp); -#endif - } -} - -void -Security::SetSessionResumeData(const Security::SessionPtr &s, const Security::SessionStatePointer &data) -{ - if (s) { -#if USE_OPENSSL - (void)SSL_set_session(s, data.get()); -#elif USE_GNUTLS - (void)gnutls_session_set_data(s, data->data, data->size); -#endif - } -} - static bool isTlsServer() { diff -u -r -N squid-4.0.12/src/security/Session.h squid-4.0.13/src/security/Session.h --- squid-4.0.12/src/security/Session.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/security/Session.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,9 +9,10 @@ #ifndef SQUID_SRC_SECURITY_SESSION_H #define SQUID_SRC_SECURITY_SESSION_H -// LockingPointer.h instead of TidyPointer.h for CtoCpp1() #include "security/LockingPointer.h" +#include + #if USE_OPENSSL #if HAVE_OPENSSL_SSL_H #include @@ -31,41 +32,22 @@ CtoCpp1(SSL_free, SSL *); typedef LockingPointer SessionPointer; -typedef SSL_SESSION* SessionStatePtr; -CtoCpp1(SSL_SESSION_free, SSL_SESSION *); -typedef LockingPointer SessionStatePointer; - #elif USE_GNUTLS typedef gnutls_session_t SessionPtr; -CtoCpp1(gnutls_deinit, gnutls_session_t); -// TODO: Convert to Locking pointer. // Locks can be implemented attaching locks counter to gnutls_session_t // objects using the gnutls_session_set_ptr()/gnutls_session_get_ptr () // library functions -typedef TidyPointer SessionPointer; - -typedef gnutls_datum_t *SessionStatePtr; -CtoCpp1(gnutls_free, gnutls_datum_t *); -typedef TidyPointer SessionStatePointer; +CtoCpp1(gnutls_deinit, gnutls_session_t); +typedef LockingPointer SessionPointer; #else // use void* so we can check against NULL typedef void* SessionPtr; -typedef TidyPointer SessionPointer; -typedef TidyPointer SessionStatePointer; +CtoCpp1(xfree, SessionPtr); +typedef LockingPointer SessionPointer; #endif -/// whether the session is a resumed one -bool SessionIsResumed(const Security::SessionPointer &); - -/// Retrieve the data needed to resume this session on a later connection -void GetSessionResumeData(const Security::SessionPointer &, Security::SessionStatePointer &); - -/// Set the data for resuming a previous session. -/// Needs to be done before using the SessionPointer for a handshake. -void SetSessionResumeData(const Security::SessionPtr &, const Security::SessionStatePointer &); - } // namespace Security #endif /* SQUID_SRC_SECURITY_SESSION_H */ diff -u -r -N squid-4.0.12/src/servers/FtpServer.cc squid-4.0.13/src/servers/FtpServer.cc --- squid-4.0.12/src/servers/FtpServer.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/servers/FtpServer.cc 2016-08-06 00:52:55.000000000 +1200 @@ -759,11 +759,15 @@ Http::Stream *const result = new Http::Stream(clientConnection, http); + StoreIOBuffer tempBuffer; + tempBuffer.data = result->reqbuf; + tempBuffer.length = HTTP_REQBUF_SZ; + ClientStreamData newServer = new clientReplyContext(http); ClientStreamData newClient = result; clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, clientReplyStatus, newServer, clientSocketRecipient, - clientSocketDetach, newClient, result->getClientStreamBuffer()); + clientSocketDetach, newClient, tempBuffer); result->flags.parsed_ok = 1; return result; diff -u -r -N squid-4.0.12/src/servers/Server.cc squid-4.0.13/src/servers/Server.cc --- squid-4.0.12/src/servers/Server.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/servers/Server.cc 2016-08-06 00:52:55.000000000 +1200 @@ -167,6 +167,7 @@ // case Comm::COMM_ERROR: default: // no other flags should ever occur debugs(33, 2, io.conn << ": got flag " << rd.flag << "; " << xstrerr(rd.xerrno)); + checkLogging(); pipeline.terminateAll(rd.xerrno); io.conn->close(); return; diff -u -r -N squid-4.0.12/src/servers/Server.h squid-4.0.13/src/servers/Server.h --- squid-4.0.12/src/servers/Server.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/servers/Server.h 2016-08-06 00:52:55.000000000 +1200 @@ -117,6 +117,9 @@ void doClientRead(const CommIoCbParams &io); void clientWriteDone(const CommIoCbParams &io); + /// Log the current [attempt at] transaction if nobody else will. + virtual void checkLogging() = 0; + AsyncCall::Pointer reader; ///< set when we are reading AsyncCall::Pointer writer; ///< set when we are writing }; diff -u -r -N squid-4.0.12/src/ssl/bio.cc squid-4.0.13/src/ssl/bio.cc --- squid-4.0.12/src/ssl/bio.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/bio.cc 2016-08-06 00:52:55.000000000 +1200 @@ -203,8 +203,10 @@ allowSplice(false), allowBump(false), holdWrite_(false), + holdRead_(true), record_(false), parsedHandshake(false), + parseError(false), bumpMode_(bumpNone), rbufConsumePos(0) { @@ -271,6 +273,13 @@ catch (const std::exception &ex) { debugs(83, 2, "parsing error on FD " << fd_ << ": " << ex.what()); parsedHandshake = true; // done parsing (due to an error) + parseError = true; + } + + if (holdRead_) { + debugs(83, 7, "Hold flag is set, retry latter. (Hold " << size << "bytes)"); + BIO_set_retry_read(table); + return -1; } return giveBuffered(buf, size); diff -u -r -N squid-4.0.12/src/ssl/bio.h squid-4.0.13/src/ssl/bio.h --- squid-4.0.12/src/ssl/bio.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/bio.h 2016-08-06 00:52:55.000000000 +1200 @@ -138,6 +138,10 @@ bool holdWrite() const {return holdWrite_;} /// Enables or disables the write hold state void holdWrite(bool h) {holdWrite_ = h;} + /// The read hold state + bool holdRead() const {return holdRead_;} + /// Enables or disables the read hold state + void holdRead(bool h) {holdRead_ = h;} /// Enables or disables the input data recording, for internal analysis. void recordInput(bool r) {record_ = r;} /// Whether we can splice or not the SSL stream @@ -148,6 +152,15 @@ void mode(Ssl::BumpMode m) {bumpMode_ = m;} Ssl::BumpMode bumpMode() {return bumpMode_;} ///< return the bumping mode + /// \retval true if the Server hello message received + bool gotHello() const { return (parsedHandshake && !parseError); } + + /// Return true if the Server Hello parsing failed + bool gotHelloFailed() const { return (parsedHandshake && parseError); } + + /// \return the server certificates list if received and parsed correctly + const Security::CertList &serverCertificatesIfAny() { return parser_.serverCertificates; } + /// \return the TLS Details advertised by TLS server. const Security::TlsDetails::Pointer &receivedHelloDetails() const {return parser_.details;} @@ -167,8 +180,10 @@ bool allowSplice; ///< True if the SSL stream can be spliced bool allowBump; ///< True if the SSL stream can be bumped bool holdWrite_; ///< The write hold state of the bio. + bool holdRead_; ///< The read hold state of the bio. bool record_; ///< If true the input data recorded to rbuf for internal use bool parsedHandshake; ///< whether we are done parsing TLS Hello + bool parseError; ///< error while parsing server hello message Ssl::BumpMode bumpMode_; /// The size of data stored in rbuf which passed to the openSSL diff -u -r -N squid-4.0.12/src/ssl/BlindPeerConnector.cc squid-4.0.13/src/ssl/BlindPeerConnector.cc --- squid-4.0.12/src/ssl/BlindPeerConnector.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/BlindPeerConnector.cc 1970-01-01 12:00:00.000000000 +1200 @@ -1,77 +0,0 @@ -/* - * Copyright (C) 1996-2016 The Squid Software Foundation and contributors - * - * Squid software is distributed under GPLv2+ license and includes - * contributions from numerous individuals and organizations. - * Please see the COPYING and CONTRIBUTORS files for details. - */ - -#include "squid.h" -#include "CachePeer.h" -#include "comm/Connection.h" -#include "fde.h" -#include "HttpRequest.h" -#include "neighbors.h" -#include "security/NegotiationHistory.h" -#include "SquidConfig.h" -#include "ssl/BlindPeerConnector.h" - -CBDATA_NAMESPACED_CLASS_INIT(Ssl, BlindPeerConnector); - -Security::ContextPtr -Ssl::BlindPeerConnector::getSslContext() -{ - if (const CachePeer *peer = serverConnection()->getPeer()) { - assert(peer->secure.encryptTransport); - Security::ContextPtr sslContext(peer->sslContext); - return sslContext; - } - return ::Config.ssl_client.sslContext; -} - -Security::SessionPtr -Ssl::BlindPeerConnector::initializeSsl() -{ - auto ssl = Ssl::PeerConnector::initializeSsl(); - if (!ssl) - return nullptr; - - if (const CachePeer *peer = serverConnection()->getPeer()) { - assert(peer); - - // NP: domain may be a raw-IP but it is now always set - assert(!peer->secure.sslDomain.isEmpty()); - - // const loss is okay here, ssl_ex_index_server is only read and not assigned a destructor - SBuf *host = new SBuf(peer->secure.sslDomain); - SSL_set_ex_data(ssl, ssl_ex_index_server, host); - - Security::SetSessionResumeData(ssl, peer->sslSession); - } else { - SBuf *hostName = new SBuf(request->url.host()); - SSL_set_ex_data(ssl, ssl_ex_index_server, (void*)hostName); - } - - return ssl; -} - -void -Ssl::BlindPeerConnector::noteNegotiationDone(ErrorState *error) -{ - if (error) { - // XXX: forward.cc calls peerConnectSucceeded() after an OK TCP connect but - // we call peerConnectFailed() if SSL failed afterwards. Is that OK? - // It is not clear whether we should call peerConnectSucceeded/Failed() - // based on TCP results, SSL results, or both. And the code is probably not - // consistent in this aspect across tunnelling and forwarding modules. - if (CachePeer *p = serverConnection()->getPeer()) - peerConnectFailed(p); - return; - } - - if (auto *peer = serverConnection()->getPeer()) { - const int fd = serverConnection()->fd; - Security::GetSessionResumeData(fd_table[fd].ssl, peer->sslSession); - } -} - diff -u -r -N squid-4.0.12/src/ssl/BlindPeerConnector.h squid-4.0.13/src/ssl/BlindPeerConnector.h --- squid-4.0.12/src/ssl/BlindPeerConnector.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/BlindPeerConnector.h 1970-01-01 12:00:00.000000000 +1200 @@ -1,52 +0,0 @@ -/* - * Copyright (C) 1996-2016 The Squid Software Foundation and contributors - * - * Squid software is distributed under GPLv2+ license and includes - * contributions from numerous individuals and organizations. - * Please see the COPYING and CONTRIBUTORS files for details. - */ - -#ifndef SQUID_SRC_SSL_BLINDPEERCONNECTOR_H -#define SQUID_SRC_SSL_BLINDPEERCONNECTOR_H - -#include "ssl/PeerConnector.h" - -#if USE_OPENSSL - -namespace Ssl -{ - -/// A simple PeerConnector for SSL/TLS cache_peers. No SslBump capabilities. -class BlindPeerConnector: public PeerConnector { - CBDATA_CLASS(BlindPeerConnector); -public: - BlindPeerConnector(HttpRequestPointer &aRequest, - const Comm::ConnectionPointer &aServerConn, - AsyncCall::Pointer &aCallback, - const AccessLogEntryPointer &alp, - const time_t timeout = 0) : - AsyncJob("Ssl::BlindPeerConnector"), - PeerConnector(aServerConn, aCallback, alp, timeout) - { - request = aRequest; - } - - /* PeerConnector API */ - - /// Calls parent initializeSSL, configure the created SSL object to try reuse SSL session - /// and sets the hostname to use for certificates validation - virtual Security::SessionPtr initializeSsl(); - - /// Return the configured Security::ContextPtr object - virtual Security::ContextPtr getSslContext(); - - /// On error calls peerConnectFailed function, on success store the used SSL session - /// for later use - virtual void noteNegotiationDone(ErrorState *error); -}; - -} // namespace Ssl - -#endif /* USE_OPENSSL */ -#endif /* SQUID_SRC_SSL_BLINDPEERCONNECTOR_H */ - diff -u -r -N squid-4.0.12/src/ssl/gadgets.cc squid-4.0.13/src/ssl/gadgets.cc --- squid-4.0.12/src/ssl/gadgets.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/gadgets.cc 2016-08-06 00:52:55.000000000 +1200 @@ -130,12 +130,12 @@ BIO_puts(bio.get(), bufferToRead); X509 * certPtr = NULL; - cert.reset(PEM_read_bio_X509(bio.get(), &certPtr, 0, 0)); + cert.resetWithoutLocking(PEM_read_bio_X509(bio.get(), &certPtr, 0, 0)); if (!cert) return false; EVP_PKEY * pkeyPtr = NULL; - pkey.reset(PEM_read_bio_PrivateKey(bio.get(), &pkeyPtr, 0, 0)); + pkey.resetWithoutLocking(PEM_read_bio_PrivateKey(bio.get(), &pkeyPtr, 0, 0)); if (!pkey) return false; @@ -148,7 +148,7 @@ BIO_puts(bio.get(), bufferToRead); X509 * certPtr = NULL; - cert.reset(PEM_read_bio_X509(bio.get(), &certPtr, 0, 0)); + cert.resetWithoutLocking(PEM_read_bio_X509(bio.get(), &certPtr, 0, 0)); if (!cert) return false; @@ -511,7 +511,7 @@ if (properties.signWithPkey.get()) pkey.resetAndLock(properties.signWithPkey.get()); else // if not exist generate one - pkey.reset(Ssl::createSslPrivateKey()); + pkey.resetWithoutLocking(Ssl::createSslPrivateKey()); if (!pkey) return false; @@ -550,8 +550,8 @@ if (!ret) return false; - certToStore.reset(cert.release()); - pkeyToStore.reset(pkey.release()); + certToStore = std::move(cert); + pkeyToStore = std::move(pkey); return true; } @@ -676,11 +676,11 @@ { if (keyFilename == NULL) keyFilename = certFilename; - pkey.reset(readSslPrivateKey(keyFilename)); - cert.reset(readSslX509Certificate(certFilename)); + pkey.resetWithoutLocking(readSslPrivateKey(keyFilename)); + cert.resetWithoutLocking(readSslX509Certificate(certFilename)); if (!pkey || !cert || !X509_check_private_key(cert.get(), pkey.get())) { - pkey.reset(NULL); - cert.reset(NULL); + pkey.reset(); + cert.reset(); } } diff -u -r -N squid-4.0.12/src/ssl/gadgets.h squid-4.0.13/src/ssl/gadgets.h --- squid-4.0.12/src/ssl/gadgets.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/gadgets.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,6 +9,7 @@ #ifndef SQUID_SSL_GADGETS_H #define SQUID_SSL_GADGETS_H +#include "base/HardFun.h" #include "security/forward.h" #include "ssl/crtd_message.h" @@ -39,53 +40,41 @@ #endif /** - \ingroup SslCrtdSslAPI - * TidyPointer typedefs for common SSL objects + * std::unique_ptr typedefs for common SSL objects */ -sk_free_wrapper(sk_X509, STACK_OF(X509) *, X509_free) -typedef TidyPointer X509_STACK_Pointer; +sk_dtor_wrapper(sk_X509, STACK_OF(X509) *, X509_free); +typedef std::unique_ptr X509_STACK_Pointer; CtoCpp1(EVP_PKEY_free, EVP_PKEY *) typedef Security::LockingPointer EVP_PKEY_Pointer; -CtoCpp1(BN_free, BIGNUM *) -typedef TidyPointer BIGNUM_Pointer; +typedef std::unique_ptr> BIGNUM_Pointer; -CtoCpp1(BIO_free, BIO *) -typedef TidyPointer BIO_Pointer; +typedef std::unique_ptr> BIO_Pointer; -CtoCpp1(ASN1_INTEGER_free, ASN1_INTEGER *) -typedef TidyPointer ASN1_INT_Pointer; +typedef std::unique_ptr> ASN1_INT_Pointer; -CtoCpp1(ASN1_OCTET_STRING_free, ASN1_OCTET_STRING *) -typedef TidyPointer ASN1_OCTET_STRING_Pointer; +typedef std::unique_ptr> ASN1_OCTET_STRING_Pointer; -CtoCpp1(TXT_DB_free, TXT_DB *) -typedef TidyPointer TXT_DB_Pointer; +typedef std::unique_ptr> TXT_DB_Pointer; -CtoCpp1(X509_NAME_free, X509_NAME *) -typedef TidyPointer X509_NAME_Pointer; +typedef std::unique_ptr> X509_NAME_Pointer; -CtoCpp1(RSA_free, RSA *) -typedef TidyPointer RSA_Pointer; +typedef std::unique_ptr> RSA_Pointer; -CtoCpp1(X509_REQ_free, X509_REQ *) -typedef TidyPointer X509_REQ_Pointer; +typedef std::unique_ptr> X509_REQ_Pointer; -sk_free_wrapper(sk_X509_NAME, STACK_OF(X509_NAME) *, X509_NAME_free) -typedef TidyPointer X509_NAME_STACK_Pointer; +sk_dtor_wrapper(sk_X509_NAME, STACK_OF(X509_NAME) *, X509_NAME_free); +typedef std::unique_ptr X509_NAME_STACK_Pointer; -CtoCpp1(AUTHORITY_KEYID_free, AUTHORITY_KEYID *) -typedef TidyPointer AUTHORITY_KEYID_Pointer; +typedef std::unique_ptr> AUTHORITY_KEYID_Pointer; -sk_free_wrapper(sk_GENERAL_NAME, STACK_OF(GENERAL_NAME) *, GENERAL_NAME_free) -typedef TidyPointer GENERAL_NAME_STACK_Pointer; +sk_dtor_wrapper(sk_GENERAL_NAME, STACK_OF(GENERAL_NAME) *, GENERAL_NAME_free); +typedef std::unique_ptr GENERAL_NAME_STACK_Pointer; -CtoCpp1(GENERAL_NAME_free, GENERAL_NAME *) -typedef TidyPointer GENERAL_NAME_Pointer; +typedef std::unique_ptr> GENERAL_NAME_Pointer; -CtoCpp1(X509_EXTENSION_free, X509_EXTENSION *) -typedef TidyPointer X509_EXTENSION_Pointer; +typedef std::unique_ptr> X509_EXTENSION_Pointer; /** \ingroup SslCrtdSslAPI diff -u -r -N squid-4.0.12/src/ssl/helper.cc squid-4.0.13/src/ssl/helper.cc --- squid-4.0.12/src/ssl/helper.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/helper.cc 2016-08-06 00:52:55.000000000 +1200 @@ -85,8 +85,7 @@ std::string msg = message.compose(); msg += '\n'; if (!ssl_crtd->trySubmit(msg.c_str(), callback, data)) { - ::Helper::Reply failReply; - failReply.result = ::Helper::BrokenHelper; + ::Helper::Reply failReply(::Helper::BrokenHelper); failReply.notes.add("message", "error 45 Temporary network problem, please retry later"); callback(data, failReply); return; @@ -198,6 +197,9 @@ if (reply.result == ::Helper::BrokenHelper) { debugs(83, DBG_IMPORTANT, "\"ssl_crtvd\" helper error response: " << reply.other().content()); validationResponse->resultCode = ::Helper::BrokenHelper; + } else if (!reply.other().hasContent()) { + debugs(83, DBG_IMPORTANT, "\"ssl_crtvd\" helper returned NULL response"); + validationResponse->resultCode = ::Helper::BrokenHelper; } else if (replyMsg.parse(reply.other().content(), reply.other().contentSize()) != Ssl::CrtdMessage::OK || !replyMsg.parseResponse(*validationResponse, peerCerts, error) ) { debugs(83, DBG_IMPORTANT, "WARNING: Reply from ssl_crtvd for " << " is incorrect"); diff -u -r -N squid-4.0.12/src/ssl/helper.h squid-4.0.13/src/ssl/helper.h --- squid-4.0.12/src/ssl/helper.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/helper.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,9 +9,12 @@ #ifndef SQUID_SSL_HELPER_H #define SQUID_SSL_HELPER_H +#if USE_OPENSSL + #include "base/AsyncJobCalls.h" #include "base/LruMap.h" #include "helper/forward.h" +#include "security/forward.h" #include "ssl/cert_validate_message.h" #include "ssl/crtd_message.h" @@ -39,13 +42,12 @@ }; #endif -class PeerConnector; class CertValidationRequest; class CertValidationResponse; class CertValidationHelper { public: - typedef UnaryMemFunT CbDialer; + typedef UnaryMemFunT CbDialer; typedef void CVHCB(void *, Ssl::CertValidationResponse const &); static CertValidationHelper * GetInstance(); ///< Instance class. @@ -64,5 +66,7 @@ }; } //namespace Ssl + +#endif /* USE_OPENSSL */ #endif // SQUID_SSL_HELPER_H diff -u -r -N squid-4.0.12/src/ssl/Makefile.am squid-4.0.13/src/ssl/Makefile.am --- squid-4.0.12/src/ssl/Makefile.am 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/Makefile.am 2016-08-06 00:52:55.000000000 +1200 @@ -14,8 +14,6 @@ libsslsquid_la_SOURCES = \ bio.cc \ bio.h \ - BlindPeerConnector.cc \ - BlindPeerConnector.h \ cert_validate_message.cc \ cert_validate_message.h \ context_storage.cc \ @@ -28,8 +26,6 @@ ErrorDetailManager.h \ PeekingPeerConnector.cc \ PeekingPeerConnector.h \ - PeerConnector.cc \ - PeerConnector.h \ ProxyCerts.h \ ServerBump.cc \ ServerBump.h \ diff -u -r -N squid-4.0.12/src/ssl/Makefile.in squid-4.0.13/src/ssl/Makefile.in --- squid-4.0.12/src/ssl/Makefile.in 2016-07-02 01:29:03.000000000 +1200 +++ squid-4.0.13/src/ssl/Makefile.in 2016-08-06 00:55:14.000000000 +1200 @@ -163,10 +163,10 @@ CONFIG_CLEAN_VPATH_FILES = LTLIBRARIES = $(noinst_LTLIBRARIES) libsslsquid_la_LIBADD = -am_libsslsquid_la_OBJECTS = bio.lo BlindPeerConnector.lo \ - cert_validate_message.lo context_storage.lo Config.lo \ - ErrorDetail.lo ErrorDetailManager.lo PeekingPeerConnector.lo \ - PeerConnector.lo ServerBump.lo support.lo helper.lo +am_libsslsquid_la_OBJECTS = bio.lo cert_validate_message.lo \ + context_storage.lo Config.lo ErrorDetail.lo \ + ErrorDetailManager.lo PeekingPeerConnector.lo ServerBump.lo \ + support.lo helper.lo libsslsquid_la_OBJECTS = $(am_libsslsquid_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) @@ -722,8 +722,6 @@ libsslsquid_la_SOURCES = \ bio.cc \ bio.h \ - BlindPeerConnector.cc \ - BlindPeerConnector.h \ cert_validate_message.cc \ cert_validate_message.h \ context_storage.cc \ @@ -736,8 +734,6 @@ ErrorDetailManager.h \ PeekingPeerConnector.cc \ PeekingPeerConnector.h \ - PeerConnector.cc \ - PeerConnector.h \ ProxyCerts.h \ ServerBump.cc \ ServerBump.h \ @@ -819,12 +815,10 @@ distclean-compile: -rm -f *.tab.c -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BlindPeerConnector.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Config.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ErrorDetail.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ErrorDetailManager.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeekingPeerConnector.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerConnector.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerBump.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bio.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cert_validate_message.Plo@am__quote@ diff -u -r -N squid-4.0.12/src/ssl/PeekingPeerConnector.cc squid-4.0.13/src/ssl/PeekingPeerConnector.cc --- squid-4.0.12/src/ssl/PeekingPeerConnector.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/PeekingPeerConnector.cc 2016-08-06 00:52:55.000000000 +1200 @@ -97,7 +97,7 @@ srvBio->holdWrite(false); srvBio->recordInput(false); debugs(83,5, "Retry the fwdNegotiateSSL on FD " << serverConn->fd); - Ssl::PeerConnector::noteWantWrite(); + Security::PeerConnector::noteWantWrite(); } else { splice = true; // Ssl Negotiation stops here. Last SSL checks for valid certificates @@ -133,12 +133,11 @@ return ::Config.ssl_client.sslContext; } -Security::SessionPtr -Ssl::PeekingPeerConnector::initializeSsl() +bool +Ssl::PeekingPeerConnector::initializeTls(Security::SessionPointer &serverSession) { - auto ssl = Ssl::PeerConnector::initializeSsl(); - if (!ssl) - return nullptr; + if (!Security::PeerConnector::initializeTls(serverSession)) + return false; if (ConnStateData *csd = request->clientConnectionManager.valid()) { @@ -147,8 +146,8 @@ assert(clientConn != NULL); SBuf *hostName = NULL; - //Enable Status_request tls extension, required to bump some clients - SSL_set_tlsext_status_type(ssl, TLSEXT_STATUSTYPE_ocsp); + //Enable Status_request TLS extension, required to bump some clients + SSL_set_tlsext_status_type(serverSession.get(), TLSEXT_STATUSTYPE_ocsp); const Security::TlsDetails::Pointer details = csd->tlsParser.details; if (details && !details->serverName.isEmpty()) @@ -164,20 +163,20 @@ } if (hostName) - SSL_set_ex_data(ssl, ssl_ex_index_server, (void*)hostName); + SSL_set_ex_data(serverSession.get(), ssl_ex_index_server, (void*)hostName); Must(!csd->serverBump() || csd->serverBump()->step <= Ssl::bumpStep2); if (csd->sslBumpMode == Ssl::bumpPeek || csd->sslBumpMode == Ssl::bumpStare) { - auto clientSsl = fd_table[clientConn->fd].ssl.get(); - Must(clientSsl); - BIO *bc = SSL_get_rbio(clientSsl); + auto clientSession = fd_table[clientConn->fd].ssl.get(); + Must(clientSession); + BIO *bc = SSL_get_rbio(clientSession); Ssl::ClientBio *cltBio = static_cast(bc->ptr); Must(cltBio); if (details && details->tlsVersion.protocol != AnyP::PROTO_NONE) { - applyTlsDetailsToSSL(ssl, details, csd->sslBumpMode); + applyTlsDetailsToSSL(serverSession.get(), details, csd->sslBumpMode); // Should we allow it for all protocols? if (details->tlsVersion.protocol == AnyP::PROTO_TLS || details->tlsVersion == AnyP::ProtocolVersion(AnyP::PROTO_SSL, 3, 0)) { - BIO *b = SSL_get_rbio(ssl); + BIO *b = SSL_get_rbio(serverSession.get()); Ssl::ServerBio *srvBio = static_cast(b->ptr); // Inherite client features, like SSL version, SNI and other srvBio->setClientFeatures(details, cltBio->rBufData()); @@ -187,7 +186,7 @@ } } else { // Set client SSL options - SSL_set_options(ssl, ::Security::ProxyOutgoingConfig.parsedOptions); + SSL_set_options(serverSession.get(), ::Security::ProxyOutgoingConfig.parsedOptions); // Use SNI TLS extension only when we connect directly // to the origin server and we know the server host name. @@ -199,20 +198,20 @@ sniServer = hostName->c_str(); if (sniServer) - Ssl::setClientSNI(ssl, sniServer); + Ssl::setClientSNI(serverSession.get(), sniServer); } if (Ssl::ServerBump *serverBump = csd->serverBump()) { - serverBump->attachServerSSL(ssl); + serverBump->attachServerSSL(serverSession.get()); // store peeked cert to check SQUID_X509_V_ERR_CERT_CHANGE if (X509 *peeked_cert = serverBump->serverCert.get()) { CRYPTO_add(&(peeked_cert->references),1,CRYPTO_LOCK_X509); - SSL_set_ex_data(ssl, ssl_ex_index_ssl_peeked_cert, peeked_cert); + SSL_set_ex_data(serverSession.get(), ssl_ex_index_ssl_peeked_cert, peeked_cert); } } } - return ssl; + return true; } void @@ -274,7 +273,7 @@ return; } - Ssl::PeerConnector::noteWantWrite(); + Security::PeerConnector::noteWantWrite(); } void @@ -319,7 +318,7 @@ } // else call parent noteNegotiationError to produce an error page - Ssl::PeerConnector::noteSslNegotiationError(result, ssl_error, ssl_lib_error); + Security::PeerConnector::noteSslNegotiationError(result, ssl_error, ssl_lib_error); } void @@ -339,7 +338,7 @@ // remember the server certificate for later use if (Ssl::ServerBump *serverBump = csd->serverBump()) { - serverBump->serverCert.reset(serverCert.release()); + serverBump->serverCert = std::move(serverCert); } } } @@ -354,7 +353,7 @@ else { const int fd = serverConnection()->fd; Security::SessionPtr ssl = fd_table[fd].ssl.get(); - serverCert.reset(SSL_get_peer_certificate(ssl)); + serverCert.resetWithoutLocking(SSL_get_peer_certificate(ssl)); } if (serverCert.get()) { csd->resetSslCommonName(Ssl::CommonHostName(serverCert.get())); diff -u -r -N squid-4.0.12/src/ssl/PeekingPeerConnector.h squid-4.0.13/src/ssl/PeekingPeerConnector.h --- squid-4.0.12/src/ssl/PeekingPeerConnector.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/PeekingPeerConnector.h 2016-08-06 00:52:55.000000000 +1200 @@ -9,7 +9,7 @@ #ifndef SQUID_SRC_SSL_PEEKINGPEERCONNECTOR_H #define SQUID_SRC_SSL_PEEKINGPEERCONNECTOR_H -#include "ssl/PeerConnector.h" +#include "security/PeerConnector.h" #if USE_OPENSSL @@ -17,7 +17,7 @@ { /// A PeerConnector for HTTP origin servers. Capable of SslBumping. -class PeekingPeerConnector: public PeerConnector { +class PeekingPeerConnector: public Security::PeerConnector { CBDATA_CLASS(PeekingPeerConnector); public: PeekingPeerConnector(HttpRequestPointer &aRequest, @@ -27,7 +27,7 @@ const AccessLogEntryPointer &alp, const time_t timeout = 0) : AsyncJob("Ssl::PeekingPeerConnector"), - PeerConnector(aServerConn, aCallback, alp, timeout), + Security::PeerConnector(aServerConn, aCallback, alp, timeout), clientConn(aClientConn), splice(false), resumingSession(false), @@ -36,8 +36,8 @@ request = aRequest; } - /* PeerConnector API */ - virtual Security::SessionPtr initializeSsl(); + /* Security::PeerConnector API */ + virtual bool initializeTls(Security::SessionPointer &); virtual Security::ContextPtr getSslContext(); virtual void noteWantWrite(); virtual void noteSslNegotiationError(const int result, const int ssl_error, const int ssl_lib_error); diff -u -r -N squid-4.0.12/src/ssl/PeerConnector.cc squid-4.0.13/src/ssl/PeerConnector.cc --- squid-4.0.12/src/ssl/PeerConnector.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/PeerConnector.cc 1970-01-01 12:00:00.000000000 +1200 @@ -1,491 +0,0 @@ -/* - * Copyright (C) 1996-2016 The Squid Software Foundation and contributors - * - * Squid software is distributed under GPLv2+ license and includes - * contributions from numerous individuals and organizations. - * Please see the COPYING and CONTRIBUTORS files for details. - */ - -/* DEBUG: section 83 TLS Server/Peer negotiation */ - -#include "squid.h" -#include "acl/FilledChecklist.h" -#include "comm/Loops.h" -#include "errorpage.h" -#include "fde.h" -#include "HttpRequest.h" -#include "security/NegotiationHistory.h" -#include "SquidConfig.h" -#include "ssl/bio.h" -#include "ssl/cert_validate_message.h" -#include "ssl/Config.h" -#include "ssl/helper.h" -#include "ssl/PeerConnector.h" - -CBDATA_NAMESPACED_CLASS_INIT(Ssl, PeerConnector); - -Ssl::PeerConnector::PeerConnector(const Comm::ConnectionPointer &aServerConn, AsyncCall::Pointer &aCallback, const AccessLogEntryPointer &alp, const time_t timeout) : - AsyncJob("Ssl::PeerConnector"), - serverConn(aServerConn), - al(alp), - callback(aCallback), - negotiationTimeout(timeout), - startTime(squid_curtime), - useCertValidator_(true) -{ - // if this throws, the caller's cb dialer is not our CbDialer - Must(dynamic_cast(callback->getDialer())); -} - -Ssl::PeerConnector::~PeerConnector() -{ - debugs(83, 5, "Peer connector " << this << " gone"); -} - -bool Ssl::PeerConnector::doneAll() const -{ - return (!callback || callback->canceled()) && AsyncJob::doneAll(); -} - -/// Preps connection and SSL state. Calls negotiate(). -void -Ssl::PeerConnector::start() -{ - AsyncJob::start(); - - if (prepareSocket() && initializeSsl()) - negotiateSsl(); -} - -void -Ssl::PeerConnector::commCloseHandler(const CommCloseCbParams ¶ms) -{ - debugs(83, 5, "FD " << params.fd << ", Ssl::PeerConnector=" << params.data); - connectionClosed("Ssl::PeerConnector::commCloseHandler"); -} - -void -Ssl::PeerConnector::connectionClosed(const char *reason) -{ - mustStop(reason); - callback = NULL; -} - -bool -Ssl::PeerConnector::prepareSocket() -{ - const int fd = serverConnection()->fd; - if (!Comm::IsConnOpen(serverConn) || fd_table[serverConn->fd].closing()) { - connectionClosed("Ssl::PeerConnector::prepareSocket"); - return false; - } - - // watch for external connection closures - typedef CommCbMemFunT Dialer; - closeHandler = JobCallback(9, 5, Dialer, this, Ssl::PeerConnector::commCloseHandler); - comm_add_close_handler(fd, closeHandler); - return true; -} - -Security::SessionPtr -Ssl::PeerConnector::initializeSsl() -{ - Security::ContextPtr sslContext(getSslContext()); - assert(sslContext); - - const int fd = serverConnection()->fd; - - auto ssl = Ssl::CreateClient(sslContext, fd, "server https start"); - if (!ssl) { - ErrorState *anErr = new ErrorState(ERR_SOCKET_FAILURE, Http::scInternalServerError, request.getRaw()); - anErr->xerrno = errno; - debugs(83, DBG_IMPORTANT, "Error allocating SSL handle: " << ERR_error_string(ERR_get_error(), NULL)); - - noteNegotiationDone(anErr); - bail(anErr); - return nullptr; - } - - // If CertValidation Helper used do not lookup checklist for errors, - // but keep a list of errors to send it to CertValidator - if (!Ssl::TheConfig.ssl_crt_validator) { - // Create the ACL check list now, while we have access to more info. - // The list is used in ssl_verify_cb() and is freed in ssl_free(). - if (acl_access *acl = ::Config.ssl_client.cert_error) { - ACLFilledChecklist *check = new ACLFilledChecklist(acl, request.getRaw(), dash_str); - check->al = al; - // check->fd(fd); XXX: need client FD here - SSL_set_ex_data(ssl, ssl_ex_index_cert_error_check, check); - } - } - return ssl; -} - -void -Ssl::PeerConnector::setReadTimeout() -{ - int timeToRead; - if (negotiationTimeout) { - const int timeUsed = squid_curtime - startTime; - const int timeLeft = max(0, static_cast(negotiationTimeout - timeUsed)); - timeToRead = min(static_cast(::Config.Timeout.read), timeLeft); - } else - timeToRead = ::Config.Timeout.read; - AsyncCall::Pointer nil; - commSetConnTimeout(serverConnection(), timeToRead, nil); -} - -void -Ssl::PeerConnector::recordNegotiationDetails() -{ - const int fd = serverConnection()->fd; - Security::SessionPtr ssl = fd_table[fd].ssl.get(); - - // retrieve TLS server negotiated information if any - serverConnection()->tlsNegotiations()->retrieveNegotiatedInfo(ssl); - // retrieve TLS parsed extra info - BIO *b = SSL_get_rbio(ssl); - Ssl::ServerBio *bio = static_cast(b->ptr); - if (const Security::TlsDetails::Pointer &details = bio->receivedHelloDetails()) - serverConnection()->tlsNegotiations()->retrieveParsedInfo(details); -} - -void -Ssl::PeerConnector::negotiateSsl() -{ - if (!Comm::IsConnOpen(serverConnection()) || fd_table[serverConnection()->fd].closing()) - return; - - const int fd = serverConnection()->fd; - Security::SessionPtr ssl = fd_table[fd].ssl.get(); - const int result = SSL_connect(ssl); - if (result <= 0) { - handleNegotiateError(result); - return; // we might be gone by now - } - - recordNegotiationDetails(); - - if (!sslFinalized()) - return; - - callBack(); -} - -bool -Ssl::PeerConnector::sslFinalized() -{ - if (Ssl::TheConfig.ssl_crt_validator && useCertValidator_) { - const int fd = serverConnection()->fd; - Security::SessionPtr ssl = fd_table[fd].ssl.get(); - - Ssl::CertValidationRequest validationRequest; - // WARNING: Currently we do not use any locking for any of the - // members of the Ssl::CertValidationRequest class. In this code the - // Ssl::CertValidationRequest object used only to pass data to - // Ssl::CertValidationHelper::submit method. - validationRequest.ssl = ssl; - validationRequest.domainName = request->url.host(); - if (Ssl::CertErrors *errs = static_cast(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors))) - // validationRequest disappears on return so no need to cbdataReference - validationRequest.errors = errs; - else - validationRequest.errors = NULL; - try { - debugs(83, 5, "Sending SSL certificate for validation to ssl_crtvd."); - AsyncCall::Pointer call = asyncCall(83,5, "Ssl::PeerConnector::sslCrtvdHandleReply", Ssl::CertValidationHelper::CbDialer(this, &Ssl::PeerConnector::sslCrtvdHandleReply, nullptr)); - Ssl::CertValidationHelper::GetInstance()->sslSubmit(validationRequest, call); - return false; - } catch (const std::exception &e) { - debugs(83, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtvd " << - "request for " << validationRequest.domainName << - " certificate: " << e.what() << "; will now block to " << - "validate that certificate."); - // fall through to do blocking in-process generation. - ErrorState *anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw()); - - noteNegotiationDone(anErr); - bail(anErr); - serverConn->close(); - return true; - } - } - - noteNegotiationDone(NULL); - return true; -} - -void -Ssl::PeerConnector::sslCrtvdHandleReply(Ssl::CertValidationResponse::Pointer validationResponse) -{ - Must(validationResponse != NULL); - - Ssl::ErrorDetail *errDetails = NULL; - bool validatorFailed = false; - if (!Comm::IsConnOpen(serverConnection())) { - return; - } - - debugs(83,5, request->url.host() << " cert validation result: " << validationResponse->resultCode); - - if (validationResponse->resultCode == ::Helper::Error) { - if (Ssl::CertErrors *errs = sslCrtvdCheckForErrors(*validationResponse, errDetails)) { - Security::SessionPtr ssl = fd_table[serverConnection()->fd].ssl.get(); - Ssl::CertErrors *oldErrs = static_cast(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors)); - SSL_set_ex_data(ssl, ssl_ex_index_ssl_errors, (void *)errs); - delete oldErrs; - } - } else if (validationResponse->resultCode != ::Helper::Okay) - validatorFailed = true; - - if (!errDetails && !validatorFailed) { - noteNegotiationDone(NULL); - callBack(); - return; - } - - ErrorState *anErr = NULL; - if (validatorFailed) { - anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw()); - } else { - anErr = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request.getRaw()); - anErr->detail = errDetails; - /*anErr->xerrno= Should preserved*/ - } - - noteNegotiationDone(anErr); - bail(anErr); - serverConn->close(); - return; -} - -/// Checks errors in the cert. validator response against sslproxy_cert_error. -/// The first honored error, if any, is returned via errDetails parameter. -/// The method returns all seen errors except SSL_ERROR_NONE as Ssl::CertErrors. -Ssl::CertErrors * -Ssl::PeerConnector::sslCrtvdCheckForErrors(Ssl::CertValidationResponse const &resp, Ssl::ErrorDetail *& errDetails) -{ - Ssl::CertErrors *errs = NULL; - - ACLFilledChecklist *check = NULL; - if (acl_access *acl = ::Config.ssl_client.cert_error) { - check = new ACLFilledChecklist(acl, request.getRaw(), dash_str); - check->al = al; - } - - Security::SessionPtr ssl = fd_table[serverConnection()->fd].ssl.get(); - typedef Ssl::CertValidationResponse::RecvdErrors::const_iterator SVCRECI; - for (SVCRECI i = resp.errors.begin(); i != resp.errors.end(); ++i) { - debugs(83, 7, "Error item: " << i->error_no << " " << i->error_reason); - - assert(i->error_no != SSL_ERROR_NONE); - - if (!errDetails) { - bool allowed = false; - if (check) { - check->sslErrors = new Ssl::CertErrors(Ssl::CertError(i->error_no, i->cert.get(), i->error_depth)); - if (check->fastCheck() == ACCESS_ALLOWED) - allowed = true; - } - // else the Config.ssl_client.cert_error access list is not defined - // and the first error will cause the error page - - if (allowed) { - debugs(83, 3, "bypassing SSL error " << i->error_no << " in " << "buffer"); - } else { - debugs(83, 5, "confirming SSL error " << i->error_no); - X509 *brokenCert = i->cert.get(); - Security::CertPointer peerCert(SSL_get_peer_certificate(ssl)); - const char *aReason = i->error_reason.empty() ? NULL : i->error_reason.c_str(); - errDetails = new Ssl::ErrorDetail(i->error_no, peerCert.get(), brokenCert, aReason); - } - if (check) { - delete check->sslErrors; - check->sslErrors = NULL; - } - } - - if (!errs) - errs = new Ssl::CertErrors(Ssl::CertError(i->error_no, i->cert.get(), i->error_depth)); - else - errs->push_back_unique(Ssl::CertError(i->error_no, i->cert.get(), i->error_depth)); - } - if (check) - delete check; - - return errs; -} - -/// A wrapper for Comm::SetSelect() notifications. -void -Ssl::PeerConnector::NegotiateSsl(int, void *data) -{ - PeerConnector *pc = static_cast(data); - // Use job calls to add done() checks and other job logic/protections. - CallJobHere(83, 7, pc, Ssl::PeerConnector, negotiateSsl); -} - -void -Ssl::PeerConnector::handleNegotiateError(const int ret) -{ - const int fd = serverConnection()->fd; - unsigned long ssl_lib_error = SSL_ERROR_NONE; - Security::SessionPtr ssl = fd_table[fd].ssl.get(); - const int ssl_error = SSL_get_error(ssl, ret); - - switch (ssl_error) { - case SSL_ERROR_WANT_READ: - noteWantRead(); - return; - - case SSL_ERROR_WANT_WRITE: - noteWantWrite(); - return; - - case SSL_ERROR_SSL: - case SSL_ERROR_SYSCALL: - ssl_lib_error = ERR_get_error(); - // proceed to the general error handling code - break; - default: - // no special error handling for all other errors - break; - } - - // Log connection details, if any - recordNegotiationDetails(); - noteSslNegotiationError(ret, ssl_error, ssl_lib_error); -} - -void -Ssl::PeerConnector::noteWantRead() -{ - setReadTimeout(); - const int fd = serverConnection()->fd; - Comm::SetSelect(fd, COMM_SELECT_READ, &NegotiateSsl, this, 0); -} - -void -Ssl::PeerConnector::noteWantWrite() -{ - const int fd = serverConnection()->fd; - Comm::SetSelect(fd, COMM_SELECT_WRITE, &NegotiateSsl, this, 0); - return; -} - -void -Ssl::PeerConnector::noteSslNegotiationError(const int ret, const int ssl_error, const int ssl_lib_error) -{ -#ifdef EPROTO - int sysErrNo = EPROTO; -#else - int sysErrNo = EACCES; -#endif - - // store/report errno when ssl_error is SSL_ERROR_SYSCALL, ssl_lib_error is 0, and ret is -1 - if (ssl_error == SSL_ERROR_SYSCALL && ret == -1 && ssl_lib_error == 0) - sysErrNo = errno; - - const int fd = serverConnection()->fd; - debugs(83, DBG_IMPORTANT, "Error negotiating SSL on FD " << fd << - ": " << ERR_error_string(ssl_lib_error, NULL) << " (" << - ssl_error << "/" << ret << "/" << errno << ")"); - - ErrorState *anErr = NULL; - if (request != NULL) - anErr = ErrorState::NewForwarding(ERR_SECURE_CONNECT_FAIL, request.getRaw()); - else - anErr = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, NULL); - anErr->xerrno = sysErrNo; - - Security::SessionPtr ssl = fd_table[fd].ssl.get(); - Ssl::ErrorDetail *errFromFailure = (Ssl::ErrorDetail *)SSL_get_ex_data(ssl, ssl_ex_index_ssl_error_detail); - if (errFromFailure != NULL) { - // The errFromFailure is attached to the ssl object - // and will be released when ssl object destroyed. - // Copy errFromFailure to a new Ssl::ErrorDetail object - anErr->detail = new Ssl::ErrorDetail(*errFromFailure); - } else { - // server_cert can be NULL here - X509 *server_cert = SSL_get_peer_certificate(ssl); - anErr->detail = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, server_cert, NULL); - X509_free(server_cert); - } - - if (ssl_lib_error != SSL_ERROR_NONE) - anErr->detail->setLibError(ssl_lib_error); - - noteNegotiationDone(anErr); - bail(anErr); -} - -void -Ssl::PeerConnector::bail(ErrorState *error) -{ - Must(error); // or the recepient will not know there was a problem - Must(callback != NULL); - CbDialer *dialer = dynamic_cast(callback->getDialer()); - Must(dialer); - dialer->answer().error = error; - - callBack(); - // Our job is done. The callabck recepient will probably close the failed - // peer connection and try another peer or go direct (if possible). We - // can close the connection ourselves (our error notification would reach - // the recepient before the fd-closure notification), but we would rather - // minimize the number of fd-closure notifications and let the recepient - // manage the TCP state of the connection. -} - -void -Ssl::PeerConnector::callBack() -{ - AsyncCall::Pointer cb = callback; - // Do this now so that if we throw below, swanSong() assert that we _tried_ - // to call back holds. - callback = NULL; // this should make done() true - - // remove close handler - comm_remove_close_handler(serverConnection()->fd, closeHandler); - - CbDialer *dialer = dynamic_cast(cb->getDialer()); - Must(dialer); - dialer->answer().conn = serverConnection(); - ScheduleCallHere(cb); -} - -void -Ssl::PeerConnector::swanSong() -{ - // XXX: unregister fd-closure monitoring and CommSetSelect interest, if any - AsyncJob::swanSong(); - if (callback != NULL) { // paranoid: we have left the caller waiting - debugs(83, DBG_IMPORTANT, "BUG: Unexpected state while connecting to a cache_peer or origin server"); - ErrorState *anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request.getRaw()); - bail(anErr); - assert(!callback); - return; - } -} - -const char * -Ssl::PeerConnector::status() const -{ - static MemBuf buf; - buf.reset(); - - // TODO: redesign AsyncJob::status() API to avoid this - // id and stop reason reporting duplication. - buf.append(" [", 2); - if (stopReason != NULL) { - buf.append("Stopped, reason:", 16); - buf.appendf("%s",stopReason); - } - if (serverConn != NULL) - buf.appendf(" FD %d", serverConn->fd); - buf.appendf(" %s%u]", id.Prefix, id.value); - buf.terminate(); - - return buf.content(); -} - diff -u -r -N squid-4.0.12/src/ssl/PeerConnector.h squid-4.0.13/src/ssl/PeerConnector.h --- squid-4.0.12/src/ssl/PeerConnector.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/PeerConnector.h 1970-01-01 12:00:00.000000000 +1200 @@ -1,188 +0,0 @@ -/* - * Copyright (C) 1996-2016 The Squid Software Foundation and contributors - * - * Squid software is distributed under GPLv2+ license and includes - * contributions from numerous individuals and organizations. - * Please see the COPYING and CONTRIBUTORS files for details. - */ - -#ifndef SQUID_SRC_SSL_PEERCONNECTOR_H -#define SQUID_SRC_SSL_PEERCONNECTOR_H - -#include "acl/Acl.h" -#include "base/AsyncCbdataCalls.h" -#include "base/AsyncJob.h" -#include "CommCalls.h" -#include "security/EncryptorAnswer.h" -#include "ssl/support.h" - -#include - -#if USE_OPENSSL - -class HttpRequest; -class ErrorState; -class AccessLogEntry; -typedef RefCount AccessLogEntryPointer; - -namespace Ssl -{ - -/** - \par - * Connects Squid to SSL/TLS-capable peers or services. - * Contains common code and interfaces of various specialized PeerConnectors, - * including peer certificate validation code. - \par - * The caller receives a call back with Security::EncryptorAnswer. If answer.error - * is not nil, then there was an error and the SSL connection to the SSL peer - * was not fully established. The error object is suitable for error response - * generation. - \par - * The caller must monitor the connection for closure because this - * job will not inform the caller about such events. - \par - * PeerConnector class curently supports a form of SSL negotiation timeout, - * which accounted only when sets the read timeout from SSL peer. - * For a complete solution, the caller must monitor the overall connection - * establishment timeout and close the connection on timeouts. This is probably - * better than having dedicated (or none at all!) timeouts for peer selection, - * DNS lookup, TCP handshake, SSL handshake, etc. Some steps may have their - * own timeout, but not all steps should be forced to have theirs. - * XXX: tunnel.cc and probably other subsystems does not have an "overall - * connection establishment" timeout. We need to change their code so that they - * start monitoring earlier and close on timeouts. This change may need to be - * discussed on squid-dev. - \par - * This job never closes the connection, even on errors. If a 3rd-party - * closes the connection, this job simply quits without informing the caller. - */ -class PeerConnector: virtual public AsyncJob -{ - CBDATA_CLASS(PeerConnector); - -public: - /// Callback dialier API to allow PeerConnector to set the answer. - class CbDialer - { - public: - virtual ~CbDialer() {} - /// gives PeerConnector access to the in-dialer answer - virtual Security::EncryptorAnswer &answer() = 0; - }; - - typedef RefCount HttpRequestPointer; - -public: - PeerConnector(const Comm::ConnectionPointer &aServerConn, - AsyncCall::Pointer &aCallback, - const AccessLogEntryPointer &alp, - const time_t timeout = 0); - virtual ~PeerConnector(); - -protected: - // AsyncJob API - virtual void start(); - virtual bool doneAll() const; - virtual void swanSong(); - virtual const char *status() const; - - /// The comm_close callback handler. - void commCloseHandler(const CommCloseCbParams ¶ms); - - /// Inform us that the connection is closed. Does the required clean-up. - void connectionClosed(const char *reason); - - /// Sets up TCP socket-related notification callbacks if things go wrong. - /// If socket already closed return false, else install the comm_close - /// handler to monitor the socket. - bool prepareSocket(); - - /// Sets the read timeout to avoid getting stuck while reading from a - /// silent server - void setReadTimeout(); - - virtual Security::SessionPtr initializeSsl(); ///< Initializes SSL state - - /// Performs a single secure connection negotiation step. - /// It is called multiple times untill the negotiation finish or aborted. - void negotiateSsl(); - - /// Called after SSL negotiations have finished. Cleans up SSL state. - /// Returns false if we are now waiting for the certs validation job. - /// Otherwise, returns true, regardless of negotiation success/failure. - bool sslFinalized(); - - /// Called when the SSL negotiation step aborted because data needs to - /// be transferred to/from SSL server or on error. In the first case - /// setups the appropriate Comm::SetSelect handler. In second case - /// fill an error and report to the PeerConnector caller. - void handleNegotiateError(const int result); - - /// Called when the openSSL SSL_connect fnction request more data from - /// the remote SSL server. Sets the read timeout and sets the - /// Squid COMM_SELECT_READ handler. - void noteWantRead(); - - /// Called when the openSSL SSL_connect function needs to write data to - /// the remote SSL server. Sets the Squid COMM_SELECT_WRITE handler. - virtual void noteWantWrite(); - - /// Called when the SSL_connect function aborts with an SSL negotiation error - /// \param result the SSL_connect return code - /// \param ssl_error the error code returned from the SSL_get_error function - /// \param ssl_lib_error the error returned from the ERR_Get_Error function - virtual void noteSslNegotiationError(const int result, const int ssl_error, const int ssl_lib_error); - - /// Called when the SSL negotiation to the server completed and the certificates - /// validated using the cert validator. - /// \param error if not NULL the SSL negotiation was aborted with an error - virtual void noteNegotiationDone(ErrorState *error) {} - - /// Must implemented by the kid classes to return the Security::ContextPtr object to use - /// for building the SSL objects. - virtual Security::ContextPtr getSslContext() = 0; - - /// mimics FwdState to minimize changes to FwdState::initiate/negotiateSsl - Comm::ConnectionPointer const &serverConnection() const { return serverConn; } - - void bail(ErrorState *error); ///< Return an error to the PeerConnector caller - - /// Callback the caller class, and pass the ready to communicate secure - /// connection or an error if PeerConnector failed. - void callBack(); - - /// If called the certificates validator will not used - void bypassCertValidator() {useCertValidator_ = false;} - - /// Called after negotiation finishes to record connection details for - /// logging - void recordNegotiationDetails(); - - HttpRequestPointer request; ///< peer connection trigger or cause - Comm::ConnectionPointer serverConn; ///< TCP connection to the peer - AccessLogEntryPointer al; ///< info for the future access.log entry - AsyncCall::Pointer callback; ///< we call this with the results -private: - PeerConnector(const PeerConnector &); // not implemented - PeerConnector &operator =(const PeerConnector &); // not implemented - - /// Process response from cert validator helper - void sslCrtvdHandleReply(Ssl::CertValidationResponsePointer); - - /// Check SSL errors returned from cert validator against sslproxy_cert_error access list - Ssl::CertErrors *sslCrtvdCheckForErrors(Ssl::CertValidationResponse const &, Ssl::ErrorDetail *&); - - /// A wrapper function for negotiateSsl for use with Comm::SetSelect - static void NegotiateSsl(int fd, void *data); - AsyncCall::Pointer closeHandler; ///< we call this when the connection closed - time_t negotiationTimeout; ///< the SSL connection timeout to use - time_t startTime; ///< when the peer connector negotiation started - bool useCertValidator_; ///< whether the certificate validator should bypassed -}; - -} // namespace Ssl - -#endif /* USE_OPENSSL */ -#endif /* SQUID_SRC_SSL_PEERCONNECTOR_H */ - diff -u -r -N squid-4.0.12/src/ssl/support.cc squid-4.0.13/src/ssl/support.cc --- squid-4.0.12/src/ssl/support.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/support.cc 2016-08-06 00:52:55.000000000 +1200 @@ -33,6 +33,9 @@ #include +// TODO: Move ssl_ex_index_* global variables from global.cc here. +int ssl_ex_index_ssl_untrusted_chain = -1; + Ipc::MemMap *Ssl::SessionCache = NULL; const char *Ssl::SessionCacheName = "ssl_session_cache"; @@ -315,7 +318,7 @@ } delete filledCheck->sslErrors; filledCheck->sslErrors = NULL; - filledCheck->serverCert.reset(NULL); + filledCheck->serverCert.reset(); } // If the certificate validator is used then we need to allow all errors and // pass them to certficate validator for more processing @@ -470,6 +473,7 @@ ssl_ex_index_ssl_errors = SSL_get_ex_new_index(0, (void *) "ssl_errors", NULL, NULL, &ssl_free_SslErrors); ssl_ex_index_ssl_cert_chain = SSL_get_ex_new_index(0, (void *) "ssl_cert_chain", NULL, NULL, &ssl_free_CertChain); ssl_ex_index_ssl_validation_counter = SSL_get_ex_new_index(0, (void *) "ssl_validation_counter", NULL, NULL, &ssl_free_int); + ssl_ex_index_ssl_untrusted_chain = SSL_get_ex_new_index(0, (void *) "ssl_untrusted_chain", NULL, NULL, &ssl_free_CertChain); } #if defined(SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS) @@ -557,63 +561,57 @@ } bool -Ssl::InitServerContext(Security::ContextPtr &sslContext, AnyP::PortCfg &port) +Ssl::InitServerContext(const Security::ContextPointer &ctx, AnyP::PortCfg &port) { - if (!sslContext) + if (!ctx) return false; - if (!SSL_CTX_use_certificate(sslContext, port.signingCert.get())) { + if (!SSL_CTX_use_certificate(ctx.get(), port.signingCert.get())) { const int ssl_error = ERR_get_error(); const auto &keys = port.secure.certs.front(); debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire TLS certificate '" << keys.certFile << "': " << ERR_error_string(ssl_error, NULL)); - SSL_CTX_free(sslContext); return false; } - if (!SSL_CTX_use_PrivateKey(sslContext, port.signPkey.get())) { + if (!SSL_CTX_use_PrivateKey(ctx.get(), port.signPkey.get())) { const int ssl_error = ERR_get_error(); const auto &keys = port.secure.certs.front(); debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire TLS private key '" << keys.privateKeyFile << "': " << ERR_error_string(ssl_error, NULL)); - SSL_CTX_free(sslContext); return false; } - Ssl::addChainToSslContext(sslContext, port.certsToChain.get()); + Ssl::addChainToSslContext(ctx.get(), port.certsToChain.get()); /* Alternate code; debugs(83, DBG_IMPORTANT, "Using certificate in " << certfile); - if (!SSL_CTX_use_certificate_chain_file(sslContext, certfile)) { + if (!SSL_CTX_use_certificate_chain_file(ctx.get(), certfile)) { ssl_error = ERR_get_error(); debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire SSL certificate '" << certfile << "': " << ERR_error_string(ssl_error, NULL)); - SSL_CTX_free(sslContext); return false; } debugs(83, DBG_IMPORTANT, "Using private key in " << keyfile); - ssl_ask_password(sslContext, keyfile); + ssl_ask_password(ctx.get(), keyfile); - if (!SSL_CTX_use_PrivateKey_file(sslContext, keyfile, SSL_FILETYPE_PEM)) { + if (!SSL_CTX_use_PrivateKey_file(ctx.get(), keyfile, SSL_FILETYPE_PEM)) { ssl_error = ERR_get_error(); debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire SSL private key '" << keyfile << "': " << ERR_error_string(ssl_error, NULL)); - SSL_CTX_free(sslContext); return false; } debugs(83, 5, "Comparing private and public SSL keys."); - if (!SSL_CTX_check_private_key(sslContext)) { + if (!SSL_CTX_check_private_key(ctx.get())) { ssl_error = ERR_get_error(); debugs(83, DBG_CRITICAL, "ERROR: SSL private key '" << certfile << "' does not match public key '" << keyfile << "': " << ERR_error_string(ssl_error, NULL)); - SSL_CTX_free(sslContext); return false; } */ - if (!configureSslContext(sslContext, port)) { + if (!configureSslContext(ctx.get(), port)) { debugs(83, DBG_CRITICAL, "ERROR: Configuring static SSL context"); - SSL_CTX_free(sslContext); return false; } @@ -1096,6 +1094,32 @@ } } +static const char * +hasAuthorityInfoAccessCaIssuers(X509 *cert) +{ + AUTHORITY_INFO_ACCESS *info; + if (!cert) + return nullptr; + info = static_cast(X509_get_ext_d2i(cert, NID_info_access, NULL, NULL)); + if (!info) + return nullptr; + + static char uri[MAX_URL]; + uri[0] = '\0'; + + for (int i = 0; i < sk_ACCESS_DESCRIPTION_num(info); i++) { + ACCESS_DESCRIPTION *ad = sk_ACCESS_DESCRIPTION_value(info, i); + if (OBJ_obj2nid(ad->method) == NID_ad_ca_issuers) { + if (ad->location->type == GEN_URI) { + xstrncpy(uri, reinterpret_cast(ASN1_STRING_data(ad->location->d.uniformResourceIdentifier)), sizeof(uri)); + } + break; + } + } + AUTHORITY_INFO_ACCESS_free(info); + return uri[0] != '\0' ? uri : nullptr; +} + bool Ssl::loadCerts(const char *certsFile, Ssl::CertsIndexedList &list) { @@ -1116,9 +1140,10 @@ return true; } -/// quickly find a certificate with a given issuer in Ssl::CertsIndexedList. +/// quickly find the issuer certificate of a certificate cert in the +/// Ssl::CertsIndexedList list static X509 * -findCertByIssuerFast(X509_STORE_CTX *ctx, Ssl::CertsIndexedList &list, X509 *cert) +findCertIssuerFast(Ssl::CertsIndexedList &list, X509 *cert) { static char buffer[2048]; @@ -1130,21 +1155,80 @@ const auto ret = list.equal_range(SBuf(buffer)); for (Ssl::CertsIndexedList::iterator it = ret.first; it != ret.second; ++it) { X509 *issuer = it->second; - if (ctx->check_issued(ctx, cert, issuer)) { + if (X509_check_issued(cert, issuer)) { return issuer; } } return NULL; } -/// slowly find a certificate with a given issuer using linear search +/// slowly find the issuer certificate of a given cert using linear search +static bool +findCertIssuer(Security::CertList const &list, X509 *cert) +{ + for (Security::CertList::const_iterator it = list.begin(); it != list.end(); ++it) { + if (X509_check_issued(it->get(), cert) == X509_V_OK) + return true; + } + return false; +} + +const char * +Ssl::uriOfIssuerIfMissing(X509 *cert, Security::CertList const &serverCertificates) +{ + if (!cert || !serverCertificates.size()) + return nullptr; + + if (!findCertIssuer(serverCertificates, cert)) { + //if issuer is missing + if (!findCertIssuerFast(SquidUntrustedCerts, cert)) { + // and issuer not found in local untrusted certificates database + if (const char *issuerUri = hasAuthorityInfoAccessCaIssuers(cert)) { + // There is a URI where we can download a certificate. + return issuerUri; + } + } + } + return nullptr; +} + +void +Ssl::missingChainCertificatesUrls(std::queue &URIs, Security::CertList const &serverCertificates) +{ + if (!serverCertificates.size()) + return; + + for (const auto &i : serverCertificates) { + if (const char *issuerUri = uriOfIssuerIfMissing(i.get(), serverCertificates)) + URIs.push(SBuf(issuerUri)); + } +} + +void +Ssl::SSL_add_untrusted_cert(SSL *ssl, X509 *cert) +{ + STACK_OF(X509) *untrustedStack = static_cast (SSL_get_ex_data(ssl, ssl_ex_index_ssl_untrusted_chain)); + if (!untrustedStack) { + untrustedStack = sk_X509_new_null(); + if (!SSL_set_ex_data(ssl, ssl_ex_index_ssl_untrusted_chain, untrustedStack)) { + sk_X509_pop_free(untrustedStack, X509_free); + throw TextException("Failed to attach untrusted certificates chain"); + } + } + sk_X509_push(untrustedStack, cert); +} + +/// Search for the issuer certificate of cert in sk list. static X509 * -findCertByIssuerSlowly(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *cert) +sk_x509_findIssuer(STACK_OF(X509) *sk, X509 *cert) { + if (!sk) + return NULL; + const int skItemsNum = sk_X509_num(sk); for (int i = 0; i < skItemsNum; ++i) { X509 *issuer = sk_X509_value(sk, i); - if (ctx->check_issued(ctx, cert, issuer)) + if (X509_check_issued(issuer, cert) == X509_V_OK) return issuer; } return NULL; @@ -1160,16 +1244,16 @@ X509 *current = ctx->cert; int i = 0; for (i = 0; current && (i < depth); ++i) { - if (ctx->check_issued(ctx, current, current)) { + if (X509_check_issued(current, current)) { // either ctx->cert is itself self-signed or untrustedCerts // aready contain the self-signed current certificate break; } // untrustedCerts is short, not worth indexing - X509 *issuer = findCertByIssuerSlowly(ctx, untrustedCerts, current); + X509 *issuer = sk_x509_findIssuer(untrustedCerts, current); if (!issuer) { - if ((issuer = findCertByIssuerFast(ctx, SquidUntrustedCerts, current))) + if ((issuer = findCertIssuerFast(SquidUntrustedCerts, current))) sk_X509_push(untrustedCerts, issuer); } current = issuer; @@ -1185,12 +1269,25 @@ { debugs(83, 4, "Try to use pre-downloaded intermediate certificates\n"); + SSL *ssl = static_cast(X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); + STACK_OF(X509) *sslUntrustedStack = static_cast (SSL_get_ex_data(ssl, ssl_ex_index_ssl_untrusted_chain)); + // OpenSSL already maintains ctx->untrusted but we cannot modify // internal OpenSSL list directly. We have to give OpenSSL our own // list, but it must include certificates on the OpenSSL ctx->untrusted STACK_OF(X509) *oldUntrusted = ctx->untrusted; STACK_OF(X509) *sk = sk_X509_dup(oldUntrusted); // oldUntrusted is always not NULL - completeIssuers(ctx, sk); + + for (int i = 0; i < sk_X509_num(sslUntrustedStack); ++i) { + X509 *cert = sk_X509_value(sslUntrustedStack, i); + sk_X509_push(sk, cert); + } + + // If the local untrusted certificates internal database is used + // run completeIssuers to add missing certificates if possible. + if (SquidUntrustedCerts.size() > 0) + completeIssuers(ctx, sk); + X509_STORE_CTX_set_chain(ctx, sk); // No locking/unlocking, just sets ctx->untrusted int ret = X509_verify_cert(ctx); X509_STORE_CTX_set_chain(ctx, oldUntrusted); // Set back the old untrusted list @@ -1201,10 +1298,7 @@ void Ssl::useSquidUntrusted(SSL_CTX *sslContext) { - if (SquidUntrustedCerts.size() > 0) - SSL_CTX_set_cert_verify_callback(sslContext, untrustedToStoreCtx_cb, NULL); - else - SSL_CTX_set_cert_verify_callback(sslContext, NULL, NULL); + SSL_CTX_set_cert_verify_callback(sslContext, untrustedToStoreCtx_cb, NULL); } bool @@ -1273,11 +1367,11 @@ // XXX: ssl_ask_password_cb needs SSL_CTX_set_default_passwd_cb_userdata() // so this may not fully work iff Config.Program.ssl_password is set. pem_password_cb *cb = ::Config.Program.ssl_password ? &ssl_ask_password_cb : NULL; - pkey.reset(readSslPrivateKey(keyFilename, cb)); - cert.reset(readSslX509CertificatesChain(certFilename, chain.get())); + pkey.resetWithoutLocking(readSslPrivateKey(keyFilename, cb)); + cert.resetWithoutLocking(readSslX509CertificatesChain(certFilename, chain.get())); if (!pkey || !cert || !X509_check_private_key(cert.get(), pkey.get())) { - pkey.reset(NULL); - cert.reset(NULL); + pkey.reset(); + cert.reset(); } } @@ -1304,26 +1398,27 @@ return Ssl::generateSslCertificate(untrustedCert, untrustedPkey, certProperties); } -SSL * -SslCreate(Security::ContextPtr sslContext, const int fd, Ssl::Bio::Type type, const char *squidCtx) +static bool +SslCreate(Security::ContextPtr sslContext, const Comm::ConnectionPointer &conn, Ssl::Bio::Type type, const char *squidCtx) { - if (fd < 0) { + if (!Comm::IsConnOpen(conn)) { debugs(83, DBG_IMPORTANT, "Gone connection"); - return NULL; + return false; } const char *errAction = NULL; int errCode = 0; if (auto ssl = SSL_new(sslContext)) { + const int fd = conn->fd; // without BIO, we would call SSL_set_fd(ssl, fd) instead if (BIO *bio = Ssl::Bio::Create(fd, type)) { Ssl::Bio::Link(ssl, bio); // cannot fail - fd_table[fd].ssl.reset(ssl); + fd_table[fd].ssl.resetWithoutLocking(ssl); fd_table[fd].read_method = &ssl_read_method; fd_table[fd].write_method = &ssl_write_method; fd_note(fd, squidCtx); - return ssl; + return true; } errCode = ERR_get_error(); errAction = "failed to initialize I/O"; @@ -1335,19 +1430,19 @@ debugs(83, DBG_IMPORTANT, "ERROR: " << squidCtx << ' ' << errAction << ": " << ERR_error_string(errCode, NULL)); - return NULL; + return false; } -SSL * -Ssl::CreateClient(Security::ContextPtr sslContext, const int fd, const char *squidCtx) +bool +Ssl::CreateClient(Security::ContextPtr sslContext, const Comm::ConnectionPointer &c, const char *squidCtx) { - return SslCreate(sslContext, fd, Ssl::Bio::BIO_TO_SERVER, squidCtx); + return SslCreate(sslContext, c, Ssl::Bio::BIO_TO_SERVER, squidCtx); } -SSL * -Ssl::CreateServer(Security::ContextPtr sslContext, const int fd, const char *squidCtx) +bool +Ssl::CreateServer(Security::ContextPtr sslContext, const Comm::ConnectionPointer &c, const char *squidCtx) { - return SslCreate(sslContext, fd, Ssl::Bio::BIO_TO_CLIENT, squidCtx); + return SslCreate(sslContext, c, Ssl::Bio::BIO_TO_CLIENT, squidCtx); } Ssl::CertError::CertError(ssl_error_t anErr, X509 *aCert, int aDepth): code(anErr), depth(aDepth) diff -u -r -N squid-4.0.12/src/ssl/support.h squid-4.0.13/src/ssl/support.h --- squid-4.0.12/src/ssl/support.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/ssl/support.h 2016-08-06 00:52:55.000000000 +1200 @@ -14,6 +14,7 @@ #if USE_OPENSSL #include "base/CbDataList.h" +#include "comm/forward.h" #include "sbuf/SBuf.h" #include "security/forward.h" #include "ssl/gadgets.h" @@ -27,6 +28,7 @@ #if HAVE_OPENSSL_ENGINE_H #include #endif +#include #include /** @@ -77,12 +79,12 @@ typedef RefCount CertValidationResponsePointer; /// Creates SSL Client connection structure and initializes SSL I/O (Comm and BIO). -/// On errors, emits DBG_IMPORTANT with details and returns NULL. -SSL *CreateClient(Security::ContextPtr sslContext, const int fd, const char *squidCtx); +/// On errors, emits DBG_IMPORTANT with details and returns false. +bool CreateClient(Security::ContextPtr sslContext, const Comm::ConnectionPointer &, const char *squidCtx); /// Creates SSL Server connection structure and initializes SSL I/O (Comm and BIO). -/// On errors, emits DBG_IMPORTANT with details and returns NULL. -SSL *CreateServer(Security::ContextPtr sslContext, const int fd, const char *squidCtx); +/// On errors, emits DBG_IMPORTANT with details and returns false. +bool CreateServer(Security::ContextPtr sslContext, const Comm::ConnectionPointer &, const char *squidCtx); /// An SSL certificate-related error. /// Pairs an error code with the certificate experiencing the error. @@ -112,7 +114,7 @@ extern const char *SessionCacheName; /// initialize a TLS server context with OpenSSL specific settings -bool InitServerContext(Security::ContextPtr &, AnyP::PortCfg &); +bool InitServerContext(const Security::ContextPointer &, AnyP::PortCfg &); /// initialize a TLS client context with OpenSSL specific settings bool InitClientContext(Security::ContextPtr &, Security::PeerOptions &, long options, long flags); @@ -182,6 +184,46 @@ return (0 <= bm && bm < Ssl::bumpEnd) ? Ssl::BumpModeStr[bm] : NULL; } +/// certificates indexed by issuer name +typedef std::multimap CertsIndexedList; + +/** + * Load PEM-encoded certificates from the given file. + */ +bool loadCerts(const char *certsFile, Ssl::CertsIndexedList &list); + +/** + * Load PEM-encoded certificates to the squid untrusteds certificates + * internal DB from the given file. + */ +bool loadSquidUntrusted(const char *path); + +/** + * Removes all certificates from squid untrusteds certificates + * internal DB and frees all memory + */ +void unloadSquidUntrusted(); + +/** + * Add the certificate cert to ssl object untrusted certificates. + * Squid uses an attached to SSL object list of untrusted certificates, + * with certificates which can be used to complete incomplete chains sent + * by the SSL server. + */ +void SSL_add_untrusted_cert(SSL *ssl, X509 *cert); + +/** + * Searches in serverCertificates list for the cert issuer and if not found + * and Authority Info Access of cert provides a URI return it. + */ +const char *uriOfIssuerIfMissing(X509 *cert, Security::CertList const &serverCertificates); + +/** + * Fill URIs queue with the uris of missing certificates from serverCertificate chain + * if this information provided by Authority Info Access. + */ +void missingChainCertificatesUrls(std::queue &URIs, Security::CertList const &serverCertificates); + /** \ingroup ServerProtocolSSLAPI * Generate a certificate to be used as untrusted signing certificate, based on a trusted CA diff -u -r -N squid-4.0.12/src/store/Controller.cc squid-4.0.13/src/store/Controller.cc --- squid-4.0.12/src/store/Controller.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store/Controller.cc 2016-08-06 00:52:55.000000000 +1200 @@ -64,7 +64,8 @@ swapDir->init(); - if (UsingSmp() && IamWorkerProcess() && Config.onoff.collapsed_forwarding) { + if (UsingSmp() && IamWorkerProcess() && Config.onoff.collapsed_forwarding && + smpAware()) { transients = new Transients; transients->init(); } @@ -498,8 +499,10 @@ Must(old); HttpReply *oldReply = const_cast(old->getReply()); Must(oldReply); - oldReply->updateOnNotModified(newer.getReply()); - old->timestampsSet(); + + const bool modified = oldReply->updateOnNotModified(newer.getReply()); + if (!old->timestampsSet() && !modified) + return; /* update stored image of the old entry */ @@ -514,7 +517,8 @@ Store::Controller::allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod) { - e->makePublic(); // this is needed for both local and SMP collapsing + const KeyScope keyScope = reqFlags.refresh ? ksRevalidation : ksDefault; + e->makePublic(keyScope); // this is needed for both local and SMP collapsing if (transients) transients->startWriting(e, reqFlags, reqMethod); debugs(20, 3, "may " << (transients && e->mem_obj->xitTable.index >= 0 ? @@ -601,6 +605,12 @@ return found; } +bool +Store::Controller::smpAware() const +{ + return memStore || (swapDir && swapDir->smpAware()); +} + namespace Store { static RefCount TheRoot; } diff -u -r -N squid-4.0.12/src/store/Controller.h squid-4.0.13/src/store/Controller.h --- squid-4.0.12/src/store/Controller.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store/Controller.h 2016-08-06 00:52:55.000000000 +1200 @@ -41,6 +41,7 @@ virtual void markForUnlink(StoreEntry &) override; virtual void unlink(StoreEntry &) override; virtual int callback() override; + virtual bool smpAware() const override; /// Additional unknown-size entry bytes required by Store in order to /// reduce the risk of selecting the wrong disk cache for the growing entry. diff -u -r -N squid-4.0.12/src/store/Disk.h squid-4.0.13/src/store/Disk.h --- squid-4.0.12/src/store/Disk.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store/Disk.h 2016-08-06 00:52:55.000000000 +1200 @@ -53,6 +53,7 @@ virtual void reference(StoreEntry &e) override; virtual bool dereference(StoreEntry &e) override; virtual void maintain() override; + virtual bool smpAware() const override { return false; } /// the size of the smallest entry this cache_dir can store int64_t minObjectSize() const; diff -u -r -N squid-4.0.12/src/store/Disks.cc squid-4.0.13/src/store/Disks.cc --- squid-4.0.12/src/store/Disks.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store/Disks.cc 2016-08-06 00:52:55.000000000 +1200 @@ -530,6 +530,19 @@ dir(collapsed.swap_dirn).updateCollapsed(collapsed); } +bool +Store::Disks::smpAware() const +{ + for (int i = 0; i < Config.cacheSwap.n_configured; ++i) { + // A mix is not supported, but we conservatively check every + // dir because features like collapsed revalidation should + // currently be disabled if any dir is SMP-aware + if (dir(i).smpAware()) + return true; + } + return false; +} + /* Store::Disks globals that should be converted to use RegisteredRunner */ void diff -u -r -N squid-4.0.12/src/store/Disks.h squid-4.0.13/src/store/Disks.h --- squid-4.0.12/src/store/Disks.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store/Disks.h 2016-08-06 00:52:55.000000000 +1200 @@ -48,6 +48,7 @@ /// Additional unknown-size entry bytes required by disks in order to /// reduce the risk of selecting the wrong disk cache for the growing entry. int64_t accumulateMore(const StoreEntry&) const; + virtual bool smpAware() const override; private: /* migration logic */ diff -u -r -N squid-4.0.12/src/store/id_rewriters/file/storeid_file_rewrite.8 squid-4.0.13/src/store/id_rewriters/file/storeid_file_rewrite.8 --- squid-4.0.12/src/store/id_rewriters/file/storeid_file_rewrite.8 2016-07-02 02:23:49.000000000 +1200 +++ squid-4.0.13/src/store/id_rewriters/file/storeid_file_rewrite.8 2016-08-06 02:26:53.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "STOREID_FILE_REWRITE 8" -.TH STOREID_FILE_REWRITE 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH STOREID_FILE_REWRITE 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/src/store/Storage.h squid-4.0.13/src/store/Storage.h --- squid-4.0.12/src/store/Storage.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store/Storage.h 2016-08-06 00:52:55.000000000 +1200 @@ -74,6 +74,11 @@ /// prepare for shutdown virtual void sync() {} + + /// whether this storage is capable of serving multiple workers; + /// a true result does not imply [lack of] non-SMP support because + /// [only] some SMP-aware storages also support non-SMP configss + virtual bool smpAware() const = 0; }; } // namespace Store diff -u -r -N squid-4.0.12/src/store.cc squid-4.0.13/src/store.cc --- squid-4.0.12/src/store.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store.cc 2016-08-06 00:52:55.000000000 +1200 @@ -139,12 +139,11 @@ } void -StoreEntry::makePublic() +StoreEntry::makePublic(const KeyScope scope) { /* This object can be cached for a long time */ - if (!EBIT_TEST(flags, RELEASE_REQUEST)) - setPublicKey(); + setPublicKey(scope); } void @@ -559,19 +558,19 @@ } StoreEntry * -storeGetPublicByRequestMethod(HttpRequest * req, const HttpRequestMethod& method) +storeGetPublicByRequestMethod(HttpRequest * req, const HttpRequestMethod& method, const KeyScope keyScope) { - return Store::Root().get(storeKeyPublicByRequestMethod(req, method)); + return Store::Root().get(storeKeyPublicByRequestMethod(req, method, keyScope)); } StoreEntry * -storeGetPublicByRequest(HttpRequest * req) +storeGetPublicByRequest(HttpRequest * req, const KeyScope keyScope) { - StoreEntry *e = storeGetPublicByRequestMethod(req, req->method); + StoreEntry *e = storeGetPublicByRequestMethod(req, req->method, keyScope); if (e == NULL && req->method == Http::METHOD_HEAD) /* We can generate a HEAD reply from a cached GET object */ - e = storeGetPublicByRequestMethod(req, Http::METHOD_GET); + e = storeGetPublicByRequestMethod(req, Http::METHOD_GET, keyScope); return e; } @@ -622,10 +621,8 @@ } void -StoreEntry::setPublicKey() +StoreEntry::setPublicKey(const KeyScope scope) { - const cache_key *newkey; - if (key && !EBIT_TEST(flags, KEY_PRIVATE)) return; /* is already public */ @@ -649,80 +646,35 @@ assert(!EBIT_TEST(flags, RELEASE_REQUEST)); - if (mem_obj->request) { - HttpRequest *request = mem_obj->request; - - if (mem_obj->vary_headers.isEmpty()) { - /* First handle the case where the object no longer varies */ - request->vary_headers.clear(); - } else { - if (!request->vary_headers.isEmpty() && request->vary_headers.cmp(mem_obj->vary_headers) != 0) { - /* Oops.. the variance has changed. Kill the base object - * to record the new variance key - */ - request->vary_headers.clear(); /* free old "bad" variance key */ - if (StoreEntry *pe = storeGetPublic(mem_obj->storeId(), mem_obj->method)) - pe->release(); - } - - /* Make sure the request knows the variance status */ - if (request->vary_headers.isEmpty()) - request->vary_headers = httpMakeVaryMark(request, mem_obj->getReply()); - } - - // TODO: storeGetPublic() calls below may create unlocked entries. - // We should add/use storeHas() API or lock/unlock those entries. - if (!mem_obj->vary_headers.isEmpty() && !storeGetPublic(mem_obj->storeId(), mem_obj->method)) { - /* Create "vary" base object */ - String vary; - StoreEntry *pe = storeCreateEntry(mem_obj->storeId(), mem_obj->logUri(), request->flags, request->method); - /* We are allowed to do this typecast */ - HttpReply *rep = new HttpReply; - rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); - vary = mem_obj->getReply()->header.getList(Http::HdrType::VARY); - - if (vary.size()) { - /* Again, we own this structure layout */ - rep->header.putStr(Http::HdrType::VARY, vary.termedBuf()); - vary.clean(); - } - -#if X_ACCELERATOR_VARY - vary = mem_obj->getReply()->header.getList(Http::HdrType::HDR_X_ACCELERATOR_VARY); - - if (vary.size() > 0) { - /* Again, we own this structure layout */ - rep->header.putStr(Http::HdrType::HDR_X_ACCELERATOR_VARY, vary.termedBuf()); - vary.clean(); - } - -#endif - pe->replaceHttpReply(rep, false); // no write until key is public - - pe->timestampsSet(); - - pe->makePublic(); + adjustVary(); + forcePublicKey(calcPublicKey(scope)); +} - pe->startWriting(); // after makePublic() +void +StoreEntry::clearPublicKeyScope() +{ + if (!key || EBIT_TEST(flags, KEY_PRIVATE)) + return; // probably the old public key was deleted or made private - pe->complete(); + // TODO: adjustVary() when collapsed revalidation supports that - pe->unlock("StoreEntry::setPublicKey+Vary"); - } + const cache_key *newKey = calcPublicKey(ksDefault); + if (!storeKeyHashCmp(key, newKey)) + return; // probably another collapsed revalidation beat us to this change - newkey = storeKeyPublicByRequest(mem_obj->request); - } else - newkey = storeKeyPublic(mem_obj->storeId(), mem_obj->method); + forcePublicKey(newKey); +} +/// Unconditionally sets public key for this store entry. +/// Releases the old entry with the same public key (if any). +void +StoreEntry::forcePublicKey(const cache_key *newkey) +{ if (StoreEntry *e2 = (StoreEntry *)hash_lookup(store_table, newkey)) { + assert(e2 != this); debugs(20, 3, "Making old " << *e2 << " private."); e2->setPrivateKey(); e2->release(); - - if (mem_obj->request) - newkey = storeKeyPublicByRequest(mem_obj->request); - else - newkey = storeKeyPublic(mem_obj->storeId(), mem_obj->method); } if (key) @@ -736,6 +688,88 @@ storeDirSwapLog(this, SWAP_LOG_ADD); } +/// Calculates correct public key for feeding forcePublicKey(). +/// Assumes adjustVary() has been called for this entry already. +const cache_key * +StoreEntry::calcPublicKey(const KeyScope keyScope) +{ + assert(mem_obj); + return mem_obj->request ? storeKeyPublicByRequest(mem_obj->request, keyScope) : + storeKeyPublic(mem_obj->storeId(), mem_obj->method, keyScope); +} + +/// Updates mem_obj->request->vary_headers to reflect the current Vary. +/// The vary_headers field is used to calculate the Vary marker key. +/// Releases the old Vary marker with an outdated key (if any). +void +StoreEntry::adjustVary() +{ + assert(mem_obj); + + if (!mem_obj->request) + return; + + HttpRequest *request = mem_obj->request; + + if (mem_obj->vary_headers.isEmpty()) { + /* First handle the case where the object no longer varies */ + request->vary_headers.clear(); + } else { + if (!request->vary_headers.isEmpty() && request->vary_headers.cmp(mem_obj->vary_headers) != 0) { + /* Oops.. the variance has changed. Kill the base object + * to record the new variance key + */ + request->vary_headers.clear(); /* free old "bad" variance key */ + if (StoreEntry *pe = storeGetPublic(mem_obj->storeId(), mem_obj->method)) + pe->release(); + } + + /* Make sure the request knows the variance status */ + if (request->vary_headers.isEmpty()) + request->vary_headers = httpMakeVaryMark(request, mem_obj->getReply()); + } + + // TODO: storeGetPublic() calls below may create unlocked entries. + // We should add/use storeHas() API or lock/unlock those entries. + if (!mem_obj->vary_headers.isEmpty() && !storeGetPublic(mem_obj->storeId(), mem_obj->method)) { + /* Create "vary" base object */ + String vary; + StoreEntry *pe = storeCreateEntry(mem_obj->storeId(), mem_obj->logUri(), request->flags, request->method); + /* We are allowed to do this typecast */ + HttpReply *rep = new HttpReply; + rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); + vary = mem_obj->getReply()->header.getList(Http::HdrType::VARY); + + if (vary.size()) { + /* Again, we own this structure layout */ + rep->header.putStr(Http::HdrType::VARY, vary.termedBuf()); + vary.clean(); + } + +#if X_ACCELERATOR_VARY + vary = mem_obj->getReply()->header.getList(Http::HdrType::HDR_X_ACCELERATOR_VARY); + + if (vary.size() > 0) { + /* Again, we own this structure layout */ + rep->header.putStr(Http::HdrType::HDR_X_ACCELERATOR_VARY, vary.termedBuf()); + vary.clean(); + } + +#endif + pe->replaceHttpReply(rep, false); // no write until key is public + + pe->timestampsSet(); + + pe->makePublic(); + + pe->startWriting(); // after makePublic() + + pe->complete(); + + pe->unlock("StoreEntry::forcePublicKey+Vary"); + } +} + StoreEntry * storeCreatePureEntry(const char *url, const char *log_url, const RequestFlags &flags, const HttpRequestMethod& method) { @@ -1486,7 +1520,7 @@ return 1; } -void +bool StoreEntry::timestampsSet() { const HttpReply *reply = getReply(); @@ -1524,14 +1558,22 @@ served_date -= (squid_curtime - request_sent); } + time_t exp = 0; if (reply->expires > 0 && reply->date > -1) - expires = served_date + (reply->expires - reply->date); + exp = served_date + (reply->expires - reply->date); else - expires = reply->expires; + exp = reply->expires; + + if (lastmod == reply->last_modified && timestamp == served_date && expires == exp) + return false; + + expires = exp; lastmod = reply->last_modified; timestamp = served_date; + + return true; } void diff -u -r -N squid-4.0.12/src/store_digest.cc squid-4.0.13/src/store_digest.cc --- squid-4.0.12/src/store_digest.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store_digest.cc 2016-08-06 00:52:55.000000000 +1200 @@ -77,36 +77,63 @@ static void storeDigestRewriteFinish(StoreEntry * e); static EVH storeDigestSwapOutStep; static void storeDigestCBlockSwapOut(StoreEntry * e); -static int storeDigestCalcCap(void); -static int storeDigestResize(void); static void storeDigestAdd(const StoreEntry *); -#endif /* USE_CACHE_DIGESTS */ - -static void -storeDigestRegisterWithCacheManager(void) +/// calculates digest capacity +static uint64_t +storeDigestCalcCap() { - Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1); -} + /* + * To-Do: Bloom proved that the optimal filter utilization is 50% (half of + * the bits are off). However, we do not have a formula to calculate the + * number of _entries_ we want to pre-allocate for. + */ + const uint64_t hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize; + const uint64_t lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize; + const uint64_t e_count = StoreEntry::inUseCount(); + uint64_t cap = e_count ? e_count : hi_cap; + debugs(71, 2, "have: " << e_count << ", want " << cap << + " entries; limits: [" << lo_cap << ", " << hi_cap << "]"); -/* - * PUBLIC FUNCTIONS - */ + if (cap < lo_cap) + cap = lo_cap; + + /* do not enforce hi_cap limit, average-based estimation may be wrong + *if (cap > hi_cap) + * cap = hi_cap; + */ + + // Bug 4534: we still have to set an upper-limit at some reasonable value though. + // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX + const uint64_t absolute_max = (INT_MAX -8) / Config.digest.bits_per_entry; + if (cap > absolute_max) { + static time_t last_loud = 0; + if (last_loud < squid_curtime - 86400) { + debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max); + last_loud = squid_curtime; + } else { + debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max); + } + cap = absolute_max; + } + + return cap; +} +#endif /* USE_CACHE_DIGESTS */ void storeDigestInit(void) { - storeDigestRegisterWithCacheManager(); + Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1); #if USE_CACHE_DIGESTS - const int cap = storeDigestCalcCap(); - if (!Config.onoff.digest_generation) { store_digest = NULL; debugs(71, 3, "Local cache digest generation disabled"); return; } + const uint64_t cap = storeDigestCalcCap(); store_digest = new CacheDigest(cap, Config.digest.bits_per_entry); debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite every " << (int) Config.digest.rebuild_period << "/" << @@ -291,6 +318,31 @@ storeDigestRebuildResume(); } +/// \returns true if we actually resized the digest +static bool +storeDigestResize() +{ + const uint64_t cap = storeDigestCalcCap(); + assert(store_digest); + uint64_t diff; + if (cap > store_digest->capacity) + diff = cap - store_digest->capacity; + else + diff = store_digest->capacity - cap; + debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " << + diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" ); + /* avoid minor adjustments */ + + if (diff <= store_digest->capacity / 10) { + debugs(71, 2, "small change, will not resize."); + return false; + } else { + debugs(71, 2, "big change, resizing."); + store_digest->updateCapacity(cap); + } + return true; +} + /* called be Rewrite to push Rebuild forward */ static void storeDigestRebuildResume(void) @@ -440,7 +492,7 @@ assert(e); /* _add_ check that nothing bad happened while we were waiting @?@ @?@ */ - if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size) + if (static_cast(sd_state.rewrite_offset + chunk_size) > store_digest->mask_size) chunk_size = store_digest->mask_size - sd_state.rewrite_offset; e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size); @@ -452,7 +504,7 @@ sd_state.rewrite_offset += chunk_size; /* are we done ? */ - if (sd_state.rewrite_offset >= store_digest->mask_size) + if (static_cast(sd_state.rewrite_offset) >= store_digest->mask_size) storeDigestRewriteFinish(e); else eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.0, 1, false); @@ -468,60 +520,10 @@ sd_state.cblock.count = htonl(store_digest->count); sd_state.cblock.del_count = htonl(store_digest->del_count); sd_state.cblock.mask_size = htonl(store_digest->mask_size); - sd_state.cblock.bits_per_entry = (unsigned char) - Config.digest.bits_per_entry; + sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry; sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount; e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock)); } -/* calculates digest capacity */ -static int -storeDigestCalcCap(void) -{ - /* - * To-Do: Bloom proved that the optimal filter utilization is 50% (half of - * the bits are off). However, we do not have a formula to calculate the - * number of _entries_ we want to pre-allocate for. - */ - const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize; - const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize; - const int e_count = StoreEntry::inUseCount(); - int cap = e_count ? e_count :hi_cap; - debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap << - " entries; limits: [" << lo_cap << ", " << hi_cap << "]"); - - if (cap < lo_cap) - cap = lo_cap; - - /* do not enforce hi_cap limit, average-based estimation may be wrong - *if (cap > hi_cap) - * cap = hi_cap; - */ - return cap; -} - -/* returns true if we actually resized the digest */ -static int -storeDigestResize(void) -{ - const int cap = storeDigestCalcCap(); - int diff; - assert(store_digest); - diff = abs(cap - store_digest->capacity); - debugs(71, 2, "storeDigestResize: " << - store_digest->capacity << " -> " << cap << "; change: " << - diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" ); - /* avoid minor adjustments */ - - if (diff <= store_digest->capacity / 10) { - debugs(71, 2, "storeDigestResize: small change, will not resize."); - return 0; - } else { - debugs(71, 2, "storeDigestResize: big change, resizing."); - store_digest->updateCapacity(cap); - return 1; - } -} - #endif /* USE_CACHE_DIGESTS */ diff -u -r -N squid-4.0.12/src/Store.h squid-4.0.13/src/Store.h --- squid-4.0.12/src/Store.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/Store.h 2016-08-06 00:52:55.000000000 +1200 @@ -82,9 +82,13 @@ void swapOutDecision(const MemObject::SwapOut::Decision &decision); void abort(); - void makePublic(); + void makePublic(const KeyScope keyScope = ksDefault); void makePrivate(); - void setPublicKey(); + void setPublicKey(const KeyScope keyScope = ksDefault); + /// Resets existing public key to a public key with default scope, + /// releasing the old default-scope entry (if any). + /// Does nothing if the existing public key already has default scope. + void clearPublicKeyScope(); void setPrivateKey(); void expireNow(); void releaseRequest(); @@ -119,7 +123,7 @@ void registerAbort(STABH * cb, void *); void reset(); void setMemStatus(mem_status_t); - void timestampsSet(); + bool timestampsSet(); void unregisterAbort(); void destroyMemObject(); int checkTooSmall(); @@ -217,6 +221,9 @@ private: bool checkTooBig() const; + void forcePublicKey(const cache_key *newkey); + void adjustVary(); + const cache_key *calcPublicKey(const KeyScope keyScope); static MemAllocator *pool; @@ -286,10 +293,10 @@ StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method); /// \ingroup StoreAPI -StoreEntry *storeGetPublicByRequest(HttpRequest * request); +StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault); /// \ingroup StoreAPI -StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method); +StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault); /// \ingroup StoreAPI /// Like storeCreatePureEntry(), but also locks the entry and sets entry key. diff -u -r -N squid-4.0.12/src/store_key_md5.cc squid-4.0.13/src/store_key_md5.cc --- squid-4.0.12/src/store_key_md5.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store_key_md5.cc 2016-08-06 00:52:55.000000000 +1200 @@ -95,7 +95,7 @@ } const cache_key * -storeKeyPublic(const char *url, const HttpRequestMethod& method) +storeKeyPublic(const char *url, const HttpRequestMethod& method, const KeyScope keyScope) { static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; unsigned char m = (unsigned char) method.id(); @@ -103,18 +103,20 @@ SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); + if (keyScope) + SquidMD5Update(&M, &keyScope, sizeof(keyScope)); SquidMD5Final(digest, &M); return digest; } const cache_key * -storeKeyPublicByRequest(HttpRequest * request) +storeKeyPublicByRequest(HttpRequest * request, const KeyScope keyScope) { - return storeKeyPublicByRequestMethod(request, request->method); + return storeKeyPublicByRequestMethod(request, request->method, keyScope); } const cache_key * -storeKeyPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method) +storeKeyPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope) { static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; unsigned char m = (unsigned char) method.id(); @@ -123,6 +125,8 @@ SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url.rawContent(), url.length()); + if (keyScope) + SquidMD5Update(&M, &keyScope, sizeof(keyScope)); if (!request->vary_headers.isEmpty()) { SquidMD5Update(&M, request->vary_headers.rawContent(), request->vary_headers.length()); diff -u -r -N squid-4.0.12/src/store_key_md5.h squid-4.0.13/src/store_key_md5.h --- squid-4.0.12/src/store_key_md5.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/store_key_md5.h 2016-08-06 00:52:55.000000000 +1200 @@ -17,14 +17,19 @@ class HttpRequestMethod; class HttpRequest; +typedef enum { + ksDefault = 0, + ksRevalidation +} KeyScope; + cache_key *storeKeyDup(const cache_key *); cache_key *storeKeyCopy(cache_key *, const cache_key *); void storeKeyFree(const cache_key *); const cache_key *storeKeyScan(const char *); const char *storeKeyText(const cache_key *); -const cache_key *storeKeyPublic(const char *, const HttpRequestMethod&); -const cache_key *storeKeyPublicByRequest(HttpRequest *); -const cache_key *storeKeyPublicByRequestMethod(HttpRequest *, const HttpRequestMethod&); +const cache_key *storeKeyPublic(const char *, const HttpRequestMethod&, const KeyScope keyScope = ksDefault); +const cache_key *storeKeyPublicByRequest(HttpRequest *, const KeyScope keyScope = ksDefault); +const cache_key *storeKeyPublicByRequestMethod(HttpRequest *, const HttpRequestMethod&, const KeyScope keyScope = ksDefault); const cache_key *storeKeyPrivate(); int storeKeyHashBuckets(int); int storeKeyNull(const cache_key *); diff -u -r -N squid-4.0.12/src/tests/stub_CacheDigest.cc squid-4.0.13/src/tests/stub_CacheDigest.cc --- squid-4.0.12/src/tests/stub_CacheDigest.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_CacheDigest.cc 2016-08-06 00:52:55.000000000 +1200 @@ -17,11 +17,11 @@ class StoreEntry; #include "CacheDigest.h" -CacheDigest::CacheDigest(int, int) {STUB} +CacheDigest::CacheDigest(uint64_t, uint8_t) {STUB} CacheDigest::~CacheDigest() {STUB} CacheDigest *CacheDigest::clone() const STUB_RETVAL(nullptr) void CacheDigest::clear() STUB -void CacheDigest::updateCapacity(int) STUB +void CacheDigest::updateCapacity(uint64_t) STUB bool CacheDigest::contains(const cache_key *) const STUB_RETVAL(false) void CacheDigest::add(const cache_key *) STUB void CacheDigest::remove(const cache_key *) STUB @@ -29,5 +29,5 @@ void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB -size_t CacheDigest::CalcMaskSize(int, int) STUB_RETVAL(1) +uint32_t CacheDigest::CalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1) diff -u -r -N squid-4.0.12/src/tests/stub_HttpReply.cc squid-4.0.13/src/tests/stub_HttpReply.cc --- squid-4.0.12/src/tests/stub_HttpReply.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_HttpReply.cc 2016-08-06 00:52:55.000000000 +1200 @@ -28,6 +28,6 @@ void HttpReply::hdrCacheInit() STUB HttpReply * HttpReply::clone() const STUB_RETVAL(NULL) bool HttpReply::inheritProperties(const HttpMsg *aMsg) STUB_RETVAL(false) - void HttpReply::updateOnNotModified(HttpReply const*) STUB + bool HttpReply::updateOnNotModified(HttpReply const*) STUB_RETVAL(false) int64_t HttpReply::bodySize(const HttpRequestMethod&) const STUB_RETVAL(0) diff -u -r -N squid-4.0.12/src/tests/stub_icp.cc squid-4.0.13/src/tests/stub_icp.cc --- squid-4.0.12/src/tests/stub_icp.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_icp.cc 2016-08-06 00:52:55.000000000 +1200 @@ -9,7 +9,6 @@ #include "squid.h" #include "comm/Connection.h" #include "ICP.h" -#include "icp_opcode.h" #define STUB_API "icp_*.cc" #include "tests/STUB.h" @@ -43,3 +42,7 @@ int icpSetCacheKey(const cache_key * key) STUB_RETVAL(0) const cache_key *icpGetCacheKey(const char *url, int reqnum) STUB_RETVAL(NULL) +#include "icp_opcode.h" +// dynamically generated +#include "icp_opcode.cc" + diff -u -r -N squid-4.0.12/src/tests/stub_liblog.cc squid-4.0.13/src/tests/stub_liblog.cc --- squid-4.0.12/src/tests/stub_liblog.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.13/src/tests/stub_liblog.cc 2016-08-06 00:52:55.000000000 +1200 @@ -0,0 +1,110 @@ +/* + * Copyright (C) 1996-2016 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#include "squid.h" +#include "fde.h" + +#define STUB_API "log/liblog.la" +#include "tests/STUB.h" + +// XXX: these should be moved to a log/ *.h file +#include "AccessLogEntry.h" +/* +AccessLogEntry::~AccessLogEntry() {STUB} +void AccessLogEntry::getLogClientIp(char *, size_t) const STUB +SBuf AccessLogEntry::getLogMethod() const STUB_RETVAL(SBuf()) +#if USE_OPENSSL +AccessLogEntry::SslDetails::SslDetails() {STUB} +#endif +*/ +void accessLogLogTo(CustomLog *, AccessLogEntry::Pointer &, ACLChecklist *) STUB +void accessLogLog(AccessLogEntry::Pointer &, ACLChecklist *) STUB +void accessLogRotate(void) STUB +void accessLogClose(void) STUB +void accessLogInit(void) STUB +const char *accessLogTime(time_t) STUB_RETVAL(nullptr) + +#include "log/access_log.h" +void fvdbCountVia(const char *) STUB +void fvdbCountForw(const char *) STUB +#if HEADERS_LOG +void headersLog(int, int, const HttpRequestMethod &, void *) STUB +#endif + +#include "log/Config.h" +namespace Log +{ +void LogConfig::parseFormats() STUB +LogConfig TheConfig; +} + +//#include "log/CustomLog.h" +#include "log/File.h" +CBDATA_CLASS_INIT(Logfile); +Logfile::Logfile(const char *) {STUB} +//void Logfile::f_linestart(Logfile *) STUB +//void Logfile::f_linewrite(Logfile *, const char *, size_t) STUB +//void Logfile::f_lineend(Logfile *) STUB +//void Logfile::f_flush(Logfile *) STUB +//void Logfile::f_rotate(Logfile *, const int16_t) STUB +//void Logfile::f_close(Logfile *) STUB +Logfile *logfileOpen(const char *, size_t, int) STUB_RETVAL(nullptr) +void logfileClose(Logfile *) STUB +void logfileRotate(Logfile *, int16_t) STUB +void logfileWrite(Logfile *, char *, size_t) STUB +void logfileFlush(Logfile *) STUB +void logfilePrintf(Logfile *, const char *, ...) STUB +void logfileLineStart(Logfile *) STUB +void logfileLineEnd(Logfile *) STUB + +#include "log/Formats.h" +namespace Log +{ +namespace Format +{ +void SquidNative(const AccessLogEntryPointer &, Logfile *) STUB +void SquidIcap(const AccessLogEntryPointer &, Logfile *) STUB +void SquidUserAgent(const AccessLogEntryPointer &, Logfile *) STUB +void SquidReferer(const AccessLogEntryPointer &, Logfile *) STUB +void SquidCustom(const AccessLogEntryPointer &, CustomLog *) STUB +void HttpdCommon(const AccessLogEntryPointer &, Logfile *) STUB +void HttpdCombined(const AccessLogEntryPointer &, Logfile *) STUB +} +} + +#include "log/ModDaemon.h" +int logfile_mod_daemon_open(Logfile *, const char *, size_t, int) STUB_RETVAL(0) + +#include "log/ModStdio.h" +int logfile_mod_stdio_open(Logfile *, const char *, size_t, int) STUB_RETVAL(0) + +#include "log/ModSyslog.h" +int logfile_mod_syslog_open(Logfile *, const char *, size_t, int) STUB_RETVAL(0) + +#include "log/ModUdp.h" +int logfile_mod_udp_open(Logfile *, const char *, size_t, int) STUB_RETVAL(0) + +#include "log/TcpLogger.h" +namespace Log +{ +CBDATA_CLASS_INIT(TcpLogger); +int TcpLogger::Open(Logfile *, const char *, size_t, int) STUB_RETVAL(0) + +/* +protected: + TcpLogger(size_t, bool, Ip::Address); + virtual ~TcpLogger(); + void endGracefully(); + void logRecord(const char *buf, size_t len); + void flush(); + virtual void start() STUB + virtual bool doneAll() const STUB_RETVAL(true) + virtual void swanSong() STUB +*/ +} + diff -u -r -N squid-4.0.12/src/tests/stub_libsecurity.cc squid-4.0.13/src/tests/stub_libsecurity.cc --- squid-4.0.12/src/tests/stub_libsecurity.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_libsecurity.cc 2016-08-06 00:52:55.000000000 +1200 @@ -7,15 +7,64 @@ */ #include "squid.h" +#include "AccessLogEntry.h" #include "comm/Connection.h" +#include "HttpRequest.h" #define STUB_API "security/libsecurity.la" #include "tests/STUB.h" +#include "security/BlindPeerConnector.h" +CBDATA_NAMESPACED_CLASS_INIT(Security, BlindPeerConnector); +namespace Security +{ +bool BlindPeerConnector::initializeTls(Security::SessionPointer &) STUB_RETVAL(false) +Security::ContextPtr BlindPeerConnector::getSslContext() STUB_RETVAL(nullptr) +void BlindPeerConnector::noteNegotiationDone(ErrorState *) STUB +} #include "security/EncryptorAnswer.h" Security::EncryptorAnswer::~EncryptorAnswer() {} std::ostream &Security::operator <<(std::ostream &os, const Security::EncryptorAnswer &) STUB_RETVAL(os) +#include "security/Handshake.h" +Security::HandshakeParser::HandshakeParser() STUB +bool Security::HandshakeParser::parseHello(const SBuf &) STUB_RETVAL(false) + +#include "security/NegotiationHistory.h" +Security::NegotiationHistory::NegotiationHistory() STUB +void Security::NegotiationHistory::retrieveNegotiatedInfo(Security::SessionPtr) STUB +void Security::NegotiationHistory::retrieveParsedInfo(Security::TlsDetails::Pointer const &) STUB +const char *Security::NegotiationHistory::cipherName() const STUB +const char *Security::NegotiationHistory::printTlsVersion(AnyP::ProtocolVersion const &v) const STUB + +#include "security/PeerConnector.h" +CBDATA_NAMESPACED_CLASS_INIT(Security, PeerConnector); +namespace Security +{ +PeerConnector::PeerConnector(const Comm::ConnectionPointer &, AsyncCall::Pointer &, const AccessLogEntryPointer &, const time_t) : + AsyncJob("Security::PeerConnector") {STUB} +PeerConnector::~PeerConnector() {STUB} +void PeerConnector::start() STUB +bool PeerConnector::doneAll() const STUB_RETVAL(true) +void PeerConnector::swanSong() STUB +const char *PeerConnector::status() const STUB_RETVAL("") +void PeerConnector::commCloseHandler(const CommCloseCbParams &) STUB +void PeerConnector::connectionClosed(const char *) STUB +bool PeerConnector::prepareSocket() STUB_RETVAL(false) +void PeerConnector::setReadTimeout() STUB +bool PeerConnector::initializeTls(Security::SessionPointer &) STUB_RETVAL(false) +void PeerConnector::negotiateSsl() STUB +bool PeerConnector::sslFinalized() STUB_RETVAL(false) +void PeerConnector::handleNegotiateError(const int) STUB +void PeerConnector::noteWantRead() STUB +void PeerConnector::noteWantWrite() STUB +void PeerConnector::noteSslNegotiationError(const int, const int, const int) STUB +// virtual Security::ContextPtr getSslContext() = 0; +void PeerConnector::bail(ErrorState *) STUB +void PeerConnector::callBack() STUB +void PeerConnector::recordNegotiationDetails() STUB +} + #include "security/PeerOptions.h" Security::PeerOptions Security::ProxyOutgoingConfig; void Security::PeerOptions::parse(char const*) STUB @@ -34,24 +83,6 @@ void Security::ServerOptions::parse(const char *) STUB void Security::ServerOptions::dumpCfg(Packable *, const char *) const STUB Security::ContextPtr Security::ServerOptions::createBlankContext() const STUB -Security::ContextPtr Security::ServerOptions::createStaticServerContext(AnyP::PortCfg &) STUB_RETVAL(nullptr) +bool Security::ServerOptions::createStaticServerContext(AnyP::PortCfg &) STUB_RETVAL(false) void Security::ServerOptions::updateContextEecdh(Security::ContextPtr &) STUB -#include "security/NegotiationHistory.h" -Security::NegotiationHistory::NegotiationHistory() STUB -void Security::NegotiationHistory::retrieveNegotiatedInfo(Security::SessionPtr) STUB -void Security::NegotiationHistory::retrieveParsedInfo(Security::TlsDetails::Pointer const &) STUB -const char *Security::NegotiationHistory::cipherName() const STUB -const char *Security::NegotiationHistory::printTlsVersion(AnyP::ProtocolVersion const &v) const STUB - -#include "security/Handshake.h" -Security::HandshakeParser::HandshakeParser() STUB -bool Security::HandshakeParser::parseHello(const SBuf &) STUB_RETVAL(false) - -#include "security/Session.h" -namespace Security { -bool SessionIsResumed(const Security::SessionPointer &) STUB_RETVAL(false) -void GetSessionResumeData(const Security::SessionPointer &, Security::SessionStatePointer &) STUB -void SetSessionResumeData(const Security::SessionPtr &, const Security::SessionStatePointer &) STUB -} // namespace Security - diff -u -r -N squid-4.0.12/src/tests/stub_libsslsquid.cc squid-4.0.13/src/tests/stub_libsslsquid.cc --- squid-4.0.12/src/tests/stub_libsslsquid.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_libsslsquid.cc 2016-08-06 00:52:55.000000000 +1200 @@ -55,7 +55,7 @@ CertError & CertError::operator = (const CertError &old) STUB_RETVAL(*this) bool CertError::operator == (const CertError &ce) const STUB_RETVAL(false) bool CertError::operator != (const CertError &ce) const STUB_RETVAL(false) -bool InitServerContext(Security::ContextPtr &, AnyP::PortCfg &) STUB_RETVAL(false) +bool InitServerContext(const Security::ContextPointer &, AnyP::PortCfg &) STUB_RETVAL(false) bool InitClientContext(Security::ContextPtr &, Security::PeerOptions &, long, const char *) STUB_RETVAL(false) } // namespace Ssl int ssl_read_method(int, char *, int) STUB_RETVAL(0) diff -u -r -N squid-4.0.12/src/tests/Stub.list squid-4.0.13/src/tests/Stub.list --- squid-4.0.12/src/tests/Stub.list 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/Stub.list 2016-08-06 00:52:55.000000000 +1200 @@ -48,6 +48,7 @@ tests/stub_libeui.cc \ tests/stub_libformat.cc \ tests/stub_libicmp.cc \ + tests/stub_liblog.cc \ tests/stub_libmem.cc \ tests/stub_libmgr.cc \ tests/stub_libsecurity.cc \ diff -u -r -N squid-4.0.12/src/tests/stub_MemBuf.cc squid-4.0.13/src/tests/stub_MemBuf.cc --- squid-4.0.12/src/tests/stub_MemBuf.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_MemBuf.cc 2016-08-06 00:52:55.000000000 +1200 @@ -22,7 +22,7 @@ void MemBuf::init() STUB void MemBuf::clean() STUB void MemBuf::reset() STUB -int MemBuf::isNull() STUB_RETVAL(1) +int MemBuf::isNull() const STUB_RETVAL(1) FREE *MemBuf::freeFunc() STUB_RETVAL(NULL) void MemBuf::append(const char *, int) STUB void MemBuf::vappendf(const char *fmt, va_list ap) STUB diff -u -r -N squid-4.0.12/src/tests/stub_store.cc squid-4.0.13/src/tests/stub_store.cc --- squid-4.0.12/src/tests/stub_store.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tests/stub_store.cc 2016-08-06 00:52:55.000000000 +1200 @@ -37,9 +37,9 @@ bool StoreEntry::mayStartSwapOut() STUB_RETVAL(false) void StoreEntry::trimMemory(const bool preserveSwappable) STUB void StoreEntry::abort() STUB -void StoreEntry::makePublic() STUB +void StoreEntry::makePublic(const KeyScope scope) STUB void StoreEntry::makePrivate() STUB -void StoreEntry::setPublicKey() STUB +void StoreEntry::setPublicKey(const KeyScope scope) STUB void StoreEntry::setPrivateKey() STUB void StoreEntry::expireNow() STUB void StoreEntry::releaseRequest() STUB @@ -62,7 +62,7 @@ void StoreEntry::registerAbort(STABH * cb, void *) STUB void StoreEntry::reset() STUB void StoreEntry::setMemStatus(mem_status_t) STUB -void StoreEntry::timestampsSet() STUB +bool StoreEntry::timestampsSet() STUB_RETVAL(false) void StoreEntry::unregisterAbort() STUB void StoreEntry::destroyMemObject() STUB int StoreEntry::checkTooSmall() STUB_RETVAL(0) @@ -121,8 +121,8 @@ size_t storeEntryInUse() STUB_RETVAL(0) void storeEntryReplaceObject(StoreEntry *, HttpReply *) STUB StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method) STUB_RETVAL(NULL) -StoreEntry *storeGetPublicByRequest(HttpRequest * request) STUB_RETVAL(NULL) -StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method) STUB_RETVAL(NULL) +StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope scope) STUB_RETVAL(NULL) +StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope scope) STUB_RETVAL(NULL) StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&) STUB_RETVAL(NULL) StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&) STUB_RETVAL(NULL) void storeConfigure(void) STUB diff -u -r -N squid-4.0.12/src/Transients.cc squid-4.0.13/src/Transients.cc --- squid-4.0.12/src/Transients.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/Transients.cc 2016-08-06 00:52:55.000000000 +1200 @@ -184,7 +184,8 @@ e->mem_obj->xitTable.io = MemObject::ioReading; e->mem_obj->xitTable.index = index; - e->setPublicKey(); + // TODO: Support collapsed revalidation for SMP-aware caches. + e->setPublicKey(ksDefault); assert(e->key); // How do we know its SMP- and not just locally-collapsed? A worker gets diff -u -r -N squid-4.0.12/src/Transients.h squid-4.0.13/src/Transients.h --- squid-4.0.12/src/Transients.h 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/Transients.h 2016-08-06 00:52:55.000000000 +1200 @@ -72,6 +72,7 @@ virtual void markForUnlink(StoreEntry &e) override; virtual void unlink(StoreEntry &e) override; virtual void maintain() override; + virtual bool smpAware() const override { return true; } static int64_t EntryLimit(); diff -u -r -N squid-4.0.12/src/tunnel.cc squid-4.0.13/src/tunnel.cc --- squid-4.0.12/src/tunnel.cc 2016-07-02 01:26:44.000000000 +1200 +++ squid-4.0.13/src/tunnel.cc 2016-08-06 00:52:55.000000000 +1200 @@ -34,9 +34,9 @@ #include "MemBuf.h" #include "PeerSelectState.h" #include "sbuf/SBuf.h" +#include "security/BlindPeerConnector.h" #include "SquidConfig.h" #include "SquidTime.h" -#include "ssl/BlindPeerConnector.h" #include "StatCounters.h" #if USE_OPENSSL #include "ssl/bio.h" @@ -174,9 +174,8 @@ void connectToPeer(); private: -#if USE_OPENSSL /// Gives PeerConnector access to Answer in the TunnelStateData callback dialer. - class MyAnswerDialer: public CallDialer, public Ssl::PeerConnector::CbDialer + class MyAnswerDialer: public CallDialer, public Security::PeerConnector::CbDialer { public: typedef void (TunnelStateData::*Method)(Security::EncryptorAnswer &); @@ -191,7 +190,7 @@ os << '(' << tunnel_.get() << ", " << answer_ << ')'; } - /* Ssl::PeerConnector::CbDialer API */ + /* Security::PeerConnector::CbDialer API */ virtual Security::EncryptorAnswer &answer() { return answer_; } private: @@ -199,7 +198,6 @@ CbcPointer tunnel_; Security::EncryptorAnswer answer_; }; -#endif /// callback handler after connection setup (including any encryption) void connectedToPeer(Security::EncryptorAnswer &answer); @@ -1109,19 +1107,16 @@ void TunnelStateData::connectToPeer() { -#if USE_OPENSSL if (CachePeer *p = server.conn->getPeer()) { if (p->secure.encryptTransport) { AsyncCall::Pointer callback = asyncCall(5,4, "TunnelStateData::ConnectedToPeer", MyAnswerDialer(&TunnelStateData::connectedToPeer, this)); - Ssl::BlindPeerConnector *connector = - new Ssl::BlindPeerConnector(request, server.conn, callback, al); + auto *connector = new Security::BlindPeerConnector(request, server.conn, callback, al); AsyncJob::Start(connector); // will call our callback return; } } -#endif Security::EncryptorAnswer nil; connectedToPeer(nil); @@ -1243,14 +1238,13 @@ { debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd); /* Create state structure. */ - TunnelStateData *tunnelState = NULL; const SBuf url(request->effectiveRequestUri()); debugs(26, 3, request->method << " " << url << " " << request->http_ver); ++statCounter.server.all.requests; ++statCounter.server.other.requests; - tunnelState = new TunnelStateData; + TunnelStateData *tunnelState = new TunnelStateData; tunnelState->url = SBufToCstring(url); tunnelState->request = request; tunnelState->server.size_ptr = NULL; //Set later if Http::Stream is available @@ -1260,10 +1254,9 @@ tunnelState->status_ptr = &status_code; tunnelState->client.conn = clientConn; - ConnStateData *conn; - if ((conn = request->clientConnectionManager.get())) { + if (auto conn = request->clientConnectionManager.get()) { Http::StreamPointer context = conn->pipeline.front(); - if (context != nullptr && context->http != nullptr) { + if (context && context->http) { tunnelState->logTag_ptr = &context->http->logType; tunnelState->server.size_ptr = &context->http->out.size; tunnelState->al = context->http->al; @@ -1286,23 +1279,23 @@ fd_table[clientConn->fd].read_method = &default_read_method; fd_table[clientConn->fd].write_method = &default_write_method; - tunnelState->request->hier.note(srvConn, tunnelState->getHost()); + request->hier.note(srvConn, tunnelState->getHost()); tunnelState->server.conn = srvConn; - tunnelState->request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : NULL; + request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : nullptr; comm_add_close_handler(srvConn->fd, tunnelServerClosed, tunnelState); debugs(26, 4, "determine post-connect handling pathway."); if (srvConn->getPeer()) { - tunnelState->request->peer_login = srvConn->getPeer()->login; - tunnelState->request->peer_domain = srvConn->getPeer()->domain; - tunnelState->request->flags.auth_no_keytab = srvConn->getPeer()->options.auth_no_keytab; - tunnelState->request->flags.proxying = !(srvConn->getPeer()->options.originserver); + request->peer_login = srvConn->getPeer()->login; + request->peer_domain = srvConn->getPeer()->domain; + request->flags.auth_no_keytab = srvConn->getPeer()->options.auth_no_keytab; + request->flags.proxying = !(srvConn->getPeer()->options.originserver); } else { - tunnelState->request->peer_login = NULL; - tunnelState->request->peer_domain = NULL; - tunnelState->request->flags.auth_no_keytab = false; - tunnelState->request->flags.proxying = false; + request->peer_login = nullptr; + request->peer_domain = nullptr; + request->flags.auth_no_keytab = false; + request->flags.proxying = false; } timeoutCall = commCbCall(5, 4, "tunnelTimeout", diff -u -r -N squid-4.0.12/test-suite/stub_MemBuf.cc squid-4.0.13/test-suite/stub_MemBuf.cc --- squid-4.0.12/test-suite/stub_MemBuf.cc 2016-07-02 02:25:10.000000000 +1200 +++ squid-4.0.13/test-suite/stub_MemBuf.cc 2016-08-06 02:29:27.000000000 +1200 @@ -22,7 +22,7 @@ void MemBuf::init() STUB void MemBuf::clean() STUB void MemBuf::reset() STUB -int MemBuf::isNull() STUB_RETVAL(1) +int MemBuf::isNull() const STUB_RETVAL(1) FREE *MemBuf::freeFunc() STUB_RETVAL(NULL) void MemBuf::append(const char *, int) STUB void MemBuf::vappendf(const char *fmt, va_list ap) STUB diff -u -r -N squid-4.0.12/tools/helper-mux/helper-mux.8 squid-4.0.13/tools/helper-mux/helper-mux.8 --- squid-4.0.12/tools/helper-mux/helper-mux.8 2016-07-02 02:25:14.000000000 +1200 +++ squid-4.0.13/tools/helper-mux/helper-mux.8 2016-08-06 02:29:36.000000000 +1200 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "HELPER-MUX 8" -.TH HELPER-MUX 8 "2016-07-01" "perl v5.22.2" "User Contributed Perl Documentation" +.TH HELPER-MUX 8 "2016-08-05" "perl v5.22.2" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.12/tools/MemBuf.cc squid-4.0.13/tools/MemBuf.cc --- squid-4.0.12/tools/MemBuf.cc 2016-07-02 02:25:12.000000000 +1200 +++ squid-4.0.13/tools/MemBuf.cc 2016-08-06 02:29:31.000000000 +1200 @@ -154,7 +154,7 @@ * Unfortunate hack to test if the buffer has been Init()ialized */ int -MemBuf::isNull() +MemBuf::isNull() const { if (!buf && !max_capacity && !capacity && !size) return 1; /* is null (not initialized) */