ludra/nginx-build
1
0 Fork 0
forked from Hakase/nginx-build
Code Issues Pull Requests Releases Wiki Activity

Fix pagespeed and modified shell script

Browse Source
This commit is contained in:
Hakase
2018-05-08 14:23:09 +09:00
parent 4c2f8bfc59
commit 75163996ff
44 changed files with 19 additions and 14180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
auto.sh
+19 -8
View File
@@ -16,7 +16,7 @@
## --with-openssl-opt="enable-weak-ssl-ciphers -DCFLAGS='-march=native -O3 -fuse-linker-plugin'" \
### Remove Old file
rm -rf /usr/sbin/nginx.old
rm -f /usr/sbin/nginx.old
### Multithread build
BUILD_MTS="-j$(expr $(nproc) \+ 1)"
@@ -38,16 +38,27 @@ if [ ! -f "lib/zlib/Makefile" ]; then
fi
### PSOL Download (PageSpeed)
if [ ! -d "lib/ngx_pagespeed/psol" ]; then
cd lib/ngx_pagespeed
curl "$(scripts/format_binary_url.sh PSOL_BINARY_URL)" | tar xz
cd ../../
if [ ! -d "lib/pagespeed" ]; then
### Download pagespeed
cd lib
wget -c https://github.com/apache/incubator-pagespeed-ngx/archive/v1.13.35.2-stable.zip
unzip v1.13.35.2-stable.zip
rm -f v1.13.35.2-stable.zip
cd incubator-pagespeed-ngx-1.13.35.2-stable
### Download psol
wget -c https://dl.google.com/dl/page-speed/psol/1.13.35.2-x64.tar.gz
tar -xzvf 1.13.35.2-x64.tar.gz
rm -f 1.13.35.2-x64.tar.gz
cd ..
mv incubator-pagespeed-ngx-1.13.35.2-stable pagespeed
cd ..
fi
auto/configure \
--with-cc-opt='-DTCP_FASTOPEN=23 -m64 -flto -g -O3 -march=native -fstack-protector-strong -fuse-ld=gold -fuse-linker-plugin --param=ssp-buffer-size=4 -Wformat -Werror=format-security -Wno-strict-aliasing -Wp,-D_FORTIFY_SOURCE=2 -gsplit-dwarf -DNGX_HTTP_HEADERS' \
--with-ld-opt='-ljemalloc -Wl,-z,relro' \
--with-openssl-opt="enable-tls13downgrade enable-ec_nistp_64_gcc_128 enable-weak-ssl-ciphers no-ssl3-method" \
--with-ld-opt='-lrt -ljemalloc -Wl,-z,relro -Wl,-z,now -fPIC' \
--with-openssl-opt="enable-tls13downgrade enable-ec_nistp_64_gcc_128 enable-weak-ssl-ciphers no-ssl3-method -DCFLAGS='-O3 -march=native -fuse-linker-plugin -ljemalloc'" \
--builddir=objs --prefix=/usr/local/nginx \
--conf-path=/etc/nginx/nginx.conf \
--pid-path=/var/run/nginx.pid \
@@ -98,7 +109,7 @@ auto/configure \
--with-stream_ssl_preread_module \
--add-module=./lib/ngx_devel_kit \
--add-module=./lib/ngx_brotli \
--add-module=./lib/ngx_pagespeed ${PS_NGX_EXTRA_FLAGS} \
--add-module=./lib/pagespeed ${PS_NGX_EXTRA_FLAGS} \
--add-module=./lib/ngx-fancyindex \
--add-module=./lib/naxsi/naxsi_src \
--add-module=./lib/nginx-dav-ext-module \
lib/ngx_pagespeed/.gitignore
-5
View File
@@ -1,5 +0,0 @@
test/tmp
psol/
psol-*.tar.gz
*.*.*.*.tar.gz
nginx
lib/ngx_pagespeed/.gitmodules
View File
lib/ngx_pagespeed/.travis.yml
-33
View File
@@ -1,33 +0,0 @@
language: c++
env:
global:
- MAKEFLAGS=-j3
# By default travis loads submodules serially, but we can load them in parallel
# if we install an updated git and use --jobs. Some timing numbers:
# serial: 257s
# jobs=4: 182s (29s to install new git, 153s to run the downloads)
# jobs=8: 179s (29s to install new git, 150s to run the downloads)
# We can't use --depth=1, though, because github doesn't have
# allowReachableSHA1InWant set.
#
# TODO(jefftk): once we're running on a server with git 2.8 or later we can have
# --jobs without installing a new git.
git:
submodules: false
before_install:
- sudo add-apt-repository --yes ppa:git-core/ppa
- sudo apt-get update
- sudo apt-get install git
- git submodule update --init --recursive --jobs=8
install:
scripts/build_ngx_pagespeed.sh --devel --assume-yes
script:
test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \
$PWD/nginx/sbin/nginx
dist: trusty
sudo: required
compiler:
- gcc
notifications:
email:
- pagespeed-ci@googlegroups.com
lib/ngx_pagespeed/LICENSE
-202
View File
@@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
lib/ngx_pagespeed/PSOL_BINARY_URL
-1
View File
@@ -1 +0,0 @@
https://dl.google.com/dl/page-speed/psol/1.13.35.2-$BIT_SIZE_NAME.tar.gz
lib/ngx_pagespeed/README.md
-40
View File
@@ -1,40 +0,0 @@
![ngx_pagespeed](https://lh6.googleusercontent.com/-qufedJIJq7Y/UXEvVYxyYvI/AAAAAAAADo8/JHDFQhs91_c/s401/04_ngx_pagespeed.png)
[![Build Status](https://travis-ci.org/pagespeed/ngx_pagespeed.svg?branch=master)](https://travis-ci.org/pagespeed/ngx_pagespeed)
ngx_pagespeed speeds up your site and reduces page load time by automatically
applying web performance best practices to pages and associated assets (CSS,
JavaScript, images) without requiring you to modify your existing content or
workflow. Features include:
- Image optimization: stripping meta-data, dynamic resizing, recompression
- CSS & JavaScript minification, concatenation, inlining, and outlining
- Small resource inlining
- Deferring image and JavaScript loading
- HTML rewriting
- Cache lifetime extension
- and
[more](https://developers.google.com/speed/docs/mod_pagespeed/config_filters)
To see ngx_pagespeed in action, with example pages for each of the
optimizations, see our <a href="http://ngxpagespeed.com">demonstration site</a>.
## How to build
Follow the steps on <a
href="https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source">build
ngx_pagespeed from source</a>.
## How to use
Follow the steps on <a
href="https://developers.google.com/speed/pagespeed/module/configuration">PageSpeed
configuration</a>.
For feedback, questions, and to follow
the progress of the project:
- [ngx-pagespeed-discuss mailing
list](https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss)
- [ngx-pagespeed-announce mailing
list](https://groups.google.com/forum/#!forum/ngx-pagespeed-announce)
lib/ngx_pagespeed/config
-390
View File
@@ -1,390 +0,0 @@
# Copyright 2012 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Environment Variables (Optional):
# MOD_PAGESPEED_DIR: absolute path to the mod_pagespeed/src directory
# PSOL_BINARY: absolute path to pagespeed_automatic.a
# PSOL_BUILDTYPE: Release or Debug
mod_pagespeed_dir="${MOD_PAGESPEED_DIR:-unset}"
position_aux="${POSITION_AUX:-unset}"
psol_buildtype="${PSOL_BUILDTYPE:-unset}"
if [ "$mod_pagespeed_dir" = "unset" ] ; then
mod_pagespeed_dir="$ngx_addon_dir/psol/include"
build_from_source=false
if [ ! -e "$mod_pagespeed_dir" ] ; then
echo "ngx_pagespeed: pagespeed optimization library not found:"
psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
$ngx_addon_dir/PSOL_BINARY_URL)"
if [[ "$psol_binary_url" != https://* ]]; then
echo "
This is a development branch of ngx_pagespeed, which means there is no
precompiled PSOL library available to link against. Either build from a
release tag, like latest-beta, or build PSOL from source:
https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source"
exit 1
fi
echo "
You need to separately download the pagespeed library:
$ cd $ngx_addon_dir
$ wget $psol_binary_url
$ tar -xzvf $(basename $psol_binary_url) # expands to psol/
Or see the installation instructions:
https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source"
exit 1
fi
else
build_from_source=true
fi
os_name='unknown_os'
arch_name='unknown_arch'
uname_os=`uname`
uname_arch=`uname -m`
if [ $uname_os = 'Linux' ]; then
os_name='linux'
elif [ $uname_os = 'Darwin' ]; then
os_name='mac'
else
echo "OS not supported: $uname_os"
exit 1
fi
if [ $uname_arch = 'x86_64' -o $uname_arch = 'amd64' ]; then
arch_name='x64'
elif [ $uname_arch = 'x86_32' -o $uname_arch = 'i686' \
-o $uname_arch = 'i386' ]; then
arch_name='ia32'
else
echo "Architecture not supported: $uname_arch"
exit 1
fi
if [ "$psol_buildtype" = "unset" ] ; then
if [ "$NGX_DEBUG" = "YES" ]; then
buildtype=Debug
# If we're using a psol tarball that doesn't contain Debug/ (which is the case
# from 1.12 onward) then this will be overriden to buildtype=Release below.
else
buildtype=Release
fi
else
buildtype=$psol_buildtype
fi
# If the compiler is gcc, we want to use g++ to link, if at all possible,
# so that -static-libstdc++ works.
# Annoyingly, the feature test doesn't even use $LINK for linking, so that
# needs an explicit -lstdc++
pagespeed_libs=
ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
LINK=$ps_maybe_gpp
fi
pagespeed_libs="-lstdc++"
# The compiler needs to know that __sync_add_and_fetch_4 is ok,
# and this requires an instruction that didn't exist on i586 or i386.
if [ "$uname_arch" = "i686" ]; then
FLAG_MARCH='-march=i686'
fi
CFLAGS="$CFLAGS $FLAG_MARCH"
# For now, standardize on gcc-4.x ABI --- if we don't set this, people building
# with new gcc defaulting to gcc-5 C++11 ABI will have build trouble linking
# to our libpsol.a
# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
CC_OLD_TEST_FLAGS="$CC_TEST_FLAGS"
CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0 --std=c++11"
case "$NGX_GCC_VER" in
4.8*)
# On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs. This breaks
# on VerifySizesAreEqual in bit_cast in chromium/src/base/basictypes.h which
# has a typedef that is intentionally unused.
CFLAGS="$CFLAGS -Wno-unused-local-typedefs"
# On GCC 4.8 and above, we get the following compiler warning:
# chromium/src/base/memory/scoped_ptr.h:133:7: warning: declaration of ‘class scoped_ptr<C>’ [enabled by default]
# Based on discussion at http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54055,
# this is invalid code, but hasn't been fixed yet in chromium.
# Unfortunately, there also does not appear to be a flag for just disabling
# that warning, so we add Wno-error to override nginx's default -Werror
# option.
CFLAGS="$CFLAGS -Wno-error"
;;
esac
# workaround for a bug in nginx-1.9.11, see:
# http://hg.nginx.org/nginx/rev/ff1e625ae55b
NGX_VERSION=`grep nginx_version src/core/nginx.h | sed -e 's/^.* \(.*\)$/\1/'`
if [ "$NGX_VERSION" = "1009011" ]; then
CFLAGS="$CFLAGS -Wno-write-strings"
fi
if [ "$WNO_ERROR" = "YES" ]; then
CFLAGS="$CFLAGS -Wno-error"
fi
psol_binary="${PSOL_BINARY:-unset}"
if [ "$psol_binary" = "unset" ] ; then
if $build_from_source ; then
psol_binary="\
$mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
else
if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
echo "
You have set --with-debug for building nginx, but precompiled Debug binaries for
PSOL, which ngx_pagespeed depends on, aren't available. If you're trying to
debug PSOL you need to build it from source. If you just want to run nginx with
debug-level logging you can use the Release binaries."
echo -n "
Use the available Release binaries?"
read -p " [Y/n] " yn
if [[ "$yn" == N* || "$yn" == n* ]]; then
echo "Cancelled."
exit 1
fi
buildtype=Release
fi
psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
psol_binary="$psol_library_dir/pagespeed_automatic.a"
fi
fi
echo "mod_pagespeed_dir=$mod_pagespeed_dir"
echo "build_from_source=$build_from_source"
ngx_feature="psol"
ngx_feature_name=""
ngx_feature_run=no
ngx_feature_incs="
#include \"pagespeed/kernel/base/string.h\"
#include \"pagespeed/kernel/base/string_writer.h\"
#include \"pagespeed/kernel/base/null_message_handler.h\"
#include \"pagespeed/kernel/html/html_parse.h\"
#include \"pagespeed/kernel/html/html_writer_filter.h\"
"
pagespeed_include="\
$mod_pagespeed_dir \
$mod_pagespeed_dir/third_party/chromium/src \
$mod_pagespeed_dir/third_party/google-sparsehash/src/src \
$mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
$mod_pagespeed_dir/third_party/grpc/src/include \
$mod_pagespeed_dir/third_party/protobuf/src/src \
$mod_pagespeed_dir/third_party/re2/src \
$mod_pagespeed_dir/out/$buildtype/obj/gen \
$mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
$mod_pagespeed_dir/third_party/apr/src/include \
$mod_pagespeed_dir/third_party/aprutil/src/include \
$mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
$mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
$mod_pagespeed_dir/url"
ngx_feature_path="$pagespeed_include"
pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm -luuid"
ngx_feature_libs="$pagespeed_libs"
ngx_feature_test="
GoogleString output_buffer;
net_instaweb::StringWriter write_to_string(&output_buffer);
net_instaweb::NullMessageHandler handler;
net_instaweb::HtmlParse html_parse(&handler);
net_instaweb::HtmlWriterFilter html_writer_filter(&html_parse);
html_writer_filter.set_writer(&write_to_string);
html_parse.AddFilter(&html_writer_filter);
html_parse.StartParse(\"http:example.com\");
html_parse.ParseText(
\"<html ><body ><h1 >Test</h1 ><p>Test Text</p></body></html>\n\");
html_parse.FinishParse();
printf(\"parsed as: %s\", output_buffer.c_str())"
# Test whether we have pagespeed and can compile and link against it.
. "$ngx_addon_dir/cpp_feature"
if [ $ngx_found = no ]; then
cat << END
$0: error: module ngx_pagespeed requires the pagespeed optimization library.
Look in $PWD/$NGX_AUTOCONF_ERR for more details.
END
exit 1
fi
ps_src="$ngx_addon_dir/src"
ngx_addon_name=ngx_pagespeed
NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
$ps_src/log_message_handler.h \
$ps_src/ngx_base_fetch.h \
$ps_src/ngx_caching_headers.h \
$ps_src/ngx_event_connection.h \
$ps_src/ngx_fetch.h \
$ps_src/ngx_gzip_setter.h \
$ps_src/ngx_list_iterator.h \
$ps_src/ngx_message_handler.h \
$ps_src/ngx_pagespeed.h \
$ps_src/ngx_rewrite_driver_factory.h \
$ps_src/ngx_rewrite_options.h \
$ps_src/ngx_server_context.h \
$ps_src/ngx_url_async_fetcher.h \
$psol_binary"
NPS_SRCS=" \
$ps_src/log_message_handler.cc \
$ps_src/ngx_base_fetch.cc \
$ps_src/ngx_caching_headers.cc \
$ps_src/ngx_event_connection.cc \
$ps_src/ngx_fetch.cc \
$ps_src/ngx_gzip_setter.cc \
$ps_src/ngx_list_iterator.cc \
$ps_src/ngx_message_handler.cc \
$ps_src/ngx_pagespeed.cc \
$ps_src/ngx_rewrite_driver_factory.cc \
$ps_src/ngx_rewrite_options.cc \
$ps_src/ngx_server_context.cc \
$ps_src/ngx_url_async_fetcher.cc"
# Save our sources in a separate var since we may need it in config.make
PS_NGX_SRCS="$NGX_ADDON_SRCS \
$NPS_SRCS"
# Make pagespeed run immediately before gzip and Brotli.
if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
next=ngx_http_brotli_filter_module
elif [ $HTTP_GZIP = YES ]; then
next=ngx_http_gzip_filter_module
else
next=ngx_http_range_header_filter_module
fi
if [ -n "$ngx_module_link" ]; then
# nginx-1.9.11+
ngx_module_type=HTTP_FILTER
ngx_module_name="ngx_pagespeed ngx_pagespeed_etag_filter"
ngx_module_incs="$ngx_feature_path"
ngx_module_deps=
ngx_module_srcs="$NPS_SRCS"
ngx_module_libs="$ngx_feature_libs"
ngx_module_order="ngx_http_range_header_filter_module\
ngx_pagespeed_etag_filter\
ngx_http_gzip_filter_module \
ngx_http_brotli_filter_module \
ngx_pagespeed \
ngx_http_postpone_filter_module \
ngx_http_ssi_filter_module \
ngx_http_charset_filter_module \
ngx_http_xslt_filter_module \
ngx_http_image_filter_module \
ngx_http_sub_filter_module \
ngx_http_addition_filter_module \
ngx_http_gunzip_filter_module \
ngx_http_userid_filter_module \
ngx_http_headers_filter_module"
. auto/module
if [ $ngx_module_link != DYNAMIC ]; then
# ngx_module_order doesn't work with static modules,
# so we must re-order filters here.
if [ "$position_aux" = "true" ] ; then
HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
else
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
| sed "s/ngx_pagespeed//" \
| sed "s/$next/$next ngx_pagespeed/")
fi
# Make the etag header filter run immediately before range header filter.
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
| sed "s/ngx_pagespeed_etag_filter//" \
| sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
else
if [ "$position_aux" = "true" ] ; then
ngx_module_type=HTTP_AUX_FILTER
ngx_module_order=""
fi
fi
else
CORE_LIBS="$CORE_LIBS $pagespeed_libs"
CORE_INCS="$CORE_INCS $pagespeed_include"
NGX_ADDON_SRCS="$PS_NGX_SRCS"
if [ "$position_aux" = "true" ] ; then
HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
else
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES | sed "s/$next/$next $ngx_addon_name/")
fi
# Make the etag header filter run immediately before range header filter.
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
fi
echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
# Test whether the compiler is compatible
ngx_feature="psol-compiler-compat"
ngx_feature_name=""
ngx_feature_run=no
ngx_feature_incs=""
ngx_feature_path=""
ngx_feature_libs="-lstdc++"
ngx_feature_test="
#if defined(__clang__) && defined(__GLIBCXX__)
// See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
// for a list of various values of __GLIBCXX__. Note that they're not monotonic
// with respect to version numbers.
#if __GLIBCXX__ == 20120322 || __GLIBCXX__ == 20120614
#error \"clang is using libstdc++ 4.7.0 or 4.7.1, which can cause binary incompatibility.\"
#endif
#endif
#if !defined(__clang__) && defined(__GNUC__)
#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
#error \"GCC < 4.8 no longer supported. Please use gcc >= 4.8 or clang >= 3.3\"
#endif
#endif
#if defined(__clang__)
#if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3)
#error \"Please use gcc >= 4.8 or clang >= 3.3\"
#endif
#endif
"
. "$ngx_addon_dir/cpp_feature"
if [ $ngx_found = no ]; then
cat << END
$0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
Look in $PWD/$NGX_AUTOCONF_ERR for more details.
END
exit 1
fi
have=NGX_PAGESPEED . auto/have
CC_TEST_FLAGS="$CC_OLD_TEST_FLAGS"
lib/ngx_pagespeed/config.make
-7
View File
@@ -1,7 +0,0 @@
# Since nginx build system doesn't normally do C++, there is no CXXFLAGS for us
# to touch, and compilers are understandably unhappy with --std=c++11 on C
# files. Hence, we hack the makefile to add it for just our sources.
for ps_src_file in $PS_NGX_SRCS; do
ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
done
lib/ngx_pagespeed/cpp_feature
-122
View File
@@ -1,122 +0,0 @@
# Copyright (C) Igor Sysoev
# Copyright (C) Nginx, Inc.
# 2012-10-01 Modified from auto/feature by jefftk to support c++ test files.
echo $ngx_n "checking for $ngx_feature ...$ngx_c"
cat << END >> $NGX_AUTOCONF_ERR
----------------------------------------
checking for $ngx_feature
END
ngx_found=no
if test -n "$ngx_feature_name"; then
ngx_have_feature=`echo $ngx_feature_name \
| tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ`
fi
if test -n "$ngx_feature_path"; then
for ngx_temp in $ngx_feature_path; do
ngx_feature_inc_path="$ngx_feature_inc_path -I $ngx_temp"
done
fi
cat << END > $NGX_AUTOTEST.cc
#include <sys/types.h>
$NGX_INCLUDE_UNISTD_H
$ngx_feature_incs
int main() {
$ngx_feature_test;
return 0;
}
END
ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path \
-o $NGX_AUTOTEST $NGX_AUTOTEST.cc $NGX_TEST_LD_OPT $ngx_feature_libs"
ngx_feature_inc_path=
eval "/bin/sh -c \"$ngx_test\" >> $NGX_AUTOCONF_ERR 2>&1"
if [ -x $NGX_AUTOTEST ]; then
case "$ngx_feature_run" in
yes)
# /bin/sh is used to intercept "Killed" or "Abort trap" messages
if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
echo " found"
ngx_found=yes
if test -n "$ngx_feature_name"; then
have=$ngx_have_feature . auto/have
fi
else
echo " found but is not working"
fi
;;
value)
# /bin/sh is used to intercept "Killed" or "Abort trap" messages
if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
echo " found"
ngx_found=yes
cat << END >> $NGX_AUTO_CONFIG_H
#ifndef $ngx_feature_name
#define $ngx_feature_name `$NGX_AUTOTEST`
#endif
END
else
echo " found but is not working"
fi
;;
bug)
# /bin/sh is used to intercept "Killed" or "Abort trap" messages
if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
echo " not found"
else
echo " found"
ngx_found=yes
if test -n "$ngx_feature_name"; then
have=$ngx_have_feature . auto/have
fi
fi
;;
*)
echo " found"
ngx_found=yes
if test -n "$ngx_feature_name"; then
have=$ngx_have_feature . auto/have
fi
;;
esac
else
echo " not found"
echo "----------" >> $NGX_AUTOCONF_ERR
cat $NGX_AUTOTEST.cc >> $NGX_AUTOCONF_ERR
echo "----------" >> $NGX_AUTOCONF_ERR
echo $ngx_test >> $NGX_AUTOCONF_ERR
echo "----------" >> $NGX_AUTOCONF_ERR
fi
rm $NGX_AUTOTEST*
lib/ngx_pagespeed/scripts/build_ngx_pagespeed.sh
-814
View File
@@ -1,814 +0,0 @@
#!/bin/bash
#
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function usage() {
echo "
Usage: build_ngx_pagespeed.sh [options]
Installs ngx_pagespeed and its dependencies. Can optionally build and install
nginx as well. Can be run either as:
bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options]
Or:
git clone git@github.com:pagespeed/ngx_pagespeed.git
cd ngx_pagespeed/
git checkout <branch>
scripts/build_ngx_pagespeed.sh [options]
Options:
-v, --ngx-pagespeed-version <ngx_pagespeed version>
What version of ngx_pagespeed to build. Valid options include:
* latest-beta
* latest-stable
* a version number, such as 1.11.33.4
If you don't specify a version, defaults to latest-stable unless --devel
is specified, in which case it defaults to master.
This option doesn't make sense if we're running within an existing
ngx_pagespeed checkout.
-n, --nginx-version <nginx version>
What version of nginx to build. If not set, this script only prepares the
ngx_pagespeed module, and expects you to handle including it when you
build nginx.
If you pass in 'latest' then this script scrapes the nginx download page
and attempts to determine the latest version automatically.
-m, --dynamic-module
Build ngx_pagespeed as a dynamic module.
-b, --builddir <directory>
Where to build. Defaults to \$HOME.
-p, --no-deps-check
By default, this script checks for the packages it depends on and tries to
install them. If you have installed dependencies from source or are on a
non-deb non-rpm system, this won't work. In that case, install the
dependencies yourself and pass --no-deps-check.
-s, --psol-from-source
Build PSOL from source instead of downloading a pre-built binary module.
-l, --devel
Sets up a development environment in ngx_pagespeed/nginx, building with
testing-only dependencies. Includes --psol-from-source, conflicts with
--nginx-version. Uses a 'git clone' checkout for ngx_pagespeed and nginx
instead of downloading a tarball.
-t, --build-type
When building PSOL from source, what to tell it for BUILD_TYPE. Defaults
to 'Release' unless --devel is set in which case it defaults to 'Debug'.
-y, --assume-yes
Assume the answer to all prompts is 'yes, please continue'. Intended for
automated usage, such as buildbots.
-a, --additional-nginx-configure-arguments
When running ./configure for nginx, you may want to specify additional
arguments, such as --with-http_ssl_module. By default this script will
pause and prompt you for them, but this option lets you pass them in. For
example, you might do:
-a '--with-http_ssl_module --with-cc-opt=\"-I /usr/local/include\"'
-d, --dryrun
Don't make any changes to the system, just print what changes you
would have made.
-h, --help
Print this message and exit."
}
RED=31
GREEN=32
YELLOW=33
function begin_color() {
color="$1"
echo -e -n "\e[${color}m"
}
function end_color() {
echo -e -n "\e[0m"
}
function echo_color() {
color="$1"
shift
begin_color "$color"
echo "$@"
end_color
}
function error() {
local error_message="$@"
echo_color "$RED" -n "Error: " >&2
echo "$@" >&2
}
# Prints an error message and exits with an error code.
function fail() {
error "$@"
# Normally I'd use $0 in "usage" here, but since most people will be running
# this via curl, that wouldn't actually give something useful.
echo >&2
echo "For usage information, run this script with --help" >&2
exit 1
}
function status() {
echo_color "$GREEN" "$@"
}
# Intended to be called as:
# bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) <args>
# If we set -e or -u then users of this script will see it silently exit on
# failure. Instead we need to check the exit status of each command manually.
# The run function handles exit-status checking for system-changing commands.
# Additionally, this allows us to easily have a dryrun mode where we don't
# actually make any changes.
INITIAL_ENV=$(printenv | sort)
function run() {
if "$DRYRUN"; then
echo_color "$YELLOW" -n "would run"
echo " $@"
env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort))
if [ -n "$env_differences" ]; then
echo " with the following additional environment variables:"
echo "$env_differences" | sed 's/^/ /'
fi
else
if ! "$@"; then
error "Failure running '$@', exiting."
exit 1
fi
fi
}
function redhat_is_installed() {
local package_name="$1"
rpm -qa $package_name | grep -q .
}
function debian_is_installed() {
local package_name="$1"
dpkg -l $package_name | grep ^ii | grep -q .
}
function version_sort() {
# We'd rather use sort -V, but that's not available on Centos 5. This works
# for versions in the form A.B.C.D or shorter, which is enough for our use.
sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g
}
# Compare two numeric versions in the form "A.B.C". Works with version numbers
# having up to four components, since that's enough to handle both nginx (3) and
# ngx_pagespeed (4).
function version_older_than() {
local test_version="$1"
local compare_to="$2"
local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1)
test "$older_version" != "$compare_to"
}
function determine_latest_nginx_version() {
# Scrape nginx's download page to try to find the most recent nginx version.
nginx_download_url="https://nginx.org/en/download.html"
function report_error() {
fail "
Couldn't automatically determine the latest nginx version: failed to $@
$nginx_download_url"
}
nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \
report_error "download"
download_refs=$(echo "$nginx_download_page" | \
grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \
report_error "parse"
versions_available=$(echo "$download_refs" | \
sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \
report_error "extract versions from"
latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \
report_error "determine latest version from"
if version_older_than "$latest_version" "1.11.4"; then
fail "
Expected the latest version of nginx to be at least 1.11.4 but found
$latest_version on $nginx_download_url"
fi
echo "$latest_version"
}
# Usage:
# install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ...
#
# install_pkg_cmd is a command to install a dependency
# is_pkg_installed_cmd is a command that returns true if the dependency is
# already installed
# each dependency is a package name
function install_dependencies() {
local install_pkg_cmd="$1"
local is_pkg_installed_cmd="$2"
shift 2
local missing_dependencies=""
for package_name in "$@"; do
if ! $is_pkg_installed_cmd $package_name; then
missing_dependencies+="$package_name "
fi
done
if [ -n "$missing_dependencies" ]; then
status "Detected that we're missing the following depencencies:"
echo " $missing_dependencies"
status "Installing them:"
run sudo $install_pkg_cmd $missing_dependencies
fi
}
function gcc_too_old() {
# We need gcc >= 4.8
local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}')
if [ "$gcc_major_version" -lt 4 ]; then
return 0 # too old
elif [ "$gcc_major_version" -gt 4 ]; then
return 1 # plenty new
fi
# It's gcc 4.x, check if x >= 8:
local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}')
test "$gcc_minor_version" -lt 8
}
function continue_or_exit() {
if "$ASSUME_YES"; then
return
fi
local prompt="$1"
echo_color "$YELLOW" -n "$prompt"
read -p " [Y/n] " yn
if [[ "$yn" == N* || "$yn" == n* ]]; then
echo "Cancelled."
exit 0
fi
}
# If a string is very simple we don't need to quote it. But we should quote
# everything else to be safe.
function needs_quoting() {
echo "$@" | grep -q '[^a-zA-Z0-9./_=-]'
}
function escape_for_quotes() {
echo "$@" | sed -e 's~\\~\\\\~g' -e "s~'~\\\\'~g"
}
function quote_arguments() {
local argument_str=""
for argument in "$@"; do
if [ -n "$argument_str" ]; then
argument_str+=" "
fi
if needs_quoting "$argument"; then
argument="'$(escape_for_quotes "$argument")'"
fi
argument_str+="$argument"
done
echo "$argument_str"
}
function build_ngx_pagespeed() {
getopt --test
if [ "$?" != 4 ]; then
# Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect
# this to be hit in practice on systems that are actually able to run
# PageSpeed.
fail "Your version of getopt is too old. Exiting with no changes made."
fi
opts=$(getopt -o v:n:mb:pslt:ya:dh \
--longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
--longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \
--longoptions assume-yes,additional-nginx-configure-arguments:,dryrun,help \
-n "$(basename "$0")" -- "$@")
if [ $? != 0 ]; then
usage
exit 1
fi
eval set -- "$opts"
NPS_VERSION="DEFAULT"
NGINX_VERSION=""
BUILDDIR="$HOME"
DO_DEPS_CHECK=true
PSOL_FROM_SOURCE=false
DEVEL=false
BUILD_TYPE=""
ASSUME_YES=false
DRYRUN=false
DYNAMIC_MODULE=false
while true; do
case "$1" in
-v | --ngx-pagespeed-version) shift
NPS_VERSION="$1"
shift
;;
-n | --nginx-version) shift
NGINX_VERSION="$1"
shift
;;
-m | --dynamic-module) shift
DYNAMIC_MODULE=true
;;
-b | --builddir) shift
BUILDDIR="$1"
shift
;;
-p | --no-deps-check) shift
DO_DEPS_CHECK=false
;;
-s | --psol-from-source) shift
PSOL_FROM_SOURCE=true
;;
-l | --devel) shift
DEVEL=true
;;
-t | --build-type) shift
BUILD_TYPE="$1"
shift
;;
-y | --assume-yes) shift
ASSUME_YES="true"
;;
-a | --additional-nginx-configure-arguments) shift
ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS="$1"
shift
;;
-d | --dryrun) shift
DRYRUN="true"
;;
-h | --help) shift
usage
exit 0
;;
--) shift
break
;;
*)
echo "Invalid argument: $1"
usage
exit 1
;;
esac
done
USE_GIT_CHECKOUT="$DEVEL"
ALREADY_CHECKED_OUT=false
if [ -e PSOL_BINARY_URL ]; then
status "Detected that we're running in an existing ngx_pagespeed checkout."
USE_GIT_CHECKOUT=true
ALREADY_CHECKED_OUT=true
fi
if "$ALREADY_CHECKED_OUT"; then
if [ "$NPS_VERSION" != "DEFAULT" ]; then
fail \
"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout."
fi
elif [ "$NPS_VERSION" = "DEFAULT" ]; then
if "$DEVEL"; then
NPS_VERSION="master"
else
NPS_VERSION="latest-stable"
fi
fi
if [ ! -d "$BUILDDIR" ]; then
fail "Told to build in $BUILDDIR, but that directory doesn't exist."
fi
BUILD_NGINX=false
if [ -n "$NGINX_VERSION" ]; then
BUILD_NGINX=true
fi
if "$DEVEL"; then
PSOL_FROM_SOURCE=true
BUILD_NGINX=true
if [ -n "$NGINX_VERSION" ]; then
fail \
"The --devel argument conflicts with --nginx. In devel mode we use the version of nginx that's included as a submodule."
fi
if "$DYNAMIC_MODULE"; then
fail "Can't currently build a dynamic module in --devel mode."
fi
fi
if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then
if "$DEVEL"; then
BUILD_TYPE="Debug"
else
BUILD_TYPE="Release"
fi
elif [ -n "$BUILD_TYPE" ]; then
fail "Setting --build-type requires --psol-from-source or --devel."
fi
if [ "$NGINX_VERSION" = "latest" ]; then
# When this function fails it prints the debugging information needed first
# to stderr.
NGINX_VERSION=$(determine_latest_nginx_version) || exit 1
fi
if "$DYNAMIC_MODULE"; then
# Check that ngx_pagespeed and nginx are recent enough to support dynamic
# modules. Unfortunately NPS_VERSION might be a tag, in which case we don't
# know. If it's not a numeric version number, then assume it's recent
# enough and if it's not they'll get an ugly compilation error later.
# Luckily 1.10.33.5 was a while ago now.
#
# I'd like to use =~ here, but they changed syntax between v3 and v4 (quotes
# moved from mandatory to optional to prohibited).
if [[ "${NPS_VERSION#*[^0-9.]}" = "$NPS_VERSION" ]] &&
version_older_than "$NPS_VERSION" "1.10.33.5"; then
fail "
You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
fi
if [ ! -z "NGINX_VERSION" ]; then
if version_older_than "$NGINX_VERSION" "1.9.13"; then
fail "
You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't
add support for dynamic modules in a way compatible with ngx_pagespeed until
1.9.13."
fi
fi
fi
if "$DRYRUN"; then
TEMPDIR="/tmp/output-of-mktemp"
else
TEMPDIR=$(mktemp -d)
function cleanup_tempdir {
rm -rf "$TEMPDIR"
}
trap cleanup_tempdir EXIT
fi
extra_flags=()
# Now make sure our dependencies are installed.
if "$DO_DEPS_CHECK"; then
if [ -f /etc/debian_version ]; then
status "Detected debian-based distro."
install_dependencies "apt-get install" debian_is_installed \
build-essential zlib1g-dev libpcre3 libpcre3-dev unzip uuid-dev
if gcc_too_old; then
if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
status "Detected that gcc is older than 4.8. Installing gcc-mozilla"
status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
status "affect your global gcc installation."
run sudo apt-get install gcc-mozilla
fi
extra_flags=("--with-cc=/usr/lib/gcc-mozilla/bin/gcc" \
"--with-ld-opt=-static-libstdc++")
fi
elif [ -f /etc/redhat-release ]; then
status "Detected redhat-based distro."
install_dependencies "yum install" redhat_is_installed \
gcc-c++ pcre-devel zlib-devel make unzip wget libuuid-devel
if gcc_too_old; then
if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
redhat_major_version=$(
cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1)
if [ "$redhat_major_version" == 5 ]; then
slc_version=5
elif [ "$redhat_major_version" == 6 ]; then
slc_version=6
else
fail "
Unexpected major version $redhat_major_version in /etc/redhat-release:
$(cat /etc/redhat-release) Expected 5 or 6."
fi
status "Detected that gcc is older than 4.8. Scientific Linux"
status "provides a gcc package that installs gcc-4.8 into /opt/ and"
status "doesn't affect your global gcc installation."
slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
run sudo wget "$slc_key" -O "$slc_key_out"
run sudo rpm --import "$slc_key_out"
repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo"
if [ -e "$repo_fname" ]; then
fail "Expected $repo_fname not to exist; aborting."
fi
repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/"
repo_url+="/docs/repository/cern/devtoolset/"
repo_url+="slc${slc_version}-devtoolset.repo"
run sudo wget -O "$repo_fname" "$repo_url"
run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils
fi
extra_flags=("--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc")
fi
else
fail "
This doesn't appear to be a deb-based distro or an rpm-based one. Not going to
be able to install dependencies. Please install dependencies manually and rerun
with --no-deps-check."
fi
status "Operating system dependencies are all set."
else
status "Not checking whether operating system dependencies are installed."
fi
function delete_if_already_exists() {
if "$DRYRUN"; then return; fi
local directory="$1"
if [ -d "$directory" ]; then
if [ ${#directory} -lt 8 ]; then
fail "
Not deleting $directory; name is suspiciously short. Something is wrong."
fi
continue_or_exit "OK to delete $directory?"
run rm -rf "$directory"
fi
}
# In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
# but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION
if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
# We've been given a numeric version number. This has an associated tag
# in the form vVERSION-beta.
tag_name="v${NPS_VERSION}-beta"
nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
else
# We've been given a tag name, like latest-beta. Download that directly.
tag_name="$NPS_VERSION"
nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
fi
install_dir="this-only-makes-sense-in-devel-mode"
if "$USE_GIT_CHECKOUT"; then
# We're either doing a --devel build, or someone is running us from an
# existing git checkout.
nps_module_dir="$PWD"
install_dir="$nps_module_dir"
if "$ALREADY_CHECKED_OUT"; then
run cd "$nps_module_dir"
else
status "Checking out ngx_pagespeed..."
run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \
"$nps_module_dir"
run cd "$nps_module_dir"
run git checkout "$tag_name"
fi
submodules_dir="$nps_module_dir/testing-dependencies"
if "$DEVEL"; then
status "Downloading dependencies..."
run git submodule update --init --recursive
if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then
status "Switching submodules over to git protocol."
# This lets us push to github by public key.
for config in $(find .git/ -name config) ; do
run sed -i s~https://github.com/~git@github.com:~ $config ;
done
fi
fi
else
nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
status "Downloading ngx_pagespeed..."
run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded"
nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
delete_if_already_exists "$nps_module_dir"
status "Extracting ngx_pagespeed..."
run unzip -q "$nps_downloaded" -d "$BUILDDIR"
run cd "$nps_module_dir"
fi
MOD_PAGESPEED_DIR=""
PSOL_BINARY=""
if "$PSOL_FROM_SOURCE"; then
MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed"
git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR"
run pushd "$MOD_PAGESPEED_DIR"
if "$DEVEL"; then
if [ ! -d "$HOME/apache2" ]; then
run install/build_development_apache.sh 2.2 prefork
fi
cd devel
run make apache_debug_psol
PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
else
if "$DO_DEPS_CHECK"; then
skip_deps_arg=""
else
skip_deps_arg="--skip_deps"
fi
run install/build_psol.sh --skip_tests --skip_packaging "$skip_deps_arg"
PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
fi
run popd
else
# Now we need to figure out what precompiled version of PSOL to build
# ngx_pagespeed against.
if "$DRYRUN"; then
psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
elif [ -e PSOL_BINARY_URL ]; then
# Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
# where to look.
psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
if [[ "$psol_url" != https://* ]]; then
fail "Got bad psol binary location information: $psol_url"
fi
else
# For past releases we have to grep it from the config file. The url has
# always looked like this, and the config file has contained it since
# before we started tagging our ngx_pagespeed releases.
psol_url="$(grep -o \
"https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
if [ -z "$psol_url" ]; then
fail "Couldn't find PSOL url in $PWD/config"
fi
fi
status "Downloading PSOL binary..."
run wget "$psol_url"
status "Extracting PSOL..."
run tar -xzf $(basename "$psol_url") # extracts to psol/
fi
if "$DYNAMIC_MODULE"; then
add_module="--add-dynamic-module=$nps_module_dir"
else
add_module="--add-module=$nps_module_dir"
fi
configure_args=("$add_module" "${extra_flags[@]}")
if "$DEVEL"; then
configure_args=("${configure_args[@]}"
"--prefix=$install_dir/nginx"
"--add-module=$submodules_dir/ngx_cache_purge"
"--add-module=$submodules_dir/ngx_devel_kit"
"--add-module=$submodules_dir/set-misc-nginx-module"
"--add-module=$submodules_dir/headers-more-nginx-module"
"--with-ipv6"
"--with-http_v2_module")
if [ "$BUILD_TYPE" = "Debug" ]; then
configure_args=("${configure_args[@]}" "--with-debug")
fi
fi
echo
if ! "$BUILD_NGINX"; then
# Just prepare the module for them to install.
status "ngx_pagespeed is ready to be built against nginx."
echo "When running ./configure:"
if "$PSOL_FROM_SOURCE"; then
echo " Set the following environment variables:"
echo " MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR"
echo " PSOL_BINARY=$PSOL_BINARY"
fi
echo " Give ./configure the following arguments:"
echo " $(quote_arguments "${configure_args[@]}")"
echo
if [ ${#extra_flags[@]} -eq 0 ]; then
echo "If this is for integration with an already-built nginx, make sure"
echo "to include any other arguments you originally passed to"
echo "./configure. You can see these with 'nginx -V'."
else
echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
echo "on this platform, if you want to integrate ngx_pagespeed with an"
echo "already-built nginx you're going to need to rebuild your nginx with"
echo "those flags set."
fi
else
if "$DEVEL"; then
# Use the nginx we loaded as a submodule
nginx_dir="$submodules_dir/nginx"
configure_location="auto"
else
# Download and build the specified nginx version.
nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
nginx_fname="$TEMPDIR/$nginx_leaf"
status "Downloading nginx..."
run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
delete_if_already_exists "$nginx_dir"
status "Extracting nginx..."
run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
configure_location="."
fi
run cd "$nginx_dir"
configure=("$configure_location/configure" "${configure_args[@]}")
additional_configure_args=""
if [ -z "${ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS+x}" ]; then
if ! "$ASSUME_YES"; then
echo "About to build nginx. Do you have any additional ./configure"
echo "arguments you would like to set? For example, if you would like"
echo "to build nginx with https support give --with-http_ssl_module"
echo "If you don't have any, just press enter."
read -p "> " additional_configure_args
fi
else
additional_configure_args="$ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS"
fi
if [ -n "$additional_configure_args" ]; then
# Split additional_configure_args respecting any internal quotation.
# Otherwise things like --with-cc-opt='-foo -bar' won't work.
eval additional_configure_args=("$additional_configure_args")
configure=("${configure[@]}" "${additional_configure_args[@]}")
fi
echo "About to configure nginx with:"
echo " $(quote_arguments "${configure[@]}")"
continue_or_exit "Does this look right?"
MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
PSOL_BINARY="$PSOL_BINARY" \
run "${configure[@]}"
if ! "$DEVEL"; then
continue_or_exit "Build nginx?"
fi
run make
if "$DEVEL"; then
run make install
status "Nginx installed with ngx_pagespeed, and set up for development."
echo "To run tests:"
echo " cd $nps_module_dir"
echo " test/run_tests.sh"
echo
echo "To rebuild after changes:"
echo " scripts/rebuild.sh"
else
continue_or_exit "Install nginx?"
run sudo make install
echo
if "$DYNAMIC_MODULE"; then
echo "Nginx installed with ngx_pagespeed support available as a"
echo "loadable module."
echo
echo "To load the ngx_pagespeed module, you'll need to add:"
echo " load_module \"modules/ngx_pagespeed.so\";"
echo "at the top of your main nginx configuration file."
else
echo "Nginx installed with ngx_pagespeed support compiled-in."
fi
echo
echo "If this is a new installation you probably need an init script to"
echo "manage starting and stopping the nginx service. See:"
echo " http://wiki.nginx.org/InitScripts"
echo
echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
echo " https://developers.google.com/speed/pagespeed/module/configuration"
fi
fi
if "$DRYRUN"; then
echo_color "$YELLOW" "[this was a dry run; your system is unchanged]"
fi
}
# Start running things from a call at the end so if this script is executed
# after a partial download it doesn't do anything.
build_ngx_pagespeed "$@"
lib/ngx_pagespeed/scripts/format_binary_url.sh
-38
View File
@@ -1,38 +0,0 @@
#!/bin/bash
#
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
set -u
if [ $# -ne 1 ]; then
echo "Usage: $(basename $0) <url_file>" >&2
exit 1
fi
url_file=$1
if [ ! -e "$url_file" ]; then
echo "Url file '$url_file' missing!" >&2
fi
# The size names must match install/build_psol.sh in mod_pagespeed
if [ "$(uname -m)" = x86_64 ]; then
bit_size_name=x64
else
bit_size_name=ia32
fi
sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file
lib/ngx_pagespeed/scripts/pagespeed_libraries_generator.sh
-36
View File
@@ -1,36 +0,0 @@
#!/bin/bash
#
# Copyright 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
# supporting the canonicalize_javascript_libraries filter.
# Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532
#
# Usage:
# scripts/pagespeed_libraries_generator.sh > pagespeed_libraries.conf
#
# Then have nginx include that configuration file and enable the filter
# canonicalize_javascript_libraries.
#
# Author: vid@zippykid.com (Vid Luther)
# jefftk@google.com (Jeff Kaufman)
URL="https://github.com/pagespeed/mod_pagespeed/raw/master/"
URL+="net/instaweb/genfiles/conf/pagespeed_libraries.conf"
curl -L -s -S "$URL" \
| grep ModPagespeedLibrary \
| while read library size hash url ; do
echo " pagespeed Library $size $hash $url;"
done
lib/ngx_pagespeed/scripts/rebuild.sh
-35
View File
@@ -1,35 +0,0 @@
#!/bin/bash
#
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Usage: scripts/rebuild.sh
#
# After building with "scripts/build_ngx_pagespeed.sh --devel", if you make
# changes to ngx_pagespeed you'll need to rebuild it. The underlying commands
# aren't complicated, but it's faster to work if it's automated.
set -e # exit script if any command returns an error
set -u # exit the script if any variable is uninitialized
this_dir="$(dirname $0)"
cd "$this_dir/.."
nps_dir="$PWD"
cd "$nps_dir/testing-dependencies/mod_pagespeed/devel"
make apache_debug_psol
cd "$nps_dir/testing-dependencies/nginx/"
make
make install
lib/ngx_pagespeed/src/log_message_handler.cc
-114
View File
@@ -1,114 +0,0 @@
// Copyright 2010 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: jmarantz@google.com (Joshua Marantz)
// Author: sligocki@google.com (Shawn Ligocki)
// Author: jefftk@google.com (Jeff Kaufman)
// TODO(jefftk): share more of this code with apache's log_message_handler
#include "log_message_handler.h"
#include <unistd.h>
#include <limits>
#include <string>
#include "base/debug/debugger.h"
#include "base/debug/stack_trace.h"
#include "base/logging.h"
#include "net/instaweb/public/version.h"
#include "pagespeed/kernel/base/string_util.h"
// Make sure we don't attempt to use LOG macros here, since doing so
// would cause us to go into an infinite log loop.
#undef LOG
#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION
namespace {
ngx_log_t* ngx_log = NULL;
ngx_uint_t GetNgxLogLevel(int severity) {
switch (severity) {
case logging::LOG_INFO:
return NGX_LOG_INFO;
case logging::LOG_WARNING:
return NGX_LOG_WARN;
case logging::LOG_ERROR:
return NGX_LOG_ERR;
case logging::LOG_ERROR_REPORT:
case logging::LOG_FATAL:
return NGX_LOG_ALERT;
default: // For VLOG(s)
return NGX_LOG_DEBUG;
}
}
bool LogMessageHandler(int severity, const char* file, int line,
size_t message_start, const GoogleString& str) {
ngx_uint_t this_log_level = GetNgxLogLevel(severity);
GoogleString message = str;
if (severity == logging::LOG_FATAL) {
if (base::debug::BeingDebugged()) {
base::debug::BreakDebugger();
} else {
base::debug::StackTrace trace;
std::ostringstream stream;
trace.OutputToStream(&stream);
message.append(stream.str());
}
}
// Trim the newline off the end of the message string.
size_t last_msg_character_index = message.length() - 1;
if (message[last_msg_character_index] == '\n') {
message.resize(last_msg_character_index);
}
ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
net_instaweb::kModPagespeedVersion,
message.c_str());
if (severity == logging::LOG_FATAL) {
// Crash the process to generate a dump.
base::debug::BreakDebugger();
}
return true;
}
} // namespace
namespace net_instaweb {
namespace log_message_handler {
void Install(ngx_log_t* log_in) {
ngx_log = log_in;
logging::SetLogMessageHandler(&LogMessageHandler);
// All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
// level is DEBUG.
if (ngx_log->log_level >= NGX_LOG_DEBUG) {
logging::SetMinLogLevel(-2);
}
}
} // namespace log_message_handler
} // namespace net_instaweb
lib/ngx_pagespeed/src/log_message_handler.h
-44
View File
@@ -1,44 +0,0 @@
// Copyright 2010 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: jmarantz@google.com (Joshua Marantz)
// Author: sligocki@google.com (Shawn Ligocki)
// Author: jefftk@google.com (Jeff Kaufman)
#ifndef LOG_MESSAGE_HANDLER_H_
#define LOG_MESSAGE_HANDLER_H_
extern "C" {
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
#include <ngx_log.h>
}
namespace net_instaweb {
namespace log_message_handler {
// Install a log message handler that routes LOG() messages to the
// server error log. Should be called once at startup. If server blocks define
// their own logging files you would expect that LOG() messages would be routed
// appropriately, but because logging is all done with global variables this
// isn't possible.
void Install(ngx_log_t* log_in);
} // namespace log_message_handler
} // namespace net_instaweb
#endif // LOG_MESSAGE_HANDLER_H_
lib/ngx_pagespeed/src/ngx_base_fetch.cc
-360
View File
@@ -1,360 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Author: jefftk@google.com (Jeff Kaufman)
*/
#include "ngx_pagespeed.h" // Must come first, see comments in CollectHeaders.
#include <unistd.h> //for usleep
#include "ngx_base_fetch.h"
#include "ngx_event_connection.h"
#include "ngx_list_iterator.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "pagespeed/kernel/base/google_message_handler.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/posix_timer.h"
#include "pagespeed/kernel/http/response_headers.h"
namespace net_instaweb {
const char kHeadersComplete = 'H';
const char kFlush = 'F';
const char kDone = 'D';
NgxEventConnection* NgxBaseFetch::event_connection = NULL;
int NgxBaseFetch::active_base_fetches = 0;
NgxBaseFetch::NgxBaseFetch(StringPiece url,
ngx_http_request_t* r,
NgxServerContext* server_context,
const RequestContextPtr& request_ctx,
PreserveCachingHeaders preserve_caching_headers,
NgxBaseFetchType base_fetch_type,
const RewriteOptions* options)
: AsyncFetch(request_ctx),
url_(url.data(), url.size()),
request_(r),
server_context_(server_context),
options_(options),
need_flush_(false),
done_called_(false),
last_buf_sent_(false),
references_(2),
base_fetch_type_(base_fetch_type),
preserve_caching_headers_(preserve_caching_headers),
detached_(false),
suppress_(false) {
if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
__sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
}
NgxBaseFetch::~NgxBaseFetch() {
pthread_mutex_destroy(&mutex_);
__sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
}
bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
CHECK(event_connection == NULL) << "event connection already set";
event_connection = new NgxEventConnection(ReadCallback);
return event_connection->Init(cycle);
}
void NgxBaseFetch::Terminate() {
if (event_connection != NULL) {
GoogleMessageHandler handler;
PosixTimer timer;
int64 timeout_us = Timer::kSecondUs * 30;
int64 end_us = timer.NowUs() + timeout_us;
static unsigned int sleep_microseconds = 100;
handler.Message(
kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
NgxBaseFetch::active_base_fetches);
// Try to continue processing and get the active base fetch count to 0
// untill the timeout expires.
// TODO(oschaaf): This needs more work.
while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
event_connection->Drain();
usleep(sleep_microseconds);
}
if (NgxBaseFetch::active_base_fetches != 0) {
handler.Message(
kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
NgxBaseFetch::active_base_fetches);
}
// Close down the named pipe.
event_connection->Shutdown();
delete event_connection;
event_connection = NULL;
}
}
const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
switch(type) {
case kPageSpeedResource:
return "ps resource";
case kHtmlTransform:
return "html transform";
case kAdminPage:
return "admin page";
case kIproLookup:
return "ipro lookup";
case kPageSpeedProxy:
return "pagespeed proxy";
}
CHECK(false);
return "can't get here";
}
// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
// MessageHandler and use that.
void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
ngx_http_request_t* r = base_fetch->request();
bool detached = base_fetch->detached();
#if (NGX_DEBUG) // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
#endif
int refcount = base_fetch->DecrementRefCount();
#if (NGX_DEBUG)
ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
"pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
#endif
// If we ended up destructing the base fetch, or the request context is
// detached, skip this event.
if (refcount == 0 || detached) {
return;
}
ps_request_ctx_t* ctx = ps_get_request_context(r);
// If our request context was zeroed, skip this event.
// See https://github.com/pagespeed/ngx_pagespeed/issues/1081
if (ctx == NULL) {
// Should not happen normally, when it does this message will cause our
// system tests to fail.
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
"pagespeed [%p] skipping event: request context gone", r);
return;
}
// Normally we expect the sender to equal the active NgxBaseFetch instance.
DCHECK(data.sender == ctx->base_fetch);
// If someone changed our request context or NgxBaseFetch, skip processing.
if (data.sender != ctx->base_fetch) {
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
"pagespeed [%p] skipping event: event originating from disassociated"
" NgxBaseFetch instance.", r);
return;
}
int rc;
bool run_posted = true;
// If we are unlucky enough to have our connection finalized mid-ipro-lookup,
// we must enter a different flow. Also see ps_in_place_check_header_filter().
if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
&& r->connection->error) {
ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
"pagespeed [%p] request already finalized %d", r, r->count);
rc = NGX_ERROR;
run_posted = false;
} else {
rc = ps_base_fetch::ps_base_fetch_handler(r);
}
#if (NGX_DEBUG)
ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
"pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
r, rc, data.type);
#endif
ngx_connection_t* c = r->connection;
ngx_http_finalize_request(r, rc);
if (run_posted) {
// See http://forum.nginx.org/read.php?2,253006,253061
ngx_http_run_posted_requests(c);
}
}
void NgxBaseFetch::Lock() {
pthread_mutex_lock(&mutex_);
}
void NgxBaseFetch::Unlock() {
pthread_mutex_unlock(&mutex_);
}
bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
MessageHandler* handler) {
Lock();
buffer_.append(sp.data(), sp.size());
Unlock();
return true;
}
// should only be called in nginx thread
ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
CHECK(!(done_called_ && last_buf_sent_))
<< "CopyBufferToNginx() was called after the last buffer was sent";
// there is no buffer to send
if (!done_called_ && buffer_.empty()) {
*link_ptr = NULL;
return NGX_AGAIN;
}
int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
done_called_ /* send_last_buf */,
need_flush_);
need_flush_ = false;
if (rc != NGX_OK) {
return rc;
}
// Done with buffer contents now.
buffer_.clear();
if (done_called_) {
last_buf_sent_ = true;
return NGX_OK;
}
return NGX_AGAIN;
}
// There may also be a race condition if this is called between the last Write()
// and Done() such that we're sending an empty buffer with last_buf set, which I
// think nginx will reject.
ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
ngx_int_t rc;
Lock();
rc = CopyBufferToNginx(link_ptr);
Unlock();
return rc;
}
ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
// nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
// If a standard header is accidentally included before the nginx header,
// on a 32-bit system off_t will be 4 bytes and we don't assign all the
// bits of content_length_n. Sanity check that did not happen.
// This could use static_assert, but this file is not built with --std=c++11.
bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
const ResponseHeaders* pagespeed_headers = response_headers();
if (content_length_known()) {
headers_out->content_length = NULL;
headers_out->content_length_n = content_length();
}
return copy_response_headers_to_ngx(request_, *pagespeed_headers,
preserve_caching_headers_);
}
void NgxBaseFetch::RequestCollection(char type) {
if (suppress_) {
return;
}
// We must optimistically increment the refcount, and decrement it
// when we conclude we failed. If we only increment on a successfull write,
// there's a small chance that between writing and adding to the refcount
// both pagespeed and nginx will release their refcount -- destructing
// this NgxBaseFetch instance.
IncrementRefCount();
if (!event_connection->WriteEvent(type, this)) {
DecrementRefCount();
}
}
void NgxBaseFetch::HandleHeadersComplete() {
int status_code = response_headers()->status_code();
bool status_ok = (status_code != 0) && (status_code < 400);
if ((base_fetch_type_ != kIproLookup) || status_ok) {
// If this is a 404 response we need to count it in the stats.
if (response_headers()->status_code() == HttpStatus::kNotFound) {
server_context_->rewrite_stats()->resource_404_count()->Add(1);
}
}
RequestCollection(kHeadersComplete); // Headers available.
// For the IPRO lookup, supress notification of the nginx side here.
// If we send both the headerscomplete event and the one from done, nasty
// stuff will happen if we loose the race with with the nginx side destructing
// this base fetch instance.
if (base_fetch_type_ == kIproLookup && !status_ok) {
suppress_ = true;
}
}
bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
Lock();
need_flush_ = true;
Unlock();
RequestCollection(kFlush); // A new part of the response body is available
return true;
}
int NgxBaseFetch::DecrementRefCount() {
return DecrefAndDeleteIfUnreferenced();
}
int NgxBaseFetch::IncrementRefCount() {
return __sync_add_and_fetch(&references_, 1);
}
int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
// Creates a full memory barrier.
int r = __sync_add_and_fetch(&references_, -1);
if (r == 0) {
delete this;
}
return r;
}
void NgxBaseFetch::HandleDone(bool success) {
// TODO(jefftk): it's possible that instead of locking here we can just modify
// CopyBufferToNginx to only read done_called_ once.
CHECK(!done_called_) << "Done already called!";
Lock();
done_called_ = true;
Unlock();
RequestCollection(kDone);
DecrefAndDeleteIfUnreferenced();
}
bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
return OptionsAwareHTTPCacheCallback::IsCacheValid(
url_, *options_, request_context(), headers);
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_base_fetch.h
-192
View File
@@ -1,192 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
//
// Collects output from pagespeed and buffers it until nginx asks for it.
// Notifies nginx via NgxEventConnection to call ReadCallback() when
// the headers are computed, when a flush should be performed, and when done.
//
// - nginx creates a base fetch and passes it to a new proxy fetch.
// - The proxy fetch manages rewriting and thread complexity, and through
// several chained steps passes rewritten html to HandleWrite().
// - Written data is buffered.
// - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
// PSOL, events are written to NgxEventConnection which will end up being
// handled by ReadCallback() on nginx's thread.
// When applicable, request processing will be continued via a call to
// ps_base_fetch_handler().
// - ps_base_fetch_handler() will pull the header and body bytes from PSOL
// via CollectAccumulatedWrites() and write those to the module's output.
//
// This class is referred to in three places: the proxy fetch, nginx's request,
// and pending events written to the associated NgxEventConnection. It must stay
// alive until the proxy fetch and nginx request are finished, and no more
// events are pending.
// - The proxy fetch will call Done() to indicate this.
// - nginx will call Detach() when the associated request is handled
// completely (e.g. the request context is about to be destroyed).
// - ReadCallback() will call DecrementRefCount() on instances associated to
// events it handles.
//
// When the last reference is dropped, this class will delete itself.
//
// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
// with ApacheFetch, using a common base class.
#ifndef NGX_BASE_FETCH_H_
#define NGX_BASE_FETCH_H_
extern "C" {
#include <ngx_http.h>
}
#include <pthread.h>
#include "ngx_pagespeed.h"
#include "ngx_event_connection.h"
#include "ngx_server_context.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/http/headers.h"
namespace net_instaweb {
enum NgxBaseFetchType {
kIproLookup,
kHtmlTransform,
kPageSpeedResource,
kAdminPage,
kPageSpeedProxy
};
class NgxBaseFetch : public AsyncFetch {
public:
NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
NgxServerContext* server_context,
const RequestContextPtr& request_ctx,
PreserveCachingHeaders preserve_caching_headers,
NgxBaseFetchType base_fetch_type,
const RewriteOptions* options);
virtual ~NgxBaseFetch();
// Statically initializes event_connection, require for PSOL and nginx to
// communicate.
static bool Initialize(ngx_cycle_t* cycle);
// Attempts to finish up request processing queued up in the named pipe and
// PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
// is attempted.
// Statically terminates and NULLS event_connection.
static void Terminate();
static void ReadCallback(const ps_event_data& data);
// Puts a chain in link_ptr if we have any output data buffered. Returns
// NGX_OK on success, NGX_ERROR on errors. If there's no data to send, sends
// data only if Done() has been called. Indicates the end of output by
// setting last_buf on the last buffer in the chain.
//
// Sets link_ptr to a chain of as many buffers are needed for the output.
//
// Called by nginx in response to seeing a byte on the pipe.
ngx_int_t CollectAccumulatedWrites(ngx_chain_t** link_ptr);
// Copies response headers into headers_out.
//
// Called by nginx before calling CollectAccumulatedWrites() for the first
// time for resource fetches. Not called at all for proxy fetches.
ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);
// Called by nginx to decrement the refcount.
int DecrementRefCount();
// Called by pagespeed to increment the refcount.
int IncrementRefCount();
// Detach() is called when the nginx side releases this base fetch. It
// sets detached_ to true and decrements the refcount. We need to know
// this to be able to handle events which nginx request context has been
// released while the event was in-flight.
void Detach() { detached_ = true; DecrementRefCount(); }
bool detached() { return detached_; }
ngx_http_request_t* request() { return request_; }
NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
bool IsCachedResultValid(const ResponseHeaders& headers) override;
private:
virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
virtual bool HandleFlush(MessageHandler* handler);
virtual void HandleHeadersComplete();
virtual void HandleDone(bool success);
// Indicate to nginx that we would like it to call
// CollectAccumulatedWrites().
void RequestCollection(char type);
// Lock must be acquired first.
// Returns:
// NGX_ERROR: failure
// NGX_AGAIN: still has buffer to send, need to checkout link_ptr
// NGX_OK: done, HandleDone has been called
// Allocates an nginx buffer, copies our buffer_ contents into it, clears
// buffer_.
ngx_int_t CopyBufferToNginx(ngx_chain_t** link_ptr);
void Lock();
void Unlock();
// Called by Done() and Release(). Decrements our reference count, and if
// it's zero we delete ourself.
int DecrefAndDeleteIfUnreferenced();
static NgxEventConnection* event_connection;
// Live count of NgxBaseFetch instances that are currently in use.
static int active_base_fetches;
GoogleString url_;
ngx_http_request_t* request_;
GoogleString buffer_;
NgxServerContext* server_context_;
const RewriteOptions* options_;
bool need_flush_;
bool done_called_;
bool last_buf_sent_;
// How many active references there are to this fetch. Starts at two,
// decremented once when Done() is called and once when Detach() is called.
// Incremented for each event written by pagespeed for this NgxBaseFetch, and
// decremented on the nginx side for each event read for it.
int references_;
pthread_mutex_t mutex_;
NgxBaseFetchType base_fetch_type_;
PreserveCachingHeaders preserve_caching_headers_;
// Set to true just before the nginx side releases its reference
bool detached_;
bool suppress_;
DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
};
} // namespace net_instaweb
#endif // NGX_BASE_FETCH_H_
lib/ngx_pagespeed/src/ngx_caching_headers.cc
-51
View File
@@ -1,51 +0,0 @@
/*
* Copyright 2013 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#include "ngx_caching_headers.h"
#include "ngx_list_iterator.h"
#include "ngx_pagespeed.h"
namespace net_instaweb {
bool NgxCachingHeaders::Lookup(const StringPiece& key,
StringPieceVector* values) {
ngx_table_elt_t* header;
NgxListIterator it(&(request_->headers_out.headers.part));
while ((header = it.Next()) != NULL) {
if (header->hash != 0 && key == str_to_string_piece(header->key)) {
// This will be called multiple times if there are multiple headers with
// this name. Each time it will append to values.
SplitStringPieceToVector(str_to_string_piece(header->value), ",", values,
true /* omit empty strings */);
}
}
if (values->size() == 0) {
// No header found with this name.
return false;
}
for (int i = 0, n = values->size(); i < n; ++i) {
TrimWhitespace(&((*values)[i]));
}
return true;
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_caching_headers.h
-60
View File
@@ -1,60 +0,0 @@
/*
* Copyright 2013 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#ifndef NGX_CACHING_HEADERS_H_
#define NGX_CACHING_HEADERS_H_
extern "C" {
#include <ngx_http.h>
}
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/http/caching_headers.h"
namespace net_instaweb {
// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
// Needed so ps_header_filter can call GenerateDisabledCacheControl().
class NgxCachingHeaders : public CachingHeaders {
public:
explicit NgxCachingHeaders(ngx_http_request_t* request)
: CachingHeaders(request->headers_out.status),
request_(request) {
}
virtual bool Lookup(const StringPiece& key, StringPieceVector* values);
virtual bool IsLikelyStaticResourceType() const {
DCHECK(false); // not called in our use-case.
return false;
}
virtual bool IsCacheableResourceStatusCode() const {
DCHECK(false); // not called in our use-case.
return false;
}
private:
ngx_http_request_t* request_;
DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
};
} // namespace net_instaweb
#endif // NGX_CACHING_HEADERS_H_
lib/ngx_pagespeed/src/ngx_event_connection.cc
-180
View File
@@ -1,180 +0,0 @@
/*
* Copyright 2014 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: oschaaf@we-amp.com (Otto van der Schaaf)
extern "C" {
#include <ngx_channel.h>
}
#include "ngx_event_connection.h"
#include "pagespeed/kernel/base/google_message_handler.h"
#include "pagespeed/kernel/base/message_handler.h"
namespace net_instaweb {
NgxEventConnection::NgxEventConnection(callbackPtr callback)
: event_handler_(callback) {
}
bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
int file_descriptors[2];
if (pipe(file_descriptors) != 0) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
return false;
}
if (ngx_nonblocking(file_descriptors[0]) == -1) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
ngx_nonblocking_n "pagespeed: pipe[0] failed");
} else if (ngx_nonblocking(file_descriptors[1]) == -1) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
ngx_nonblocking_n "pagespeed: pipe[1] failed");
} else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
"pagespeed: failed to create connection.");
} else {
pipe_read_fd_ = file_descriptors[0];
pipe_write_fd_ = file_descriptors[1];
// Attempt to bump the pipe capacity, because running out of buffer space
// can potentially lead up to writes spinning on EAGAIN.
// See https://github.com/pagespeed/ngx_pagespeed/issues/1380
// TODO(oschaaf): Consider implementing a queueing mechanism for retrying
// failed writes.
#ifdef F_SETPIPE_SZ
fcntl(pipe_write_fd_, F_SETPIPE_SZ, 200*1024 /* minimal amount of bytes */);
#endif
return true;
}
close(file_descriptors[0]);
close(file_descriptors[1]);
return false;
}
bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
ngx_fd_t pipe_fd) {
// pipe_fd (the read side of the pipe will end up as c->fd on the
// underlying ngx_connection_t that gets created here)
ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
&NgxEventConnection::ReadEventHandler);
return rc == NGX_OK;
}
void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
ngx_int_t result = ngx_handle_read_event(ev, 0);
if (result != NGX_OK) {
CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
}
if (ev->timedout) {
ev->timedout = 0;
return;
}
if (!NgxEventConnection::ReadAndNotify(c->fd)) {
// This was copied from ngx_channel_handler(): for epoll, we need to call
// ngx_del_conn(). Sadly, no documentation as to why.
if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
ngx_del_conn(c, 0);
}
ngx_close_connection(c);
ngx_del_event(ev, NGX_READ_EVENT, 0);
}
}
// Deserialize ps_event_data's from the pipe as they become available.
// Subsequently do some bookkeeping, cleanup, and error checking to keep
// the mess out of ps_base_fetch_handler.
bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
while (true) {
// We read only one ps_event_data at a time for now:
// We can end up recursing all the way and end up calling ourselves here.
// If that happens in the middle of looping over multiple ps_event_data's we
// have obtained with read(), the results from the next read() will make us
// process events out of order. Which can give headaches.
// Alternatively, we could maintain a queue to make sure we process in
// sequence
ps_event_data data;
ngx_int_t size = read(fd, static_cast<void*>(&data), sizeof(data));
if (size == -1) {
if (errno == EINTR) {
continue;
// TODO(oschaaf): should we worry about spinning here?
} else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
return true;
}
}
if (size <= 0) {
return false;
}
data.connection->event_handler_(data);
return true;
}
}
bool NgxEventConnection::WriteEvent(void* sender) {
return WriteEvent('X' /* Anything char is fine */, sender);
}
bool NgxEventConnection::WriteEvent(char type, void* sender) {
ssize_t size = 0;
ps_event_data data;
ngx_memzero(&data, sizeof(data));
data.type = type;
data.sender = sender;
data.connection = this;
while (true) {
size = write(pipe_write_fd_,
static_cast<void*>(&data), sizeof(data));
if (size == sizeof(data)) {
return true;
} else if (size == -1) {
// TODO(oschaaf): should we worry about spinning here?
if (ngx_errno == EINTR || ngx_errno == EAGAIN
|| ngx_errno == EWOULDBLOCK) {
continue;
} else {
return false;
}
} else {
CHECK(false) << "pagespeed: unexpected return value from write(): "
<< size;
}
}
CHECK(false) << "Should not get here";
return false;
}
// Reads and processes what is available in the pipe.
void NgxEventConnection::Drain() {
NgxEventConnection::ReadAndNotify(pipe_read_fd_);
}
void NgxEventConnection::Shutdown() {
close(pipe_write_fd_);
close(pipe_read_fd_);
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_event_connection.h
-84
View File
@@ -1,84 +0,0 @@
/*
* Copyright 2014 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: oschaaf@we-amp.com (Otto van der Schaaf)
//
// NgxEventConnection implements a means to send events from other threads to
// nginx's event loop, and is implemented by a named pipe under the hood.
// A single instance is used by NgxBaseFetch, and one instance is created per
// NgxUrlAsyncFetcher when native fetching is on.
#ifndef NGX_EVENT_CONNECTION_H_
#define NGX_EVENT_CONNECTION_H_
extern "C" {
#include <ngx_http.h>
}
#include <pthread.h>
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/http/headers.h"
namespace net_instaweb {
class NgxEventConnection;
// Represents a single event that can be written to or read from the pipe.
// Technically, sender is the only data we need to send. type and connection are
// included to provide a means to trace the events along with some more
// info.
typedef struct {
char type;
void* sender;
NgxEventConnection* connection;
} ps_event_data;
// Handler signature for receiving events
typedef void (*callbackPtr)(const ps_event_data&);
// Abstracts a connection to nginx through which events can be written.
class NgxEventConnection {
public:
explicit NgxEventConnection(callbackPtr handler);
// Creates the file descriptors and ngx_connection_t required for event
// messaging between pagespeed and nginx.
bool Init(ngx_cycle_t* cycle);
// Shuts down the underlying file descriptors and connection created in Init()
void Shutdown();
// Constructs a ps_event_data and writes it to the underlying named pipe.
bool WriteEvent(char type, void* sender);
// Convenience overload for clients that have a single event type.
bool WriteEvent(void* sender);
// Reads and processes what is available in the named pipe's buffer.
void Drain();
private:
static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
static void ReadEventHandler(ngx_event_t* e);
static bool ReadAndNotify(ngx_fd_t fd);
callbackPtr event_handler_;
// We own these file descriptors
ngx_fd_t pipe_write_fd_;
ngx_fd_t pipe_read_fd_;
DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
};
} // namespace net_instaweb
#endif // NGX_EVENT_CONNECTION_H_
lib/ngx_pagespeed/src/ngx_fetch.cc
-951
View File
@@ -1,951 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: x.dinic@gmail.com (Junmin Xiong)
//
// - The fetch is started by the main thread.
// - Resolver event was hooked when a NgxFetch starting. It could
// lookup the IP of the domain asynchronously from the DNS server.
// - When NgxFetchResolveDone is called, It will create the request and the
// connection. Add the write and read event to the epoll structure.
// - The read handler parses the response. Add the response to the buffer at
// last.
// TODO(oschaaf): Currently the first applicable connection is picked from the
// pool when re-using connections. Perhaps it would be worth it to pick the one
// that was active the longest time ago to keep a larger pool available.
// TODO(oschaaf): style: reindent namespace according to google C++ style guide
// TODO(oschaaf): Retry mechanism for failures on a re-used k-a connection.
// Currently we don't think it's going to be an issue, see the comments at
// https://github.com/pagespeed/ngx_pagespeed/pull/781.
extern "C" {
#include <nginx.h>
}
#include "ngx_fetch.h"
#include "base/logging.h"
#include <algorithm>
#include <string>
#include <typeinfo>
#include <vector>
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/inflating_fetch.h"
#include "net/instaweb/public/version.h"
#include "net/instaweb/public/global_constants.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/condvar.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/pool.h"
#include "pagespeed/kernel/base/pool_element.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string_writer.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/thread_system.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/base/writer.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/response_headers_parser.h"
namespace net_instaweb {
NgxConnection::NgxConnectionPool NgxConnection::connection_pool;
PthreadMutex NgxConnection::connection_pool_mutex;
// Default keepalive 60s.
const int64 NgxConnection::keepalive_timeout_ms = 60000;
const GoogleString NgxConnection::ka_header =
StrCat("keep-alive ",
Integer64ToString(NgxConnection::keepalive_timeout_ms));
NgxConnection::NgxConnection(MessageHandler* handler,
int max_keepalive_requests) {
c_ = NULL;
max_keepalive_requests_ = max_keepalive_requests;
handler_ = handler;
// max_keepalive_requests specifies the number of http requests that are
// allowed to be performed over a single connection. So, a
// max_keepalive_requests of 1 effectively disables keepalive.
keepalive_ = max_keepalive_requests_ > 1;
}
NgxConnection::~NgxConnection() {
CHECK(c_ == NULL) << "NgxConnection: Underlying connection should be NULL";
}
void NgxConnection::Terminate() {
for (NgxConnectionPool::iterator p = connection_pool.begin();
p != connection_pool.end(); ++p) {
NgxConnection* nc = *p;
ngx_close_connection(nc->c_);
nc->c_ = NULL;
delete nc;
}
connection_pool.Clear();
}
NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
MessageHandler* handler,
int max_keepalive_requests) {
NgxConnection* nc;
{
ScopedMutex lock(&NgxConnection::connection_pool_mutex);
for (NgxConnectionPool::iterator p = connection_pool.begin();
p != connection_pool.end(); ++p) {
nc = *p;
if (ngx_memn2cmp(static_cast<u_char*>(nc->sockaddr_),
reinterpret_cast<u_char*>(pc->sockaddr),
nc->socklen_, pc->socklen) == 0) {
CHECK(nc->c_->idle) << "Pool should only contain idle connections!";
nc->c_->idle = 0;
nc->c_->log = pc->log;
nc->c_->read->log = pc->log;
nc->c_->write->log = pc->log;
if (nc->c_->pool != NULL) {
nc->c_->pool->log = pc->log;
}
if (nc->c_->read->timer_set) {
ngx_del_timer(nc->c_->read);
}
connection_pool.Remove(nc);
ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
"NgxFetch: re-using connection %p (pool size: %l)",
nc, connection_pool.size());
return nc;
}
}
}
int rc = ngx_event_connect_peer(pc);
if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
return NULL;
}
// NgxConnection deletes itself if NgxConnection::Close()
nc = new NgxConnection(handler, max_keepalive_requests);
nc->SetSock(reinterpret_cast<u_char*>(pc->sockaddr), pc->socklen);
nc->c_ = pc->connection;
return nc;
}
void NgxConnection::Close() {
bool removed_from_pool = false;
{
ScopedMutex lock(&NgxConnection::connection_pool_mutex);
for (NgxConnectionPool::iterator p = connection_pool.begin();
p != connection_pool.end(); ++p) {
if (*p == this) {
// When we get here, that means that the connection either has timed
// out or has been closed remotely.
connection_pool.Remove(this);
ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
"NgxFetch: removed connection %p (pool size: %l)",
this, connection_pool.size());
removed_from_pool = true;
break;
}
}
}
max_keepalive_requests_--;
if (c_->read->timer_set) {
ngx_del_timer(c_->read);
}
if (c_->write->timer_set) {
ngx_del_timer(c_->write);
}
if (!keepalive_ || max_keepalive_requests_ <= 0 || removed_from_pool) {
ngx_close_connection(c_);
c_ = NULL;
delete this;
return;
}
ngx_add_timer(c_->read, static_cast<ngx_msec_t>(
NgxConnection::keepalive_timeout_ms));
c_->data = this;
c_->read->handler = NgxConnection::IdleReadHandler;
c_->write->handler = NgxConnection::IdleWriteHandler;
c_->idle = 1;
// This connection should not be associated with current fetch.
c_->log = ngx_cycle->log;
c_->read->log = ngx_cycle->log;
c_->write->log = ngx_cycle->log;
if (c_->pool != NULL) {
c_->pool->log = ngx_cycle->log;
}
// Allow this connection to be re-used, by adding it to the connection pool.
{
ScopedMutex lock(&NgxConnection::connection_pool_mutex);
connection_pool.Add(this);
ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
"NgxFetch: Added connection %p (pool size: %l - "
" max_keepalive_requests_ %d)",
this, connection_pool.size(), max_keepalive_requests_);
}
}
void NgxConnection::IdleWriteHandler(ngx_event_t* ev) {
ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
u_char buf[1];
int n = c->recv(c, buf, 1);
if (c->write->timedout) {
DCHECK(false) << "NgxFetch: write timeout not expected." << n;
}
if (n == NGX_AGAIN) {
return;
}
}
void NgxConnection::IdleReadHandler(ngx_event_t* ev) {
ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
NgxConnection* nc = static_cast<NgxConnection*>(c->data);
if (c->read->timedout) {
nc->set_keepalive(false);
nc->Close();
return;
}
char buf[1];
int n;
// not a timeout event, we should check connection
n = recv(c->fd, buf, 1, MSG_PEEK);
if (n == -1 && ngx_socket_errno == NGX_EAGAIN) {
if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
nc->set_keepalive(false);
nc->Close();
return;
}
return;
}
nc->set_keepalive(false);
nc->Close();
}
NgxFetch::NgxFetch(const GoogleString& url,
AsyncFetch* async_fetch,
MessageHandler* message_handler,
ngx_log_t* log)
: str_url_(url),
fetcher_(NULL),
async_fetch_(async_fetch),
parser_(async_fetch->response_headers()),
message_handler_(message_handler),
bytes_received_(0),
fetch_start_ms_(0),
fetch_end_ms_(0),
done_(false),
content_length_(-1),
content_length_known_(false),
resolver_ctx_(NULL) {
ngx_memzero(&url_, sizeof(url_));
log_ = log;
pool_ = NULL;
timeout_event_ = NULL;
connection_ = NULL;
}
NgxFetch::~NgxFetch() {
if (timeout_event_ != NULL && timeout_event_->timer_set) {
ngx_del_timer(timeout_event_);
}
if (connection_ != NULL) {
connection_->Close();
connection_ = NULL;
}
if (pool_ != NULL) {
ngx_destroy_pool(pool_);
pool_ = NULL;
}
}
// This function is called by NgxUrlAsyncFetcher::StartFetch.
bool NgxFetch::Start(NgxUrlAsyncFetcher* fetcher) {
fetcher_ = fetcher;
bool ok = Init();
if (ok) {
ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized",
this);
} // else Init() will have emitted a reason
return ok;
}
// Create the pool, parse the url, add the timeout event and
// hook the DNS resolver if needed. Else we connect directly.
// When this returns false, our caller (NgxUrlAsyncFetcher::StartFetch)
// will call fetch->CallbackDone()
bool NgxFetch::Init() {
pool_ = ngx_create_pool(12288, log_);
if (pool_ == NULL) {
message_handler_->Message(kError, "NgxFetch: ngx_create_pool failed");
return false;
}
if (!ParseUrl()) {
message_handler_->Message(kError,
"NgxFetch: ParseUrl() failed for [%s]:%s",
str_url_.c_str(), url_.err);
return false;
}
timeout_event_ = static_cast<ngx_event_t*>(
ngx_pcalloc(pool_, sizeof(ngx_event_t)));
if (timeout_event_ == NULL) {
message_handler_->Message(kError,
"NgxFetch: ngx_pcalloc failed for timeout");
return false;
}
timeout_event_->data = this;
timeout_event_->handler = NgxFetch::TimeoutHandler;
timeout_event_->log = log_;
ngx_add_timer(timeout_event_, fetcher_->fetch_timeout_);
r_ = static_cast<ngx_http_request_t*>(
ngx_pcalloc(pool_, sizeof(ngx_http_request_t)));
if (r_ == NULL) {
message_handler_->Message(kError,
"NgxFetch: ngx_pcalloc failed for timer");
return false;
}
status_ = static_cast<ngx_http_status_t*>(
ngx_pcalloc(pool_, sizeof(ngx_http_status_t)));
if (status_ == NULL) {
message_handler_->Message(kError,
"NgxFetch: ngx_pcalloc failed for status");
return false;
}
// The host is either a domain name or an IP address. First check
// if it's a valid IP address and only if that fails fall back to
// using the DNS resolver.
// Maybe we have a Proxy.
ngx_url_t* tmp_url = &url_;
if (fetcher_->proxy_.url.len != 0) {
tmp_url = &fetcher_->proxy_;
}
GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
tmp_url->host.len);
ngx_memzero(&sin_, sizeof(sin_));
sin_.sin_family = AF_INET;
sin_.sin_port = htons(tmp_url->port);
sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
if (sin_.sin_addr.s_addr == INADDR_NONE) {
// inet_addr returned INADDR_NONE, which means the hostname
// isn't a valid IP address. Check DNS.
ngx_resolver_ctx_t temp;
temp.name.data = tmp_url->host.data;
temp.name.len = tmp_url->host.len;
resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
// TODO(oschaaf): this spams the log, but is useful in the fetcher's
// current state
message_handler_->Message(
kError, "NgxFetch: Couldn't start resolving, "
"is there a proper resolver configured in nginx.conf?");
return false;
} else {
ngx_log_error(NGX_LOG_DEBUG, log_, 0,
"NgxFetch %p: start resolve for: %s",
this, s_ipaddress.c_str());
}
resolver_ctx_->data = this;
resolver_ctx_->name.data = tmp_url->host.data;
resolver_ctx_->name.len = tmp_url->host.len;
#if (nginx_version < 1005008)
resolver_ctx_->type = NGX_RESOLVE_A;
#endif
resolver_ctx_->handler = NgxFetch::ResolveDoneHandler;
resolver_ctx_->timeout = fetcher_->resolver_timeout_;
if (ngx_resolve_name(resolver_ctx_) != NGX_OK) {
message_handler_->Message(kWarning,
"NgxFetch: ngx_resolve_name failed");
return false;
}
} else {
if (InitRequest() != NGX_OK) {
message_handler()->Message(kError, "NgxFetch: InitRequest failed");
return false;
}
}
return true;
}
const char* NgxFetch::str_url() {
return str_url_.c_str();
}
// This function should be called only once. The only argument is sucess or
// not.
void NgxFetch::CallbackDone(bool success) {
ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s",
this, success ? "OK":"FAIL");
if (async_fetch_ == NULL) {
LOG(FATAL)
<< "BUG: NgxFetch callback called more than once on same fetch"
<< str_url_.c_str() << "(" << this << ").Please report this at"
<< "https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss";
return;
}
release_resolver();
if (timeout_event_ && timeout_event_->timer_set) {
ngx_del_timer(timeout_event_);
timeout_event_ = NULL;
}
if (connection_ != NULL) {
// Connection will be re-used only on responses that specify
// 'Connection: keep-alive' in their headers.
bool keepalive = false;
if (success) {
ConstStringStarVector v;
if (async_fetch_->response_headers()->Lookup(
StringPiece(HttpAttributes::kConnection), &v)) {
for (size_t i = 0; i < v.size(); i++) {
if (*v[i] == "keep-alive") {
keepalive = true;
break;
} else if (*v[i] == "close") {
break;
}
}
}
ngx_log_error(NGX_LOG_DEBUG, log_, 0,
"NgxFetch %p: connection %p attempt keep-alive: %s",
this, connection_, keepalive ? "Yes":"No");
}
connection_->set_keepalive(keepalive);
connection_->Close();
connection_ = NULL;
}
if (fetcher_ != NULL) {
if (fetcher_->track_original_content_length()
&& async_fetch_->response_headers()->Has(
HttpAttributes::kXOriginalContentLength)) {
async_fetch_->extra_response_headers()->SetOriginalContentLength(
bytes_received_);
}
fetcher_->FetchComplete(this);
}
async_fetch_->Done(success);
async_fetch_ = NULL;
}
size_t NgxFetch::bytes_received() {
return bytes_received_;
}
void NgxFetch::bytes_received_add(int64 x) {
bytes_received_ += x;
}
int64 NgxFetch::fetch_start_ms() {
return fetch_start_ms_;
}
void NgxFetch::set_fetch_start_ms(int64 start_ms) {
fetch_start_ms_ = start_ms;
}
int64 NgxFetch::fetch_end_ms() {
return fetch_end_ms_;
}
void NgxFetch::set_fetch_end_ms(int64 end_ms) {
fetch_end_ms_ = end_ms;
}
MessageHandler* NgxFetch::message_handler() {
return message_handler_;
}
bool NgxFetch::ParseUrl() {
url_.url.len = str_url_.length();
url_.url.data = static_cast<u_char*>(ngx_palloc(pool_, url_.url.len));
if (url_.url.data == NULL) {
return false;
}
str_url_.copy(reinterpret_cast<char*>(url_.url.data), str_url_.length(), 0);
return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
}
// Issue a request after the resolver is done
void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
NgxFetch* fetch = static_cast<NgxFetch*>(resolver_ctx->data);
NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
if (resolver_ctx->state != NGX_OK) {
if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
ngx_del_timer(fetch->timeout_event());
fetch->set_timeout_event(NULL);
}
fetch->message_handler()->Message(
kWarning, "NgxFetch %p: failed to resolve host [%.*s]", fetch,
static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
fetch->CallbackDone(false);
return;
}
ngx_uint_t i;
// Find the first ipv4 address. We don't support ipv6 yet.
for (i = 0; i < resolver_ctx->naddrs; i++) {
// Old versions of nginx and tengine have a different definition of addrs,
// work around to make sure we are using the right type (ngx_addr_t*).
ngx_addr_t* ngx_addrs = reinterpret_cast<ngx_addr_t*>(resolver_ctx->addrs);
if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
if (reinterpret_cast<struct sockaddr_in*>(ngx_addrs[i].sockaddr)
->sin_family == AF_INET) {
break;
}
} else {
// We're using an old version that uses in_addr_t* for addrs.
break;
}
}
// If no suitable ipv4 address was found, we fail.
if (i == resolver_ctx->naddrs) {
if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
ngx_del_timer(fetch->timeout_event());
fetch->set_timeout_event(NULL);
}
fetch->message_handler()->Message(
kWarning, "NgxFetch %p: no suitable address for host [%.*s]", fetch,
static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
fetch->CallbackDone(false);
}
ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
#if (nginx_version < 1005008)
fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
#else
struct sockaddr_in* sin;
sin = reinterpret_cast<struct sockaddr_in*>(
resolver_ctx->addrs[i].sockaddr);
fetch->sin_.sin_family = sin->sin_family;
fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
#endif
fetch->sin_.sin_family = AF_INET;
fetch->sin_.sin_port = htons(fetch->url_.port);
// Maybe we have Proxy
if (0 != fetcher->proxy_.url.len) {
fetch->sin_.sin_port = htons(fetcher->proxy_.port);
}
char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: Resolved host [%V] to [%s]", fetch,
&resolver_ctx->name, ip_address);
fetch->release_resolver();
if (fetch->InitRequest() != NGX_OK) {
fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
fetch->CallbackDone(false);
}
}
// Prepare the request data for this fetch, and hook the write event.
int NgxFetch::InitRequest() {
in_ = ngx_create_temp_buf(pool_, 4096);
if (in_ == NULL) {
return NGX_ERROR;
}
FixUserAgent();
RequestHeaders* request_headers = async_fetch_->request_headers();
ConstStringStarVector v;
size_t size = 0;
bool have_host = false;
GoogleString port;
response_handler = NgxFetch::HandleStatusLine;
int rc = Connect();
if (rc == NGX_AGAIN || rc == NGX_OK) {
if (connection_->keepalive()) {
request_headers->Add(HttpAttributes::kConnection,
NgxConnection::ka_header);
}
const char* method = request_headers->method_string();
size_t method_len = strlen(method);
size = (method_len +
1 /* for the space */ +
url_.uri.len +
sizeof(" HTTP/1.0\r\n") - 1);
for (int i = 0; i < request_headers->NumAttributes(); i++) {
// if no explicit host header is given in the request headers,
// we need to derive it from the url.
if (StringCaseEqual(request_headers->Name(i), "Host")) {
have_host = true;
}
// name: value\r\n
size += request_headers->Name(i).length()
+ request_headers->Value(i).length() + 4; // 4 for ": \r\n"
}
if (!have_host) {
port = StrCat(":", IntegerToString(url_.port));
// for "Host: " + host + ":" + port + "\r\n"
size += url_.host.len + 8 + port.size();
}
size += 2; // "\r\n";
out_ = ngx_create_temp_buf(pool_, size);
if (out_ == NULL) {
return NGX_ERROR;
}
out_->last = ngx_cpymem(out_->last, method, method_len);
out_->last = ngx_cpymem(out_->last, " ", 1);
out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
if (!have_host) {
out_->last = ngx_cpymem(out_->last, "Host: ", 6);
out_->last = ngx_cpymem(out_->last, url_.host.data, url_.host.len);
out_->last = ngx_cpymem(out_->last, port.c_str(), port.size());
out_->last = ngx_cpymem(out_->last, "\r\n", 2);
}
for (int i = 0; i < request_headers->NumAttributes(); i++) {
const GoogleString& name = request_headers->Name(i);
const GoogleString& value = request_headers->Value(i);
out_->last = ngx_cpymem(out_->last, name.c_str(), name.length());
*(out_->last++) = ':';
*(out_->last++) = ' ';
out_->last = ngx_cpymem(out_->last, value.c_str(), value.length());
*(out_->last++) = CR;
*(out_->last++) = LF;
}
*(out_->last++) = CR;
*(out_->last++) = LF;
if (rc == NGX_AGAIN) {
return NGX_OK;
}
} else if (rc < NGX_OK) {
return rc;
}
CHECK(rc == NGX_OK);
NgxFetch::ConnectionWriteHandler(connection_->c_->write);
return NGX_OK;
}
int NgxFetch::Connect() {
ngx_peer_connection_t pc;
ngx_memzero(&pc, sizeof(pc));
pc.sockaddr = (struct sockaddr*)&sin_;
pc.socklen = sizeof(struct sockaddr_in);
pc.name = &url_.host;
// get callback is dummy function, it just returns NGX_OK
pc.get = ngx_event_get_peer;
pc.log_error = NGX_ERROR_ERR;
pc.log = fetcher_->log_;
pc.rcvbuf = -1;
connection_ = NgxConnection::Connect(&pc, message_handler(),
fetcher_->max_keepalive_requests_);
ngx_log_error(NGX_LOG_DEBUG, fetcher_->log_, 0,
"NgxFetch %p Connect() connection %p for [%s]",
this, connection_, str_url());
if (connection_ == NULL) {
return NGX_ERROR;
}
connection_->c_->write->handler = NgxFetch::ConnectionWriteHandler;
connection_->c_->read->handler = NgxFetch::ConnectionReadHandler;
connection_->c_->data = this;
// Timer set in Init() is still in effect.
return NGX_OK;
}
// When the fetch sends the request completely, it will hook the read event,
// and prepare to parse the response. Timer set in Init() is still in effect.
void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
ngx_connection_t* c = static_cast<ngx_connection_t*>(wev->data);
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
ngx_buf_t* out = fetch->out_;
bool ok = true;
while (wev->ready && out->pos < out->last) {
int n = c->send(c, out->pos, out->last - out->pos);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: ConnectionWriteHandler "
"send result %d", fetch, n);
if (n >= 0) {
out->pos += n;
} else if (n == NGX_AGAIN) {
break;
} else {
ok = false;
break;
}
}
if (ok) {
if (out->pos == out->last) {
ok = ngx_handle_read_event(c->read, 0) == NGX_OK;
} else {
ok = ngx_handle_write_event(c->write, 0) == NGX_OK;
}
}
if (!ok) {
fetch->message_handler()->Message(
kWarning, "NgxFetch %p: failed to hook next event", fetch);
c->error = 1;
fetch->CallbackDone(false);
}
}
// Timer set in Init() is still in effect.
void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {
ngx_connection_t* c = static_cast<ngx_connection_t*>(rev->data);
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
bool ok = true;
while(rev->ready) {
int n = c->recv(
c, fetch->in_->start, fetch->in_->end - fetch->in_->start);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: ConnectionReadHandler "
"recv result %d", fetch, n);
if (n == NGX_AGAIN) {
break;
} else if (n == 0) {
// If the content length was not known, we assume that we have read
// all if we at least parsed the headers.
// If we do know the content length, having a mismatch on the bytes read
// will be interpreted as an error.
ok = (fetch->content_length_known_ && fetch->content_length_ == fetch->bytes_received_)
|| fetch->parser_.headers_complete();
fetch->done_ = true;
break;
} else if (n > 0) {
fetch->in_->pos = fetch->in_->start;
fetch->in_->last = fetch->in_->start + n;
ok = fetch->response_handler(c);
if (fetch->done_ || !ok) {
break;
}
}
}
if (!ok) {
fetch->CallbackDone(false);
} else if (fetch->done_) {
fetch->CallbackDone(true);
} else if (ngx_handle_read_event(rev, 0) != NGX_OK) {
fetch->CallbackDone(false);
}
}
// Parse the status line: "HTTP/1.1 200 OK\r\n"
bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: Handle status line", fetch);
// This function only works after Nginx-1.1.4. Before nginx-1.1.4,
// ngx_http_parse_status_line didn't save http_version.
ngx_int_t n = ngx_http_parse_status_line(fetch->r_, fetch->in_,
fetch->status_);
if (n == NGX_ERROR) { // parse status line error
fetch->message_handler()->Message(
kWarning, "NgxFetch: failed to parse status line");
return false;
} else if (n == NGX_AGAIN) { // not completed
return true;
}
ResponseHeaders* response_headers =
fetch->async_fetch_->response_headers();
response_headers->SetStatusAndReason(
static_cast<HttpStatus::Code>(fetch->get_status_code()));
response_headers->set_major_version(fetch->get_major_version());
response_headers->set_minor_version(fetch->get_minor_version());
fetch->in_->pos += n;
fetch->set_response_handler(NgxFetch::HandleHeader);
if ((fetch->in_->last - fetch->in_->pos) > 0) {
return fetch->response_handler(c);
}
return true;
}
// Parse the HTTP headers
bool NgxFetch::HandleHeader(ngx_connection_t* c) {
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
char* data = reinterpret_cast<char*>(fetch->in_->pos);
size_t size = fetch->in_->last - fetch->in_->pos;
size_t n = fetch->parser_.ParseChunk(StringPiece(data, size),
fetch->message_handler_);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: Handle headers", fetch);
if (n > size) {
return false;
} else if (fetch->parser_.headers_complete()) {
// TODO(oschaaf): We should also check if the request method was HEAD
// - but I don't think PSOL uses that at this point.
if (fetch->get_status_code() == 304 || fetch->get_status_code() == 204) {
fetch->done_ = true;
} else if (fetch->async_fetch_->response_headers()->FindContentLength(
&fetch->content_length_)) {
if (fetch->content_length_ < 0) {
fetch->message_handler_->Message(
kError, "Negative content-length in response header");
return false;
} else {
fetch->content_length_known_ = true;
if (fetch->content_length_ == 0) {
fetch->done_ = true;
}
}
}
if (fetch->fetcher_->track_original_content_length()
&& fetch->content_length_known_) {
fetch->async_fetch_->response_headers()->SetOriginalContentLength(
fetch->content_length_);
}
fetch->in_->pos += n;
if (!fetch->done_) {
fetch->set_response_handler(NgxFetch::HandleBody);
if ((fetch->in_->last - fetch->in_->pos) > 0) {
return fetch->response_handler(c);
}
}
} else {
fetch->in_->pos += n;
}
return true;
}
// Read the response body
bool NgxFetch::HandleBody(ngx_connection_t* c) {
NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
char* data = reinterpret_cast<char*>(fetch->in_->pos);
size_t size = fetch->in_->last - fetch->in_->pos;
fetch->bytes_received_add(size);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: Handle body (%d bytes)", fetch, size);
if ( fetch->async_fetch_->Write(StringPiece(data, size),
fetch->message_handler()) ) {
if (fetch->bytes_received_ == fetch->content_length_) {
fetch->done_ = true;
}
fetch->in_->pos += size;
} else {
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: async fetch write failure", fetch);
return false;
}
return true;
}
void NgxFetch::TimeoutHandler(ngx_event_t* tev) {
NgxFetch* fetch = static_cast<NgxFetch*>(tev->data);
ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
"NgxFetch %p: TimeoutHandler called", fetch);
fetch->CallbackDone(false);
}
void NgxFetch::FixUserAgent() {
GoogleString user_agent;
ConstStringStarVector v;
RequestHeaders* request_headers = async_fetch_->request_headers();
if (request_headers->Lookup(HttpAttributes::kUserAgent, &v)) {
for (size_t i = 0, n = v.size(); i < n; i++) {
if (i != 0) {
user_agent += " ";
}
if (v[i] != NULL) {
user_agent += *(v[i]);
}
}
request_headers->RemoveAll(HttpAttributes::kUserAgent);
}
if (user_agent.empty()) {
user_agent += "NgxNativeFetcher";
}
GoogleString version = StrCat(
" (", kModPagespeedSubrequestUserAgent,
"/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
if (!StringPiece(user_agent).ends_with(version)) {
user_agent += version;
}
request_headers->Add(HttpAttributes::kUserAgent, user_agent);
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_fetch.h
-210
View File
@@ -1,210 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: x.dinic@gmail.com (Junmin Xiong)
//
// PageSpeed needs some way to talk to the internet and request resources. For
// example, if it's optimizing www.example.com/index.html and it sees html with
// <img src="//images.example.com/cat.jpg"> and images.example.com is authorized
// for rewriting in the config, then it needs to fetch cat.jpg from
// images.example.com and optimize it. In apache (always) and nginx (by
// default) we use a fetcher called "serf". This works fine, but it does run
// its own event loop. To be more efficient, this is a "native" fetcher that
// uses nginx's event loop.
//
// The fetch is started by the main thread. It will fetch the remote resource
// from the specific url asynchronously.
#ifndef NET_INSTAWEB_NGX_FETCH_H_
#define NET_INSTAWEB_NGX_FETCH_H_
extern "C" {
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
}
#include "ngx_url_async_fetcher.h"
#include <vector>
#include "net/instaweb/http/public/url_async_fetcher.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/pool.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/response_headers_parser.h"
#include "pagespeed/kernel/thread/pthread_mutex.h"
namespace net_instaweb {
typedef bool (*response_handler_pt)(ngx_connection_t* c);
class NgxUrlAsyncFetcher;
class NgxConnection;
class NgxConnection : public PoolElement<NgxConnection> {
public:
NgxConnection(MessageHandler* handler, int max_keepalive_requests);
~NgxConnection();
void SetSock(u_char *sockaddr, socklen_t socklen) {
socklen_ = socklen;
ngx_memcpy(&sockaddr_, sockaddr, socklen);
}
// Close ensures that NgxConnection deletes itself at the appropriate time,
// which can be after receiving a non-keepalive response, or when the remote
// server closes the connection when the NgxConnection is pooled and idle.
void Close();
// Once keepalive is disabled, it can't be toggled back on.
void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
bool keepalive() { return keepalive_; }
typedef Pool<NgxConnection> NgxConnectionPool;
static NgxConnection* Connect(ngx_peer_connection_t* pc,
MessageHandler* handler,
int max_keepalive_requests);
static void IdleWriteHandler(ngx_event_t* ev);
static void IdleReadHandler(ngx_event_t* ev);
// Terminate will cleanup any idle connections upon shutdown.
static void Terminate();
static NgxConnectionPool connection_pool;
static PthreadMutex connection_pool_mutex;
// c_ is owned by NgxConnection and freed in ::Close()
ngx_connection_t* c_;
static const int64 keepalive_timeout_ms;
static const GoogleString ka_header;
private:
int max_keepalive_requests_;
bool keepalive_;
socklen_t socklen_;
u_char sockaddr_[NGX_SOCKADDRLEN];
MessageHandler* handler_;
DISALLOW_COPY_AND_ASSIGN(NgxConnection);
};
class NgxFetch : public PoolElement<NgxFetch> {
public:
NgxFetch(const GoogleString& url,
AsyncFetch* async_fetch,
MessageHandler* message_handler,
ngx_log_t* log);
~NgxFetch();
// Start the fetch.
bool Start(NgxUrlAsyncFetcher* fetcher);
// Show the completed url, for logging purposes.
const char* str_url();
// This fetch task is done. Call Done() on the async_fetch. It will copy the
// buffer to cache.
void CallbackDone(bool success);
// Show the bytes received.
size_t bytes_received();
void bytes_received_add(int64 x);
int64 fetch_start_ms();
void set_fetch_start_ms(int64 start_ms);
int64 fetch_end_ms();
void set_fetch_end_ms(int64 end_ms);
MessageHandler* message_handler();
int get_major_version() {
return static_cast<int>(status_->http_version / 1000);
}
int get_minor_version() {
return static_cast<int>(status_->http_version % 1000);
}
int get_status_code() {
return static_cast<int>(status_->code);
}
ngx_event_t* timeout_event() {
return timeout_event_;
}
void set_timeout_event(ngx_event_t* x) {
timeout_event_ = x;
}
void release_resolver() {
if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
ngx_resolve_name_done(resolver_ctx_);
resolver_ctx_ = NULL;
}
}
private:
response_handler_pt response_handler;
// Do the initialized work and start the resolver work.
bool Init();
bool ParseUrl();
// Prepare the request and write it to remote server.
int InitRequest();
// Create the connection with remote server.
int Connect();
void set_response_handler(response_handler_pt handler) {
response_handler = handler;
}
// Only the Static functions could be used in callbacks.
static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
// Write the request.
static void ConnectionWriteHandler(ngx_event_t* wev);
// Wait for the response.
static void ConnectionReadHandler(ngx_event_t* rev);
// Read and parse the first status line.
static bool HandleStatusLine(ngx_connection_t* c);
// Read and parse the HTTP headers.
static bool HandleHeader(ngx_connection_t* c);
// Read the response body.
static bool HandleBody(ngx_connection_t* c);
// Cancel the fetch when it's timeout.
static void TimeoutHandler(ngx_event_t* tev);
// Add the pagespeed User-Agent.
void FixUserAgent();
void FixHost();
const GoogleString str_url_;
ngx_url_t url_;
NgxUrlAsyncFetcher* fetcher_;
AsyncFetch* async_fetch_;
ResponseHeadersParser parser_;
MessageHandler* message_handler_;
int64 bytes_received_;
int64 fetch_start_ms_;
int64 fetch_end_ms_;
bool done_;
int64 content_length_;
bool content_length_known_;
struct sockaddr_in sin_;
ngx_log_t* log_;
ngx_buf_t* out_;
ngx_buf_t* in_;
ngx_pool_t* pool_;
ngx_http_request_t* r_;
ngx_http_status_t* status_;
ngx_event_t* timeout_event_;
NgxConnection* connection_;
ngx_resolver_ctx_t* resolver_ctx_;
DISALLOW_COPY_AND_ASSIGN(NgxFetch);
};
} // namespace net_instaweb
#endif // NET_INSTAWEB_NGX_FETCH_H_
lib/ngx_pagespeed/src/ngx_gzip_setter.cc
-403
View File
@@ -1,403 +0,0 @@
/*
* Copyright 2014 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: kspoelstra@we-amp.com (Kees Spoelstra)
#include "ngx_gzip_setter.h"
#include <ngx_conf_file.h>
namespace net_instaweb {
NgxGZipSetter g_gzip_setter;
extern "C" {
// These functions replace the setters for:
// gzip
// gzip_types
// gzip_http_version
// gzip_vary
//
// If these functions are called it means there is an explicit gzip
// configuration. The gzip configuration set by pagespeed is then rolled
// back and pagespeed will stop enabling gzip automatically.
char* ngx_gzip_redirect_conf_set_flag_slot(
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
if (g_gzip_setter.enabled()) {
g_gzip_setter.RollBackAndDisable(cf);
}
char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
return ret;
}
char* ngx_gzip_redirect_http_types_slot(
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
if (g_gzip_setter.enabled()) {
g_gzip_setter.RollBackAndDisable(cf);
}
char* ret = ngx_http_types_slot(cf, cmd, conf);
return ret;
}
char* ngx_gzip_redirect_conf_set_enum_slot(
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
if (g_gzip_setter.enabled()) {
g_gzip_setter.RollBackAndDisable(cf);
}
char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
return ret;
}
char* ngx_gzip_redirect_conf_set_bitmask_slot(
ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
if (g_gzip_setter.enabled()) {
g_gzip_setter.RollBackAndDisable(cf);
}
char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
return ret;
}
}
NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
NgxGZipSetter::~NgxGZipSetter() { }
// Helper functions to determine signature.
bool HasLocalConfig(ngx_command_t* command) {
return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
command->conf == NGX_HTTP_LOC_CONF_OFFSET);
}
bool IsNgxFlagCommand(ngx_command_t* command) {
return (command->set == ngx_conf_set_flag_slot &&
HasLocalConfig(command));
}
bool IsNgxHttpTypesCommand(ngx_command_t* command) {
return (command->set == ngx_http_types_slot &&
HasLocalConfig(command));
}
bool IsNgxEnumCommand(ngx_command_t* command) {
return (command->set == ngx_conf_set_enum_slot &&
HasLocalConfig(command));
}
bool IsNgxBitmaskCommand(ngx_command_t* command) {
return (command->set == ngx_conf_set_bitmask_slot &&
HasLocalConfig(command));
}
// Initialize the NgxGzipSetter.
// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
// gzip module. Enable if the signature of the zip command matches with what we
// trust. Also sets up redirects for the configurations. These redirect handle
// a rollback if expicit configuration is found.
// If commands are not found the method will inform the user by logging.
void NgxGZipSetter::Init(ngx_conf_t* cf) {
#if (NGX_HTTP_GZIP)
bool gzip_signature_mismatch = false;
bool other_signature_mismatch = false;
for (int m = 0; ngx_modules[m] != NULL; m++) {
if (ngx_modules[m]->commands != NULL) {
for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
ngx_command_t* current_command =& ngx_modules[m]->commands[c];
// We look for the gzip command, and the exact signature we trust
// this means configured as an config location offset
// and a ngx_flag_t setter.
// Also see:
// ngx_conf_handler in ngx_conf_file.c
// ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
if (gzip_command_.command_ == NULL &&
STR_EQ_LITERAL(current_command->name, "gzip")) {
if (IsNgxFlagCommand(current_command)) {
current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
gzip_command_.command_ = current_command;
gzip_command_.module_ = ngx_modules[m];
enabled_ = 1;
} else {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0,
"pagespeed: cannot set gzip, signature mismatch");
gzip_signature_mismatch = true;
}
}
if (!gzip_http_version_command_.command_ &&
STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
if (IsNgxEnumCommand(current_command)) {
current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
gzip_http_version_command_.command_ = current_command;
gzip_http_version_command_.module_ = ngx_modules[m];
} else {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0,
"pagespeed: cannot set gzip_http_version, signature mismatch");
other_signature_mismatch = true;
}
}
if (!gzip_proxied_command_.command_ &&
STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
if (IsNgxBitmaskCommand(current_command)) {
current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
gzip_proxied_command_.command_ = current_command;
gzip_proxied_command_.module_ = ngx_modules[m];
} else {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0,
"pagespeed: cannot set gzip_proxied, signature mismatch");
other_signature_mismatch = true;
}
}
if (!gzip_http_types_command_.command_ &&
STR_EQ_LITERAL(current_command->name, "gzip_types")) {
if (IsNgxHttpTypesCommand(current_command)) {
current_command->set = ngx_gzip_redirect_http_types_slot;
gzip_http_types_command_.command_ = current_command;
gzip_http_types_command_.module_ = ngx_modules[m];
} else {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0,
"pagespeed: cannot set gzip_types, signature mismatch");
other_signature_mismatch = true;
}
}
if (!gzip_vary_command_.command_ &&
STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
if (IsNgxFlagCommand(current_command)) {
current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
gzip_vary_command_.command_ = current_command;
gzip_vary_command_.module_ = ngx_modules[m];
} else {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0,
"pagespeed: cannot set gzip_vary, signature mismatch");
other_signature_mismatch = true;
}
}
}
}
}
if (gzip_signature_mismatch) {
return; // Already logged error.
} else if (!enabled_) {
// Looked through all the available commands and didn't find the "gzip" one.
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
return;
} else if (other_signature_mismatch) {
return; // Already logged error.
} else if (!gzip_vary_command_.command_) {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
return;
} else if (!gzip_http_types_command_.command_) {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
return;
} else if (!gzip_http_version_command_.command_) {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
return;
} else if (!gzip_proxied_command_.command_) {
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
return;
} else {
return; // Success.
}
#else
ngx_conf_log_error(
NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
return;
#endif
}
void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
return GetModuleConfPtr(cf) + command_->offset;
}
char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
return reinterpret_cast<char*>(
ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
}
void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
ngx_command_ctx* command_ctx,
ngx_flag_t value) {
ngx_flag_t* flag = reinterpret_cast<ngx_flag_t*>(command_ctx->GetConfPtr(cf));
*flag = value;
// Save the flag position for possible rollback.
ngx_flags_set_.push_back(flag);
}
void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
ngx_command_ctx* command_ctx,
ngx_uint_t value) {
ngx_uint_t* enum_to_set =
reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
*enum_to_set = value;
ngx_uint_set_.push_back(enum_to_set);
}
void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
ngx_command_ctx* command_ctx,
ngx_uint_t value) {
ngx_uint_t* enum_to_set =
reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
*enum_to_set = value;
ngx_uint_set_.push_back(enum_to_set);
}
// These are the content types we want to compress.
ngx_str_t gzip_http_types[] = {
ngx_string("application/ecmascript"),
ngx_string("application/javascript"),
ngx_string("application/json"),
ngx_string("application/pdf"),
ngx_string("application/postscript"),
ngx_string("application/x-javascript"),
ngx_string("image/svg+xml"),
ngx_string("text/css"),
ngx_string("text/csv"),
// ngx_string("text/html"), // This is the default implied value.
ngx_string("text/javascript"),
ngx_string("text/plain"),
ngx_string("text/xml"),
ngx_null_string // Indicates end of array.
};
gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
bool value) {
if (!enabled_) {
return kEnableGZipNotEnabled;
}
if (gzip_command_.command_) {
SetNgxConfFlag(cf, &gzip_command_, value);
}
return kEnableGZipOk;
}
void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
if (!enabled_) {
return;
}
// When we get called twice for the same location{}, we ignore the second call
// to prevent adding duplicate gzip http types and so on.
ngx_flag_t* flag =
reinterpret_cast<ngx_flag_t*>(gzip_command_.GetConfPtr(cf));
if (*flag == 1) {
return;
}
SetGZipForLocation(cf, true);
if (gzip_vary_command_.command_) {
SetNgxConfFlag(cf, &gzip_vary_command_, 1);
}
if (gzip_http_version_command_.command_) {
SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
}
if (gzip_proxied_command_.command_) {
SetNgxConfBitmask(
cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
}
// This is actually the most prone to future API changes, because gzip_types
// is not a simple type like ngx_flag_t. The signature check should be enough
// to prevent problems.
AddGZipHTTPTypes(cf);
return;
}
void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
if (gzip_http_types_command_.command_) {
// Following should not happen, but if it does return gracefully.
if (cf->args->nalloc < 2) {
ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
"pagespeed: unexpected small cf->args in gzip_types");
return;
}
ngx_command_t* command = gzip_http_types_command_.command_;
char* gzip_conf = reinterpret_cast<char* >(
gzip_http_types_command_.GetModuleConfPtr(cf));
// Backup the old settings.
ngx_str_t old_elt0 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[0];
ngx_str_t old_elt1 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[1];
ngx_uint_t old_nelts = cf->args->nelts;
// Setup first arg.
ngx_str_t gzip_types_string = ngx_string("gzip_types");
reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = gzip_types_string;
cf->args->nelts = 2;
ngx_str_t* http_types = gzip_http_types;
while (http_types->data) {
ngx_str_t d;
// We allocate the http type on the configuration pool and actually
// leak this if we rollback. This does not seem to be a big problem,
// because nginx also allocates tokens in ngx_conf_file.c and does not
// free them. This way they can be used safely by configurations.
// We must use a copy of gzip_http_types array here because nginx will
// manipulate the values.
// TODO(kspoelstra): better would be to allocate once on init and not
// every time we enable gzip. This needs further investigation, sharing
// tokens might be problematic.
// For now I think it is not a large problem. This might add up in case
// of a large multi server/location config with a lot of "pagespeed on"
// directives.
// Estimates are 300-400KB for 1000 times "pagespeed on".
d.data = reinterpret_cast<u_char*>(
ngx_pnalloc(cf->pool, http_types->len + 1));
snprintf(reinterpret_cast<char*>(d.data), http_types->len + 1, "%s",
reinterpret_cast<const char*>(http_types->data));
d.len = http_types->len;
reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = d;
// Call the original setter.
ngx_http_types_slot(cf, command, gzip_conf);
http_types++;
}
// Restore args.
cf->args->nelts = old_nelts;
reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = old_elt1;
reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = old_elt0;
// Backup configuration location for rollback.
ngx_httptypes_set_.push_back(gzip_conf + command->offset);
}
}
void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
"pagespeed: rollback gzip, explicit configuration");
for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
i != ngx_flags_set_.end(); ++i) {
*(*i)=NGX_CONF_UNSET;
}
for (std::vector<ngx_uint_t*>::iterator i = ngx_uint_set_.begin();
i != ngx_uint_set_.end(); ++i) {
*(*i)=NGX_CONF_UNSET_UINT;
}
for (std::vector<void*>::iterator i = ngx_httptypes_set_.begin();
i != ngx_httptypes_set_.end(); ++i) {
ngx_array_t** type_array = reinterpret_cast<ngx_array_t**>(*i);
ngx_array_destroy(*type_array);
*type_array = NULL;
}
enabled_ = 0;
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_gzip_setter.h
-124
View File
@@ -1,124 +0,0 @@
/*
* Copyright 2014 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: kspoelstra@we-amp.com (Kees Spoelstra)
/*
* NgxGZipSetter sets up gzip for pagespeed
* with the following configuration:
* gzip on;
* gzip_vary on;
* gzip_types application/ecmascript;
* gzip_types application/javascript;
* gzip_types application/json;
* gzip_types application/pdf;
* gzip_types application/postscript;
* gzip_types application/x-javascript;
* gzip_types image/svg+xml;
* gzip_types text/css;
* gzip_types text/csv;
* gzip_types text/javascript;
* gzip_types text/plain;
* gzip_types text/xml;
* gzip_http_version 1.0;
*
* If there is an explicit gzip configuration in the nginx.conf
* pagespeed will rollback the set configuration and let the
* user decide what the configuration will be.
*
* It manipulates the configuration by manipulating ngx_flag_t
* and ngx_uint_t settings directly and using the nginx setter for
* gzip_http_types.
* This is probably a safe way to do it. If this mechanism
* changes all non nginx module setup & configuration will
* fail.
*/
#ifndef NGX_GZIP_SETTER_H_
#define NGX_GZIP_SETTER_H_
extern "C" {
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
}
#include <vector>
#include "ngx_pagespeed.h"
#include "pagespeed/kernel/base/basictypes.h"
namespace net_instaweb {
// We need this class because configuration for gzip is in different modules, so
// just saving the command will not work.
class ngx_command_ctx {
public:
ngx_command_ctx() : command_(NULL), module_(NULL) { }
void* GetConfPtr(ngx_conf_t* cf);
char* GetModuleConfPtr(ngx_conf_t* cf);
ngx_command_t* command_;
ngx_module_t* module_;
};
enum gzs_enable_result {
kEnableGZipOk,
kEnableGZipPartial,
kEnableGZipNotEnabled
};
class NgxGZipSetter {
std::vector<ngx_flag_t*> ngx_flags_set_;
std::vector<ngx_uint_t*> ngx_uint_set_;
std::vector<void*> ngx_httptypes_set_;
ngx_command_ctx gzip_command_;
ngx_command_ctx gzip_http_types_command_;
ngx_command_ctx gzip_proxied_command_;
ngx_command_ctx gzip_vary_command_;
ngx_command_ctx gzip_http_version_command_;
bool enabled_;
public:
NgxGZipSetter();
~NgxGZipSetter();
void Init(ngx_conf_t* cf);
void SetNgxConfFlag(ngx_conf_t* cf,
ngx_command_ctx* command_ctx,
ngx_flag_t value);
void SetNgxConfEnum(ngx_conf_t* cf,
ngx_command_ctx* command_ctx,
ngx_uint_t value);
void SetNgxConfBitmask(ngx_conf_t* cf,
ngx_command_ctx* command_ctx,
ngx_uint_t value);
void EnableGZipForLocation(ngx_conf_t* cf);
gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
void AddGZipHTTPTypes(ngx_conf_t* cf);
void RollBackAndDisable(ngx_conf_t* cf);
bool enabled() { return enabled_; }
private:
DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
};
extern NgxGZipSetter g_gzip_setter;
} // namespace net_instaweb
#endif // NGX_GZIP_SETTER_H_
lib/ngx_pagespeed/src/ngx_list_iterator.cc
-39
View File
@@ -1,39 +0,0 @@
/*
* Copyright 2013 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#include "ngx_list_iterator.h"
namespace net_instaweb {
NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
part_(part),
index_within_part_(0) {}
ngx_table_elt_t* NgxListIterator::Next() {
if (index_within_part_ >= part_->nelts) {
if (part_->next == NULL) {
return NULL;
}
part_ = part_->next;
index_within_part_ = 0;
}
ngx_table_elt_t* elts = static_cast<ngx_table_elt_t*>(part_->elts);
return &elts[index_within_part_++]; // Intentional post-increment.
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_list_iterator.h
-50
View File
@@ -1,50 +0,0 @@
/*
* Copyright 2013 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
//
// Simplifies iteration over nginx lists.
//
//
#ifndef NGX_LIST_ITERATOR_H_
#define NGX_LIST_ITERATOR_H_
extern "C" {
#include <ngx_http.h>
}
#include "base/basictypes.h"
namespace net_instaweb {
class NgxListIterator {
public:
explicit NgxListIterator(ngx_list_part_t* part);
// Return the next element of the list if there is one, NULL otherwise.
ngx_table_elt_t* Next();
private:
ngx_list_part_t* part_;
ngx_uint_t index_within_part_;
DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
};
} // namespace net_instaweb
#endif // NGX_LIST_ITERATOR_H_
lib/ngx_pagespeed/src/ngx_message_handler.cc
-116
View File
@@ -1,116 +0,0 @@
// Copyright 2013 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Author: oschaaf@gmail.com (Otto van der Schaaf)
#include "ngx_message_handler.h"
#include <signal.h>
#include "net/instaweb/public/version.h"
#include "pagespeed/kernel/base/abstract_mutex.h"
#include "pagespeed/kernel/base/debug.h"
#include "pagespeed/kernel/base/posix_timer.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/time_util.h"
#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
namespace {
// This will be prefixed to every logged message.
const char kModuleName[] = "ngx_pagespeed";
// If set, the crash handler will use this to output a backtrace using
// ngx_log_error.
ngx_log_t* global_log = NULL;
} // namespace
extern "C" {
static void signal_handler(int sig) {
// Try to output the backtrace to the log file. Since this may end up
// crashing/deadlocking/etc. we set an alarm() to abort us if it comes to
// that.
alarm(2);
if (global_log != NULL) {
ngx_log_error(NGX_LOG_ALERT, global_log, 0, "Trapped signal [%d]\n%s",
sig, net_instaweb::StackTraceString().c_str());
} else {
fprintf(stderr, "Trapped signal [%d]\n%s\n",
sig, net_instaweb::StackTraceString().c_str());
}
kill(getpid(), SIGKILL);
}
} // extern "C"
namespace net_instaweb {
NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
: SystemMessageHandler(timer, mutex),
log_(NULL) {
}
// Installs a signal handler for common crash signals, that tries to print
// out a backtrace.
void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
global_log = log;
signal(SIGTRAP, signal_handler); // On check failures
signal(SIGABRT, signal_handler);
signal(SIGFPE, signal_handler);
signal(SIGSEGV, signal_handler);
}
ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
switch (type) {
case kInfo:
return NGX_LOG_INFO;
case kWarning:
return NGX_LOG_WARN;
case kError:
return NGX_LOG_ERR;
case kFatal:
return NGX_LOG_ALERT;
}
// This should never fall through, but some compilers seem to complain if
// we don't include this.
return NGX_LOG_ALERT;
}
void NgxMessageHandler::MessageSImpl(MessageType type,
const GoogleString& message) {
if (log_ != NULL) {
ngx_uint_t log_level = GetNgxLogLevel(type);
ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
kModuleName, kModPagespeedVersion, message.c_str());
} else {
GoogleMessageHandler::MessageSImpl(type, message);
}
AddMessageToBuffer(type, message);
}
void NgxMessageHandler::FileMessageSImpl(
MessageType type, const char* file, int line, const GoogleString& message) {
if (log_ != NULL) {
ngx_uint_t log_level = GetNgxLogLevel(type);
ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
kModuleName, kModPagespeedVersion, file, line,
message.c_str());
} else {
GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
}
AddMessageToBuffer(type, file, line, message);
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_message_handler.h
-70
View File
@@ -1,70 +0,0 @@
// Copyright 2013 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Author: oschaaf@gmail.com (Otto van der Schaaf)
#ifndef NGX_MESSAGE_HANDLER_H_
#define NGX_MESSAGE_HANDLER_H_
extern "C" {
#include <ngx_auto_config.h>
#if (NGX_THREADS)
#include <ngx_thread.h>
#endif
#include <ngx_core.h>
#include <ngx_log.h>
}
#include <cstdarg>
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/system/system_message_handler.h"
namespace net_instaweb {
class AbstractMutex;
class Timer;
// Implementation of a message handler that uses ngx_log_error()
// logging to emit messages, with a fallback to GoogleMessageHandler
class NgxMessageHandler : public SystemMessageHandler {
public:
explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);
// Installs a signal handler for common crash signals that tries to print
// out a backtrace.
static void InstallCrashHandler(ngx_log_t* log);
void set_log(ngx_log_t* log) { log_ = log; }
ngx_log_t* log() { return log_; }
protected:
virtual void MessageSImpl(MessageType type, const GoogleString& message);
virtual void FileMessageSImpl(MessageType type, const char* file,
int line, const GoogleString& message);
private:
ngx_uint_t GetNgxLogLevel(MessageType type);
ngx_log_t* log_;
DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);
};
} // namespace net_instaweb
#endif // NGX_MESSAGE_HANDLER_H_
lib/ngx_pagespeed/src/ngx_pagespeed.cc
-3291
View File
@@ -1,3291 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
/*
* Usage:
* server {
* pagespeed on|off;
* }
*/
#include "ngx_pagespeed.h"
#include <vector>
#include <set>
#include "ngx_base_fetch.h"
#include "ngx_caching_headers.h"
#include "ngx_gzip_setter.h"
#include "ngx_list_iterator.h"
#include "ngx_message_handler.h"
#include "ngx_rewrite_driver_factory.h"
#include "ngx_rewrite_options.h"
#include "ngx_server_context.h"
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/public/global_constants.h"
#include "net/instaweb/public/version.h"
#include "net/instaweb/rewriter/public/experiment_matcher.h"
#include "net/instaweb/rewriter/public/experiment_util.h"
#include "net/instaweb/rewriter/public/process_context.h"
#include "net/instaweb/rewriter/public/resource_fetch.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_query.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/util/public/fallback_property_page.h"
#include "pagespeed/automatic/proxy_fetch.h"
#include "pagespeed/kernel/base/google_message_handler.h"
#include "pagespeed/kernel/base/null_message_handler.h"
#include "pagespeed/kernel/base/posix_timer.h"
#include "pagespeed/kernel/base/stack_buffer.h"
#include "pagespeed/kernel/base/stdio_file_system.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_writer.h"
#include "pagespeed/kernel/base/time_util.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/google_url.h"
#include "pagespeed/kernel/http/query_params.h"
#include "pagespeed/kernel/html/html_keywords.h"
#include "pagespeed/kernel/thread/pthread_shared_mem.h"
#include "pagespeed/kernel/util/gzip_inflater.h"
#include "pagespeed/kernel/util/statistics_logger.h"
#include "pagespeed/system/in_place_resource_recorder.h"
#include "pagespeed/system/system_caches.h"
#include "pagespeed/system/system_request_context.h"
#include "pagespeed/system/system_rewrite_options.h"
#include "pagespeed/system/system_server_context.h"
#include "pagespeed/system/system_thread_system.h"
extern ngx_module_t ngx_pagespeed;
// Unused flag, see
// http://lxr.evanmiller.org/http/source/http/ngx_http_request.h#L130
#define NGX_HTTP_PAGESPEED_BUFFERED 0x08
#define POST_BUF_READ_SIZE 65536
// Needed for SystemRewriteDriverFactory to use shared memory.
#define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
#define NGINX_1_13_4 1013004
net_instaweb::NgxRewriteDriverFactory* active_driver_factory = NULL;
namespace net_instaweb {
const char* kInternalEtagName = "@psol-etag";
// The process context takes care of proactively initialising
// a few libraries for us, some of which are not thread-safe
// when they are initialized lazily.
ProcessContext* process_context = new ProcessContext();
bool process_context_cleanup_hooked = false;
StringPiece str_to_string_piece(ngx_str_t s) {
return StringPiece(reinterpret_cast<char*>(s.data), s.len);
}
// Nginx uses memory pools, like Apache, allocating strings a request needs from
// the pool and then throwing it all away when the request finishes. Use this
// when you need to pass a short string to nginx and want it to take ownership
// of the string.
char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
// Need space for the final null.
ngx_uint_t buffer_size = sp.size() + 1;
char* s = static_cast<char*>(ngx_palloc(pool, buffer_size));
if (s == NULL) {
LOG(ERROR) << "string_piece_to_pool_string: ngx_palloc() returned NULL";
DCHECK(false);
return NULL;
}
sp.copy(s, buffer_size /* max to copy */);
s[buffer_size-1] = '\0'; // Null terminate it.
return s;
}
// When passing the body of http responses between filters Nginx uses a linked
// list of buffers ("buffer chain"), again like Apache. This constructs one of
// those lists from a StringPiece. This is what you use when you need to pass a
// (potentially) longer string to nginx and want it to take ownership.
ngx_int_t string_piece_to_buffer_chain(
ngx_pool_t* pool, StringPiece sp, ngx_chain_t** link_ptr,
bool send_last_buf, bool send_flush) {
// Below, *link_ptr will be NULL if we're starting the chain, and the head
// chain link.
*link_ptr = NULL;
// If non-null, the current last link in the chain.
ngx_chain_t* tail_link = NULL;
// How far into sp we're currently working on.
ngx_uint_t offset;
// Other modules seem to default to ngx_pagesize.
ngx_uint_t max_buffer_size = ngx_pagesize;
for (offset = 0 ;
offset < sp.size() ||
// If we need to send the last buffer bit and there's no data, we
// should send a single empty buffer. Otherwise we shouldn't
// generate empty buffers.
(offset == 0 && sp.size() == 0);
offset += max_buffer_size) {
// Prepare a new nginx buffer to put our buffered writes into.
ngx_buf_t* b = static_cast<ngx_buf_t*>(ngx_calloc_buf(pool));
if (b == NULL) {
return NGX_ERROR;
}
if (sp.size() == 0) {
CHECK(offset == 0); // NOLINT
b->pos = b->start = b->end = b->last = NULL;
// The purpose of this buffer is just to pass along last_buf.
b->sync = 1;
} else {
CHECK(sp.size() > offset);
ngx_uint_t b_size = sp.size() - offset;
if (b_size > max_buffer_size) {
b_size = max_buffer_size;
}
b->start = b->pos = static_cast<u_char*>(ngx_palloc(pool, b_size));
if (b->pos == NULL) {
return NGX_ERROR;
}
// Copy our writes over. We're copying from sp[offset] up to
// sp[offset + b_size] into b which has size b_size.
sp.copy(reinterpret_cast<char*>(b->pos), b_size, offset);
b->last = b->end = b->pos + b_size;
b->temporary = 1; // Identify this buffer as in-memory and mutable.
}
// Prepare a chain link.
ngx_chain_t* cl = static_cast<ngx_chain_t*>(ngx_alloc_chain_link(pool));
if (cl == NULL) {
return NGX_ERROR;
}
cl->buf = b;
cl->next = NULL;
if (*link_ptr == NULL) {
// This is the first link in the returned chain.
*link_ptr = cl;
} else {
// Link us into the chain.
CHECK(tail_link != NULL);
tail_link->next = cl;
}
tail_link = cl;
}
CHECK(tail_link != NULL);
if (send_flush) {
tail_link->buf->flush = true;
}
if (send_last_buf) {
tail_link->buf->last_buf = true;
}
return NGX_OK;
}
// Get the context for this request. ps_connection_read_handler should already
// have been called to create it.
ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r) {
return static_cast<ps_request_ctx_t*>(
ngx_http_get_module_ctx(r, ngx_pagespeed));
}
// Tell nginx whether we have network activity we're waiting for so that it sets
// a write handler. See src/http/ngx_http_request.c:2083.
void ps_set_buffered(ngx_http_request_t* r, bool on) {
if (on) {
r->buffered |= NGX_HTTP_PAGESPEED_BUFFERED;
} else {
r->buffered &= ~NGX_HTTP_PAGESPEED_BUFFERED;
}
}
namespace {
void ps_release_base_fetch(ps_request_ctx_t* ctx);
} // namespace
namespace ps_base_fetch {
ngx_http_output_header_filter_pt ngx_http_next_header_filter;
ngx_http_output_body_filter_pt ngx_http_next_body_filter;
ngx_int_t ps_base_fetch_filter(ngx_http_request_t* r, ngx_chain_t* in) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (r->header_only) {
return NGX_OK;
}
if (ctx == NULL || ctx->base_fetch == NULL) {
return ngx_http_next_body_filter(r, in);
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http pagespeed write filter \"%V\"", &r->uri);
// send response body
if (in || r->connection->buffered) {
ngx_int_t rc = ngx_http_next_body_filter(r, in);
// We can't indicate that we are done yet, because we have an active base
// fetch associated to this request.
if (rc != NGX_OK) {
return rc;
}
}
return NGX_AGAIN;
}
// This runs on the nginx event loop in response to seeing the byte PageSpeed
// sent over the pipe to trigger the nginx-side code. Copy whatever is ready
// from PageSpeed out to the browser (headers and/or body).
ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
ngx_int_t rc;
ngx_chain_t* cl = NULL;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ps fetch handler: %V", &r->uri);
if (ngx_terminate || ngx_exiting) {
ps_set_buffered(r, false);
ps_release_base_fetch(ctx);
return NGX_ERROR;
}
if (!r->header_sent) {
int status_code = ctx->base_fetch->response_headers()->status_code();
bool status_ok = (status_code != 0) && (status_code < 400);
// Pass on error handling for non-success status codes to nginx for
// responses where PSOL acts as a content generator (pagespeed resource,
// ipro hit, admin pages).
// This generates nginx's default error responses, but also allows header
// modules running after us to manipulate those responses.
if (!status_ok && (ctx->base_fetch->base_fetch_type() != kHtmlTransform
&& ctx->base_fetch->base_fetch_type() != kIproLookup)) {
ps_release_base_fetch(ctx);
ngx_http_filter_finalize_request(r, NULL, status_code);
return NGX_DONE;
}
if (ctx->preserve_caching_headers != kDontPreserveHeaders) {
ngx_table_elt_t* header;
NgxListIterator it(&(r->headers_out.headers.part));
while ((header = it.Next()) != NULL) {
// We need to remember a few headers when ModifyCachingHeaders is off,
// so we can send them unmodified in copy_response_headers_to_ngx().
// This just sets the hash to 0 for all other headers. That way, we
// avoid some relatively complicated code to reconstruct these headers.
if (!(STR_CASE_EQ_LITERAL(header->key, "Cache-Control") ||
(ctx->preserve_caching_headers == kPreserveAllCachingHeaders &&
(STR_CASE_EQ_LITERAL(header->key, "Etag") ||
STR_CASE_EQ_LITERAL(header->key, "Date") ||
STR_CASE_EQ_LITERAL(header->key, "Last-Modified") ||
STR_CASE_EQ_LITERAL(header->key, "Expires"))))) {
header->hash = 0;
if (STR_CASE_EQ_LITERAL(header->key, "Location")) {
// There's a possible issue with the location header, where setting
// the hash to 0 is not enough. See:
// https://github.com/nginx/nginx/blob/master/src/http/ngx_http_header_filter_module.c#L314
r->headers_out.location = NULL;
}
}
}
} else {
ngx_http_clean_header(r);
}
// collect response headers from pagespeed
rc = ctx->base_fetch->CollectHeaders(&r->headers_out);
if (rc == NGX_ERROR) {
ps_release_base_fetch(ctx);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
// send response headers
rc = ngx_http_next_header_filter(r);
// standard nginx send header check see ngx_http_send_response
if (rc == NGX_ERROR || rc > NGX_OK) {
ps_release_base_fetch(ctx);
return ngx_http_filter_finalize_request(r, NULL, rc);
}
// for in_place_check_header_filter
if (rc < NGX_OK && rc != NGX_AGAIN) {
CHECK(rc == NGX_DONE);
return NGX_DONE;
}
ps_set_buffered(r, true);
}
// collect response body from pagespeed
// Pass the optimized content along to later body filters.
// From Weibin: This function should be called mutiple times. Store the
// whole file in one chain buffers is too aggressive. It could consume
// too much memory in busy servers.
rc = ctx->base_fetch->CollectAccumulatedWrites(&cl);
ngx_log_error(NGX_LOG_DEBUG, ctx->r->connection->log, 0,
"CollectAccumulatedWrites, %d", rc);
if (rc == NGX_ERROR) {
ps_set_buffered(r, false);
ps_release_base_fetch(ctx);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if (rc == NGX_AGAIN && cl == NULL) {
// there is no body buffer to send now.
return NGX_AGAIN;
}
if (rc == NGX_OK) {
ps_set_buffered(r, false);
ps_release_base_fetch(ctx);
}
return ps_base_fetch_filter(r, cl);
}
void ps_base_fetch_filter_init() {
ngx_http_next_header_filter = ngx_http_top_header_filter;
ngx_http_next_body_filter = ngx_http_top_body_filter;
ngx_http_top_body_filter = ps_base_fetch_filter;
}
} // namespace ps_base_fetch
namespace {
// Setting headers in nginx is tricky because it's not just a matter of adding
// them to a list. You also need to remove them if there's already one there,
// as well as setting the shortcut pointers (both upper case and lower case).
//
// Based on ngx_http_add_cache_control.
ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
// First strip existing cache-control headers.
ngx_table_elt_t* header;
NgxListIterator it(&(r->headers_out.headers.part));
while ((header = it.Next()) != NULL) {
if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
// Response headers with hash of 0 are excluded from the response.
header->hash = 0;
}
}
// Now add our new cache control header.
if (r->headers_out.cache_control.elts == NULL) {
ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
1, sizeof(ngx_table_elt_t*));
if (rc != NGX_OK) {
return NGX_ERROR;
}
}
ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
ngx_array_push(&r->headers_out.cache_control));
if (cache_control_headers == NULL) {
return NGX_ERROR;
}
cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
ngx_list_push(&r->headers_out.headers));
if (cache_control_headers[0] == NULL) {
return NGX_ERROR;
}
cache_control_headers[0]->hash = 1;
ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
cache_control_headers[0]->value.len = strlen(cache_control);
cache_control_headers[0]->value.data =
reinterpret_cast<u_char*>(cache_control);
return NGX_OK;
}
// Returns false if the header wasn't found. Otherwise sets cache_control and
// returns true;
bool ps_get_cache_control(ngx_http_request_t* r, GoogleString* cache_control) {
// Use headers_out.cache_control instead of looking for Cache-Control in
// headers_out.headers, because if an upstream sent multiple Cache-Control
// headers they're already combined in headers_out.cache_control.
auto ccp = static_cast<ngx_table_elt_t**>(r->headers_out.cache_control.elts);
if (ccp == nullptr) {
return false; // Header not present.
}
bool first_segment = true;
for (ngx_uint_t i = 0; i < r->headers_out.cache_control.nelts; i++) {
if (ccp[i]->hash == 0) {
continue; // Elements with a hash of 0 are marked as excluded.
}
if (first_segment) {
first_segment = false;
} else {
cache_control->append(", ");
}
cache_control->append(reinterpret_cast<char*>(ccp[i]->value.data),
ccp[i]->value.len);
}
return true;
}
template<class Headers>
void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
// Standard nginx idiom for iterating over a list. See ngx_list.h
ngx_uint_t i;
const ngx_list_part_t* part = &from.part;
const ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
for (i = 0 ; /* void */; i++) {
if (i >= part->nelts) {
if (part->next == NULL) {
break;
}
part = part->next;
header = static_cast<ngx_table_elt_t*>(part->elts);
i = 0;
}
// Make sure we don't copy over headers that are unset.
if (header[i].hash == 0) {
continue;
}
StringPiece key = str_to_string_piece(header[i].key);
StringPiece value = str_to_string_piece(header[i].value);
to->Add(key, value);
}
}
} // namespace
void copy_response_headers_from_ngx(const ngx_http_request_t* r,
ResponseHeaders* headers) {
headers->set_major_version(r->http_version / 1000);
headers->set_minor_version(r->http_version % 1000);
copy_headers_from_table(r->headers_out.headers, headers);
headers->set_status_code(r->headers_out.status);
// Manually copy over the content type because it's not included in
// request_->headers_out.headers.
headers->Add(HttpAttributes::kContentType,
str_to_string_piece(r->headers_out.content_type));
// When we don't have a date header, set one with the current time.
if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
PosixTimer timer;
headers->SetDate(timer.NowMs());
}
// TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
headers->ComputeCaching();
}
void copy_request_headers_from_ngx(const ngx_http_request_t* r,
RequestHeaders* headers) {
// TODO(chaizhenhua): only allow RewriteDriver::kPassThroughRequestAttributes?
headers->set_major_version(r->http_version / 1000);
headers->set_minor_version(r->http_version % 1000);
copy_headers_from_table(r->headers_in.headers, headers);
}
// PSOL produces caching headers that need some changes before we can send them
// out. Make those changes and populate r->headers_out from pagespeed_headers.
ngx_int_t copy_response_headers_to_ngx(
ngx_http_request_t* r,
const ResponseHeaders& pagespeed_headers,
PreserveCachingHeaders preserve_caching_headers) {
ngx_http_headers_out_t* headers_out = &r->headers_out;
headers_out->status = pagespeed_headers.status_code();
ngx_int_t i;
for (i = 0 ; i < pagespeed_headers.NumAttributes() ; i++) {
// For IPRO cache misses, these gs_ variables may point to freed memory
// when nginx writes the headers to the output as the NgxBaseFetch instance
// that owns this memory gets released during request processing. So we
// copy these strings later on.
const GoogleString& name_gs = pagespeed_headers.Name(i);
const GoogleString& value_gs = pagespeed_headers.Value(i);
if (preserve_caching_headers == kPreserveAllCachingHeaders) {
if (StringCaseEqual(name_gs, "ETag") ||
StringCaseEqual(name_gs, "Expires") ||
StringCaseEqual(name_gs, "Date") ||
StringCaseEqual(name_gs, "Last-Modified") ||
StringCaseEqual(name_gs, "Cache-Control")) {
continue;
}
} else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
// Retain the original Cache-Control header, but send the recomputed
// values for all other cache-related headers.
if (StringCaseEqual(name_gs, "Cache-Control")) {
continue;
}
} // else we don't preserve any headers.
ngx_str_t name, value;
value.len = value_gs.size();
value.data = reinterpret_cast<u_char*>(
string_piece_to_pool_string(r->pool, value_gs.c_str()));
// To prevent the gzip module from clearing weak etags, we output them
// using a different name here. The etag header filter module runs behind
// the gzip compressors header filter, and will rename it to 'ETag'
if (StringCaseEqual(name_gs, "etag")
&& StringCaseStartsWith(value_gs, "W/")) {
name.len = strlen(kInternalEtagName);
name.data = reinterpret_cast<u_char*>(
const_cast<char*>(kInternalEtagName));
} else {
name.len = name_gs.size();
name.data = reinterpret_cast<u_char*>(
string_piece_to_pool_string(r->pool, name_gs.c_str()));
}
// In case string_piece_to_pool_string failed:
if (name.data == NULL || value.data == NULL) {
return NGX_ERROR;
}
// TODO(jefftk): If we're setting a cache control header we'd like to
// prevent any downstream code from changing it. Specifically, if we're
// serving a cache-extended resource the url will change if the resource
// does and so we've given it a long lifetime. If the site owner has done
// something like set all css files to a 10-minute cache lifetime, that
// shouldn't apply to our generated resources. See Apache code in
// net/instaweb/apache/header_util:AddResponseHeadersToRequest
// Make copies of name and value to put into headers_out.
if (STR_EQ_LITERAL(name, "Cache-Control")) {
ps_set_cache_control(r, reinterpret_cast<char*>(value.data));
continue;
} else if (STR_EQ_LITERAL(name, "Content-Type")) {
// Unlike all the other headers, content_type is just a string.
headers_out->content_type = value;
// We should not include the charset when determining content_type_len, so
// scan for the ';' that marks the start of the charset part.
for (ngx_uint_t i = 0; i < value.len; i++) {
if (value.data[i] == ';') {
break;
}
headers_out->content_type_len = i + 1;
}
// In ngx_http_test_content_type() nginx will allocate and calculate
// content_type_lowcase if we leave it as null.
headers_out->content_type_lowcase = NULL;
continue;
// TODO(oschaaf): are there any other headers we should not try to
// copy here?
} else if (STR_EQ_LITERAL(name, "Connection")) {
continue;
} else if (STR_EQ_LITERAL(name, "Keep-Alive")) {
continue;
} else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
continue;
} else if (STR_EQ_LITERAL(name, "Vary") && value.len
&& STR_EQ_LITERAL(value, "Accept-Encoding")) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
ctx->psol_vary_accept_only = true;
}
ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(
ngx_list_push(&headers_out->headers));
if (header == NULL) {
return NGX_ERROR;
}
header->hash = 1; // Include this header in the output.
header->key.data = name.data;
header->key.len = name.len;
header->value.data = value.data;
header->value.len = value.len;
// Populate the shortcuts to commonly used headers.
if (STR_EQ_LITERAL(name, "Date")) {
headers_out->date = header;
} else if (STR_EQ_LITERAL(name, "Etag")) {
headers_out->etag = header;
} else if (STR_EQ_LITERAL(name, "Expires")) {
headers_out->expires = header;
} else if (STR_EQ_LITERAL(name, "Last-Modified")) {
headers_out->last_modified = header;
} else if (STR_EQ_LITERAL(name, "Location")) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (ctx->location_field_set) {
headers_out->location = header;
}
} else if (STR_EQ_LITERAL(name, "Server")) {
headers_out->server = header;
} else if (STR_EQ_LITERAL(name, "Content-Length")) {
int64 len;
CHECK(pagespeed_headers.FindContentLength(&len));
headers_out->content_length_n = len;
headers_out->content_length = header;
} else if (STR_EQ_LITERAL(name, "Content-Encoding")) {
headers_out->content_encoding = header;
} else if (STR_EQ_LITERAL(name, "Refresh")) {
headers_out->refresh = header;
} else if (STR_EQ_LITERAL(name, "Content-Range")) {
headers_out->content_range = header;
} else if (STR_EQ_LITERAL(name, "Accept-Ranges")) {
headers_out->accept_ranges = header;
} else if (STR_EQ_LITERAL(name, "WWW-Authenticate")) {
headers_out->www_authenticate = header;
}
}
return NGX_OK;
}
namespace {
typedef struct {
NgxRewriteDriverFactory* driver_factory;
MessageHandler* handler;
} ps_main_conf_t;
typedef struct {
// If pagespeed is configured in some server block but not this one our
// per-request code will be invoked but server context will be null. In those
// cases we need to short circuit, not changing anything. Currently our
// header filter, body filter, and content handler all do this, but if anyone
// adds another way for nginx to give us a request to process we need to check
// there as well.
NgxServerContext* server_context;
ProxyFetchFactory* proxy_fetch_factory;
// Only used while parsing config. After we merge cfg_s and cfg_m you most
// likely want cfg_s->server_context->config() as options here will be NULL.
NgxRewriteOptions* options;
MessageHandler* handler;
} ps_srv_conf_t;
typedef struct {
NgxRewriteOptions* options;
MessageHandler* handler;
} ps_loc_conf_t;
namespace RequestRouting {
enum Response {
kError,
kStaticContent,
kInvalidUrl,
kPagespeedDisabled,
kBeacon,
kStatistics,
kGlobalStatistics,
kConsole,
kMessages,
kAdmin,
kCachePurge,
kGlobalAdmin,
kPagespeedSubrequest,
kErrorResponse,
kResource,
};
} // namespace RequestRouting
char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
// We want NGX_CONF_MULTI for some very old versions:
// https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
// but it's gone in recent revisions, so provide a compat #define if needed
#ifndef NGX_CONF_MULTI
#define NGX_CONF_MULTI 0
#endif
// TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
// not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
ngx_command_t ps_commands[] = {
{ ngx_string("pagespeed"),
NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
ps_main_configure,
NGX_HTTP_SRV_CONF_OFFSET,
0,
NULL },
{ ngx_string("pagespeed"),
NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
ps_srv_configure,
NGX_HTTP_SRV_CONF_OFFSET,
0,
NULL },
{ ngx_string("pagespeed"),
NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
ps_loc_configure,
NGX_HTTP_SRV_CONF_OFFSET,
0,
NULL },
ngx_null_command
};
bool ps_disabled(ps_srv_conf_t* cfg_s) {
return cfg_s->server_context == NULL ||
cfg_s->server_context->config()->unplugged();
}
void ps_ignore_sigpipe() {
struct sigaction act;
ngx_memzero(&act, sizeof(act));
act.sa_handler = SIG_IGN;
sigemptyset(&act.sa_mask);
act.sa_flags = 0;
sigaction(SIGPIPE, &act, NULL);
}
// Given a directory path that pagespeed needs, create it and set permissions so
// the worker can access, but only if needed.
char* ps_init_dir(const StringPiece& directive,
const StringPiece& path,
ngx_conf_t* cf) {
if (path.size() == 0 || path[0] != '/') {
return string_piece_to_pool_string(
cf->pool, net_instaweb::StrCat(directive, " ", path,
" must start with a slash"));
}
net_instaweb::StdioFileSystem file_system;
net_instaweb::NullMessageHandler message_handler;
GoogleString gs_path;
path.CopyToString(&gs_path);
if (!file_system.IsDir(gs_path.c_str(), &message_handler).is_true()) {
if (!file_system.RecursivelyMakeDir(path, &message_handler)) {
return string_piece_to_pool_string(
cf->pool, net_instaweb::StrCat(
directive, " path ", path,
" does not exist and could not be created."));
}
// Directory created, but may not be readable by the worker processes.
}
if (geteuid() != 0) {
return NULL; // We're not root, so we're staying whoever we are.
}
// chown if owner differs from nginx worker user.
ngx_core_conf_t* ccf = reinterpret_cast<ngx_core_conf_t*>(
ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
CHECK(ccf != NULL);
struct stat gs_stat;
if (stat(gs_path.c_str(), &gs_stat) != 0) {
return string_piece_to_pool_string(
cf->pool, net_instaweb::StrCat(
directive, " ", path, " stat() failed"));
}
if (gs_stat.st_uid != ccf->user) {
if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
return string_piece_to_pool_string(
cf->pool, net_instaweb::StrCat(
directive, " ", path, " unable to set permissions"));
}
}
return NULL;
}
// We support interpretation of nginx variables in some configuration settings,
// but we also need to support literal dollar signs in those same settings.
// Nginx has no good solution for this, so we define $dollar to expand to '$',
// which lets people include literal dollar signs if they need them.
ngx_int_t ps_dollar(
ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) {
v->valid = 1;
v->no_cacheable = 0;
v->not_found = 0;
v->data = reinterpret_cast<u_char*>(const_cast<char*>("$"));
v->len = 1;
return NGX_OK;
}
// Parse the configuration option represented by cf and add it to options,
// creating options if necessary.
char* ps_configure(ngx_conf_t* cf,
NgxRewriteOptions** options,
MessageHandler* handler,
net_instaweb::RewriteOptions::OptionScope option_scope) {
// args[0] is always "pagespeed"; ignore it.
ngx_uint_t n_args = cf->args->nelts - 1;
// In ps_commands we only register 'pagespeed' as taking up to
// five arguments, so this check should never fire.
CHECK(n_args <= NGX_PAGESPEED_MAX_ARGS);
StringPiece args[NGX_PAGESPEED_MAX_ARGS];
ngx_str_t* value = static_cast<ngx_str_t*>(cf->args->elts);
ngx_uint_t i;
for (i = 0 ; i < n_args ; i++) {
args[i] = str_to_string_piece(value[i+1]);
}
if (n_args == 1) {
if (StringCaseEqual(args[0], "on")) {
// safe to call if the setter is disabled
g_gzip_setter.EnableGZipForLocation(cf);
} else if (StringCaseEqual(args[0], "off")) {
g_gzip_setter.SetGZipForLocation(cf, false);
}
}
if (n_args == 2 && args[0].compare("gzip") == 0) {
if (args[1].compare("on") == 0) {
g_gzip_setter.SetGZipForLocation(cf, true);
} else if (args[1].compare("off") == 0) {
g_gzip_setter.SetGZipForLocation(cf, false);
} else {
char* error_message = string_piece_to_pool_string(
cf->pool, StringPiece("Invalid pagespeed gzip setting"));
return error_message;
}
return NGX_CONF_OK;
}
// Some options require the worker process to be able to read and write to
// a specific directory. Generally the master process is root while the
// worker is nobody, so we need to change permissions and create the directory
// if necessary.
if (n_args == 2 &&
(net_instaweb::StringCaseEqual("LogDir", args[0]) ||
net_instaweb::StringCaseEqual("FileCachePath", args[0]))) {
char* error_message = ps_init_dir(args[0], args[1], cf);
if (error_message != NULL) {
return error_message;
}
// The directory has been prepared, but we haven't actually parsed the
// directive yet. That happens below in ParseAndSetOptions().
}
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
if (*options == NULL) {
*options = new NgxRewriteOptions(cfg_m->driver_factory->thread_system());
}
ProcessScriptVariablesMode script_mode =
dynamic_cast<NgxRewriteDriverFactory*>(cfg_m->driver_factory)
->process_script_variables();
if (script_mode != ProcessScriptVariablesMode::kOff) {
// To be able to use '$', we map '$ps_dollar' to '$' via a script variable.
ngx_str_t name = ngx_string("ps_dollar");
ngx_http_variable_t* var =
ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE);
if (var == NULL) {
return const_cast<char*>(
"Failed to add global configuration variable for '$ps_dollar'");
}
var->get_handler = ps_dollar;
}
const char* status = (*options)->ParseAndSetOptions(
args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope, cf,
script_mode);
// nginx expects us to return a string literal but doesn't mark it const.
return const_cast<char*>(status);
}
char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
return ps_configure(cf, &cfg_s->options, cfg_s->handler,
net_instaweb::RewriteOptions::kProcessScopeStrict);
}
char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
return ps_configure(cf, &cfg_s->options, cfg_s->handler,
net_instaweb::RewriteOptions::kServerScope);
}
char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(
ngx_http_conf_get_module_loc_conf(cf, ngx_pagespeed));
return ps_configure(cf, &cfg_l->options, cfg_l->handler,
net_instaweb::RewriteOptions::kDirectoryScope);
}
void ps_cleanup_loc_conf(void* data) {
ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(data);
delete cfg_l->handler;
cfg_l->handler = NULL;
delete cfg_l->options;
cfg_l->options = NULL;
}
bool factory_deleted = false;
void ps_cleanup_srv_conf(void* data) {
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(data);
// destroy the factory on the first call, causing all worker threads
// to be shut down when we destroy any proxy_fetch_factories. This
// will prevent any queued callbacks to destroyed proxy fetch factories
// from being executed
if (!factory_deleted && cfg_s->server_context != NULL) {
if (active_driver_factory == cfg_s->server_context->factory()) {
active_driver_factory = NULL;
}
delete cfg_s->server_context->factory();
factory_deleted = true;
}
if (cfg_s->proxy_fetch_factory != NULL) {
delete cfg_s->proxy_fetch_factory;
cfg_s->proxy_fetch_factory = NULL;
}
delete cfg_s->handler;
cfg_s->handler = NULL;
delete cfg_s->options;
cfg_s->options = NULL;
}
void ps_cleanup_main_conf(void* data) {
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(data);
delete cfg_m->handler;
cfg_m->handler = NULL;
NgxRewriteDriverFactory::Terminate();
NgxRewriteOptions::Terminate();
// reset the factory deleted flag, so we will clean up properly next time,
// in case of a configuration reload.
// TODO(oschaaf): get rid of the factory_deleted flag
factory_deleted = false;
}
template <typename ConfT> ConfT* ps_create_conf(ngx_conf_t* cf) {
ConfT* cfg = static_cast<ConfT*>(ngx_pcalloc(cf->pool, sizeof(ConfT)));
if (cfg == NULL) {
return NULL;
}
cfg->handler = new GoogleMessageHandler();
return cfg;
}
void ps_set_conf_cleanup_handler(
ngx_conf_t* cf, void (func)(void*), void* data) { // NOLINT
ngx_pool_cleanup_t* cleanup_m = ngx_pool_cleanup_add(cf->pool, 0);
if (cleanup_m == NULL) {
ngx_conf_log_error(
NGX_LOG_ERR, cf, 0, "failed to register a cleanup handler");
} else {
cleanup_m->handler = func;
cleanup_m->data = data;
}
}
void terminate_process_context() {
if (active_driver_factory != NULL) {
// If we got here, that means we are in the cache loader/manager
// or did not get a chance to cleanup otherwise.
delete active_driver_factory;
active_driver_factory = NULL;
NgxBaseFetch::Terminate();
}
delete process_context;
process_context = NULL;
}
void* ps_create_main_conf(ngx_conf_t* cf) {
if (!process_context_cleanup_hooked) {
SystemRewriteDriverFactory::InitApr();
atexit(terminate_process_context);
process_context_cleanup_hooked = true;
}
ps_main_conf_t* cfg_m = ps_create_conf<ps_main_conf_t>(cf);
if (cfg_m == NULL) {
return NGX_CONF_ERROR;
}
CHECK(!factory_deleted);
NgxRewriteOptions::Initialize();
NgxRewriteDriverFactory::Initialize();
cfg_m->driver_factory = new NgxRewriteDriverFactory(
*process_context,
new SystemThreadSystem(),
"" /* hostname, not used */,
-1 /* port, not used */);
active_driver_factory = cfg_m->driver_factory;
active_driver_factory->LoggingInit(ngx_cycle->log, false);
cfg_m->driver_factory->Init();
ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
return cfg_m;
}
void* ps_create_srv_conf(ngx_conf_t* cf) {
ps_srv_conf_t* cfg_s = ps_create_conf<ps_srv_conf_t>(cf);
if (cfg_s == NULL) {
return NGX_CONF_ERROR;
}
ps_set_conf_cleanup_handler(cf, ps_cleanup_srv_conf, cfg_s);
return cfg_s;
}
void* ps_create_loc_conf(ngx_conf_t* cf) {
ps_loc_conf_t* cfg_l = ps_create_conf<ps_loc_conf_t>(cf);
if (cfg_l == NULL) {
return NGX_CONF_ERROR;
}
ps_set_conf_cleanup_handler(cf, ps_cleanup_loc_conf, cfg_l);
return cfg_l;
}
// nginx has hierarchical configuration. It maintains configurations at many
// levels. At various points it needs to merge configurations from different
// levels, and then it calls this. First it creates the configuration at the
// new level, parsing any pagespeed directives, then it merges in the
// configuration from the level above. This function should merge the parent
// configuration into the child. It's more complex than options->Merge() both
// because of the cases where the parent or child didn't have any pagespeed
// directives and because merging is order-dependent in the opposite way we'd
// like.
void ps_merge_options(NgxRewriteOptions* parent_options,
NgxRewriteOptions** child_options) {
if (parent_options == NULL) {
// Nothing to do.
} else if (*child_options == NULL) {
*child_options = parent_options->Clone();
} else { // Both non-null.
// Unfortunately, merging configuration options is order dependent. We'd
// like to just do (*child_options)->Merge(*parent_options)
// but then if we had:
// pagespeed RewriteLevel PassThrough
// server {
// pagespeed RewriteLevel CoreFilters
// }
// it would always be stuck on PassThrough.
NgxRewriteOptions* child_specific_options = *child_options;
*child_options = parent_options->Clone();
(*child_options)->Merge(*child_specific_options);
if (child_specific_options->clear_inherited_scripts()) {
// We don't want to inherit any inherited script lines from the parent
// options here, so we just stick to the child specific ones.
child_specific_options->CopyScriptLinesTo(*child_options);
} else {
// We append the child specific script lines to the parent's script lines
// so we preserve the order in which they will be executed at request time
child_specific_options->AppendScriptLinesTo(*child_options);
}
delete child_specific_options;
}
}
namespace {
int times_ps_merge_srv_conf_called = 0;
} // namespace
// Called exactly once per server block to merge the main configuration with the
// configuration for this server.
char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
times_ps_merge_srv_conf_called += 1;
ps_srv_conf_t* parent_cfg_s = static_cast<ps_srv_conf_t*>(parent);
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(child);
ps_merge_options(parent_cfg_s->options, &cfg_s->options);
if (cfg_s->options == NULL) {
return NGX_CONF_OK; // No pagespeed options; don't do anything.
}
// ServerContext needs a hostname and port, but I don't see how to get this
// and it ignores that a server can have multiple names and ports. Because
// the server context only needs them to make a unique identifier and to make
// debugging easier, substitute our own unique identifier.
// TODO(jefftk): either figure out how to get a hostname and port for this
// server block or change ServerContext not to ask for them.
int dummy_port = -times_ps_merge_srv_conf_called;
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
cfg_m->driver_factory->SetMainConf(parent_cfg_s->options);
cfg_s->server_context = cfg_m->driver_factory->MakeNgxServerContext(
"dummy_hostname", dummy_port);
#if (NGX_HTTP_V2)
// Save the variable index of the "http2" variable, so we can use it
// at request time to lookup whether that's on. We do this conditionally
// since NGINX will complain to the user (at [emerg] level!) if it doesn't
// know about it.
ngx_str_t name = ngx_string("http2");
cfg_s->server_context->set_ngx_http2_variable_index(
ngx_http_get_variable_index(cf, &name));
#endif
// The server context sets some options when we call global_options(). So
// let it do that, then merge in options we got from the config file.
// Once we do that we're done with cfg_s->options.
cfg_s->server_context->global_options()->Merge(*cfg_s->options);
NgxRewriteOptions* ngx_options = dynamic_cast<NgxRewriteOptions*>(
cfg_s->server_context->global_options());
cfg_s->options->CopyScriptLinesTo(ngx_options);
delete cfg_s->options;
cfg_s->options = NULL;
if (!cfg_s->server_context->global_options()->unplugged()) {
// Validate FileCachePath
GoogleMessageHandler handler;
const char* file_cache_path =
cfg_s->server_context->config()->file_cache_path().c_str();
if (file_cache_path[0] == '\0') {
if (!cfg_s->server_context->global_options()->standby()) {
return const_cast<char*>("FileCachePath must be set, even for standby");
} else {
return const_cast<char*>("FileCachePath must be set");
}
} else if (!cfg_m->driver_factory->file_system()->IsDir(
file_cache_path, &handler).is_true()) {
return const_cast<char*>(
"FileCachePath must be an nginx-writeable directory");
}
}
return NGX_CONF_OK;
}
char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(child);
if (cfg_l->options == NULL) {
// No directory specific options.
return NGX_CONF_OK;
}
// While you can't put a "location" block inside a "location" block you can
// put an "if" block inside a "location" block, which is implemented by making
// a pretend "location" block. In this case we may have pagespeed options
// from the parent "location" block as well as from the current locationish
// "if" block.
ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
if (parent_cfg_l->options != NULL) {
// Rebase our options off of the ones defined in the parent location block.
ps_merge_options(parent_cfg_l->options, &cfg_l->options);
return NGX_CONF_OK;
}
// Pagespeed options are defined in this location block, and it either has no
// parent (typical case) or is an if block whose parent location block defines
// no pagespeed options. Base our options off of those in the server block.
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
if (ps_disabled(cfg_s)) {
// Pagespeed options cannot be defined only in location blocks. There must
// be at least a single "pagespeed off" in the main block or a server
// block.
return NGX_CONF_OK;
}
// If we get here we have parent options ("global options") from cfg_s, child
// options ("directory specific options") from cfg_l, and no options from
// parent_cfg_l. Rebase the directory specific options on the global options.
ps_merge_options(cfg_s->server_context->config(), &cfg_l->options);
return NGX_CONF_OK;
}
// _ef_ is a shorthand for ETag Filter
ngx_http_output_header_filter_pt ngx_http_ef_next_header_filter;
bool ps_is_https(ngx_http_request_t* r) {
// Based on ngx_http_variable_scheme.
#if (NGX_HTTP_SSL)
return r->connection->ssl;
#endif
return false;
}
int ps_determine_port(ngx_http_request_t* r) {
// Return -1 if the port isn't specified, the port number otherwise.
//
// If a Host header was provided, get the host from that. Otherwise fall back
// to the local port of the incoming connection.
int port = -1;
ngx_table_elt_t* host = r->headers_in.host;
if (host != NULL) {
// Host headers can look like:
//
// www.example.com // normal
// www.example.com:8080 // port specified
// 127.0.0.1 // IPv4
// 127.0.0.1:8080 // IPv4 with port
// [::1] // IPv6
// [::1]:8080 // IPv6 with port
//
// The IPv6 ones are the annoying ones, but the square brackets allow us to
// disambiguate. To find the port number, we can say:
//
// 1) Take the text after the final colon.
// 2) If all of those characters are digits, that's your port number
//
// In the case of a plain IPv6 address with no port number, the text after
// the final colon will include a ']', so we'll stop processing.
StringPiece host_str = str_to_string_piece(host->value);
size_t colon_index = host_str.rfind(":");
if (colon_index == host_str.npos) {
return -1;
}
// Strip everything up to and including the final colon.
host_str.remove_prefix(colon_index + 1);
bool ok = StringToInt(host_str, &port);
if (!ok) {
// Might be malformed port, or just IPv6 with no port specified.
return -1;
}
return port;
}
// Based on ngx_http_variable_server_port.
#if (NGX_HAVE_INET6)
if (r->connection->local_sockaddr->sa_family == AF_INET6) {
port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
r->connection->local_sockaddr)->sin6_port);
}
#endif
if (port == -1 /* still need port */) {
port = ntohs(reinterpret_cast<struct sockaddr_in*>(
r->connection->local_sockaddr)->sin_port);
}
return port;
}
} // namespace
StringPiece ps_determine_host(ngx_http_request_t* r) {
StringPiece host = str_to_string_piece(r->headers_in.server);
if (host.size() == 0) {
// If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
// fall back to server IP address. Based on ngx_http_variable_server_addr.
ngx_str_t s;
u_char addr[NGX_SOCKADDR_STRLEN];
s.len = NGX_SOCKADDR_STRLEN;
s.data = addr;
ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &s, 0);
if (rc != NGX_OK) {
s.len = 0;
}
host = str_to_string_piece(s);
}
return host;
}
namespace {
GoogleString ps_determine_url(ngx_http_request_t* r) {
int port = ps_determine_port(r);
GoogleString port_string;
if ((ps_is_https(r) && (port == 443 || port == -1)) ||
(!ps_is_https(r) && (port == 80 || port == -1))) {
// No port specifier needed for requests on default ports.
port_string = "";
} else {
port_string = StrCat(":", IntegerToString(port));
}
StringPiece host = ps_determine_host(r);
return StrCat(ps_is_https(r) ? "https://" : "http://",
host, port_string, str_to_string_piece(r->unparsed_uri));
}
// we are still at pagespeed phase
ngx_int_t ps_decline_request(ngx_http_request_t* r) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
CHECK(ctx != NULL);
ctx->driver->Cleanup();
ctx->driver = NULL;
ctx->location_field_set = false;
ctx->psol_vary_accept_only = false;
// re init ctx
ctx->html_rewrite = true;
ctx->in_place = false;
ps_release_base_fetch(ctx);
ps_set_buffered(r, false);
r->count++;
r->phase_handler++;
//restore read_event_handler to what it was in ps_async_wait_response
r->read_event_handler = ngx_http_block_reading;
r->write_event_handler = ngx_http_core_run_phases;
ngx_http_core_run_phases(r);
ngx_http_run_posted_requests(r->connection);
return NGX_DONE;
}
ngx_int_t ps_async_wait_response(ngx_http_request_t* r) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
CHECK(ctx != NULL);
r->count++;
// While we wait for PSOL to complete an async operation, there is a chance
// that the underlying connection gets closed, or a http/2 RST_STREAM is
// received before the async operation completes. In that case we don't want
// to continue processing this flow. So we override the requests's read event
// handler with one that will make nginx abort request processing and execute
// our cleanup handlers instead of resuming request processing.
r->read_event_handler = ngx_http_test_reading;
r->write_event_handler = ngx_http_request_empty_handler;
ps_set_buffered(r, true);
// We don't need to add a timer here, as it will be set by nginx.
return NGX_DONE;
}
// Populate cfg_* with configuration information for this request.
// Thin wrappers around ngx_http_get_module_*_conf and cast.
ps_srv_conf_t* ps_get_srv_config(ngx_http_request_t* r) {
return static_cast<ps_srv_conf_t*>(
ngx_http_get_module_srv_conf(r, ngx_pagespeed));
}
ps_loc_conf_t* ps_get_loc_config(ngx_http_request_t* r) {
return static_cast<ps_loc_conf_t*>(
ngx_http_get_module_loc_conf(r, ngx_pagespeed));
}
RewriteOptions* ps_determine_remote_options(ps_srv_conf_t* cfg_s) {
if (!cfg_s || !cfg_s->server_context ||
!cfg_s->server_context->global_options()) {
return NULL;
}
if (!cfg_s->server_context->global_options()
->remote_configuration_url()
.empty()) {
RewriteOptions* remote_options =
cfg_s->server_context->global_options()->Clone();
// This fetch is blocking for up to remote_configuration_timeout_ms ms.
cfg_s->server_context->GetRemoteOptions(remote_options, false);
return remote_options;
}
return NULL;
}
// Wrapper around GetQueryOptions()
RewriteOptions* ps_determine_request_options(
ngx_http_request_t* r,
const RewriteOptions* domain_options, /* may be null */
RequestHeaders* request_headers,
ResponseHeaders* response_headers,
RequestContextPtr request_context,
ps_srv_conf_t* cfg_s,
GoogleUrl* url,
GoogleString* pagespeed_query_params,
GoogleString* pagespeed_option_cookies) {
// Sets option from request headers and url.
RewriteQuery rewrite_query;
if (!cfg_s->server_context->GetQueryOptions(
request_context, domain_options, url, request_headers,
response_headers, &rewrite_query)) {
// Failed to parse query params or request headers. Treat this as if there
// were no query params given.
ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
"ps_determine_request_options: parsing headers or query params failed.");
return NULL;
}
*pagespeed_query_params =
rewrite_query.pagespeed_query_params().ToEscapedString();
*pagespeed_option_cookies =
rewrite_query.pagespeed_option_cookies().ToEscapedString();
// Will be NULL if there aren't any options set with query params or in
// headers.
return rewrite_query.ReleaseOptions();
}
// Check whether this visitor is already in an experiment. If they're not,
// classify them into one by setting a cookie. Then set options appropriately
// for their experiment.
//
// See InstawebContext::SetExperimentStateAndCookie()
bool ps_set_experiment_state_and_cookie(ngx_http_request_t* r,
RequestHeaders* request_headers,
RewriteOptions* options,
const StringPiece& host) {
CHECK(options->running_experiment());
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
bool need_cookie =
cfg_s->server_context->experiment_matcher()->ClassifyIntoExperiment(
*request_headers, *cfg_s->server_context->user_agent_matcher(),
options);
if (need_cookie && host.length() > 0) {
PosixTimer timer;
int64 time_now_ms = timer.NowMs();
int64 expiration_time_ms = (time_now_ms +
options->experiment_cookie_duration_ms());
// TODO(jefftk): refactor SetExperimentCookie to expose the value we want to
// set on the cookie.
int state = options->experiment_id();
GoogleString expires;
ConvertTimeToString(expiration_time_ms, &expires);
GoogleString value = StringPrintf(
"%s=%s; Expires=%s; Domain=.%s; Path=/",
experiment::kExperimentCookie,
experiment::ExperimentStateToCookieString(state).c_str(),
expires.c_str(), host.as_string().c_str());
// Set the PagespeedExperiment cookie.
ngx_table_elt_t* cookie = static_cast<ngx_table_elt_t*>(
ngx_list_push(&r->headers_out.headers));
if (cookie == NULL) {
return false;
}
cookie->hash = 1; // Include this header in the response.
ngx_str_set(&cookie->key, "Set-Cookie");
// It's not safe to use value.c_str here because cookie header only keeps a
// pointer to the string data.
cookie->value.data = reinterpret_cast<u_char*>(
string_piece_to_pool_string(r->pool, value));
cookie->value.len = value.size();
}
return true;
}
// There are many sources of options:
// - the request (query parameters, headers, and cookies)
// - location block
// - global server options
// - experiment framework
// Consider them all, returning appropriate options for this request, of which
// the caller takes ownership. If the only applicable options are global,
// set options to NULL so we can use server_context->global_options().
bool ps_determine_options(ngx_http_request_t* r,
RequestHeaders* request_headers,
ResponseHeaders* response_headers,
RewriteOptions** options,
RequestContextPtr request_context,
ps_srv_conf_t* cfg_s,
GoogleUrl* url,
GoogleString* pagespeed_query_params,
GoogleString* pagespeed_option_cookies,
bool html_rewrite) {
ps_loc_conf_t* cfg_l = ps_get_loc_config(r);
// Global options for this server. Never null.
RewriteOptions* global_options = cfg_s->server_context->global_options();
// Directory-specific options, usually null. They've already been rebased off
// of the global options as part of the configuration process.
RewriteOptions* directory_options = cfg_l->options;
// Request-specific options, nearly always null. If set they need to be
// rebased on the directory options or the global options.
RewriteOptions* request_options = ps_determine_request_options(
r, directory_options, request_headers, response_headers, request_context,
cfg_s, url, pagespeed_query_params, pagespeed_option_cookies);
bool have_request_options = request_options != NULL;
// Because the caller takes ownership of any options we return, the only
// situation in which we can avoid allocating a new RewriteOptions is if the
// global options are ok as they are and we don't have script variables we
// need to evaluate at this point.
NgxRewriteOptions* ngx_global_options =
dynamic_cast<NgxRewriteOptions*>(global_options);
if (!have_request_options && directory_options == NULL &&
!global_options->running_experiment() &&
ngx_global_options->script_lines().size() == 0) {
return true;
}
// Start with directory options if we have them, otherwise request options.
if (directory_options != NULL) {
if (*options != NULL) {
(*options)->Merge(*directory_options);
} else {
*options = directory_options->Clone();
}
} else {
if (*options == NULL) {
*options = global_options->Clone();
}
}
NgxRewriteDriverFactory* ngx_factory =
dynamic_cast<NgxRewriteDriverFactory*>(cfg_s->server_context->factory());
NgxRewriteOptions* ngx_options = dynamic_cast<NgxRewriteOptions*>(*options);
// ExecuteScriptVariables() sets 'pagespeed off' on ngx_options when execution
// fails and then returns false. When that happens we return, as we don't want
// to allow enabling pagespeed by request and execute without the intended
// configuration.
if (!ngx_options->ExecuteScriptVariables(r, cfg_s->handler, ngx_factory)) {
return false;
}
// Modify our options in response to request options if specified.
if (have_request_options) {
(*options)->Merge(*request_options);
delete request_options;
request_options = NULL;
}
// If we're running an experiment and processing html then modify our options
// in response to the experiment. Except we generally don't want experiments
// to be contaminated with unexpected settings, so ignore experiments if we
// have request-specific options. Unless EnrollExperiment is on, probably set
// by a query parameter, in which case we want to go ahead and apply the
// experimental settings even if it means bad data, because we're just seeing
// what it looks like.
if ((*options)->running_experiment() &&
html_rewrite &&
(!have_request_options ||
(*options)->enroll_experiment())) {
bool ok = ps_set_experiment_state_and_cookie(
r, request_headers, *options, url->Host());
if (!ok) {
delete *options;
*options = NULL;
return false;
}
}
return true;
}
// Fix URL based on X-Forwarded-Proto.
// http://code.google.com/p/modpagespeed/issues/detail?id=546 For example, if
// Apache gives us the URL "http://www.example.com/" and there is a header:
// "X-Forwarded-Proto: https", then we update this base URL to
// "https://www.example.com/". This only ever changes the protocol of the url.
//
// Returns true if it modified url, false otherwise.
bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
// First check for an X-Forwarded-Proto header.
const ngx_str_t* x_forwarded_proto_header = NULL;
ngx_table_elt_t* header;
NgxListIterator it(&(r->headers_in.headers.part));
while ((header = it.Next()) != NULL) {
if (STR_CASE_EQ_LITERAL(header->key, "X-Forwarded-Proto")) {
x_forwarded_proto_header = &header->value;
break;
}
}
if (x_forwarded_proto_header == NULL) {
return false; // No X-Forwarded-Proto header found.
}
StringPiece x_forwarded_proto =
str_to_string_piece(*x_forwarded_proto_header);
if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
LOG(WARNING) << "Unsupported X-Forwarded-Proto: " << x_forwarded_proto
<< " for URL " << url << " protocol not changed.";
return false;
}
StringPiece url_sp(*url);
StringPiece::size_type colon_pos = url_sp.find(":");
if (colon_pos == StringPiece::npos) {
return false; // URL appears to have no protocol; give up.
}
// Replace URL protocol with that specified in X-Forwarded-Proto.
*url = StrCat(x_forwarded_proto, url_sp.substr(colon_pos));
return true;
}
bool is_pagespeed_subrequest(ngx_http_request_t* r) {
ngx_table_elt_t* user_agent_header = r->headers_in.user_agent;
if (user_agent_header == NULL) {
return false;
}
StringPiece user_agent = str_to_string_piece(user_agent_header->value);
return (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos);
}
void ps_release_base_fetch(ps_request_ctx_t* ctx) {
// In the normal flow BaseFetch doesn't delete itself in HandleDone() because
// we still need to receive notification via pipe and call
// CollectAccumulatedWrites. If there's an error and we're cleaning up early
// then HandleDone() hasn't been called yet and we need the base fetch to wait
// for that and then delete itself.
if (ctx->base_fetch != NULL) {
ctx->base_fetch->Detach();
ctx->base_fetch = NULL;
}
}
// TODO(chaizhenhua): merge into NgxBaseFetch ctor
void ps_create_base_fetch(StringPiece url,
ps_request_ctx_t* ctx,
RequestContextPtr request_context,
RequestHeaders* request_headers,
NgxBaseFetchType type,
const RewriteOptions* options) {
CHECK(ctx->base_fetch == NULL) << "Pre-existing base fetch!";
ngx_http_request_t* r = ctx->r;
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
// Handles its own deletion. We need to call Release when we're done with
// it, and call Done() on the associated parent (Proxy or Resource) fetch. If
// we fail before creating the associated fetch then we need to call Done() on
// the BaseFetch ourselves.
ctx->base_fetch = new NgxBaseFetch(url, r, cfg_s->server_context, request_context,
ctx->preserve_caching_headers, type,
options);
ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
}
void ps_release_request_context(void* data) {
ps_request_ctx_t* ctx = static_cast<ps_request_ctx_t*>(data);
// proxy_fetch deleted itself if we called Done(), but if an error happened
// before then we need to tell it to delete itself.
//
// If this is a resource fetch then proxy_fetch was never initialized.
if (ctx->proxy_fetch != NULL) {
ctx->proxy_fetch->Done(false /* failure */);
ctx->proxy_fetch = NULL;
}
if (ctx->inflater_ != NULL) {
delete ctx->inflater_;
ctx->inflater_ = NULL;
}
if (ctx->driver != NULL) {
ctx->driver->Cleanup();
ctx->driver = NULL;
}
if (ctx->recorder != NULL) {
// Deletes recorder.
ctx->recorder->DoneAndSetHeaders(NULL, false /* incomplete response */);
ctx->recorder = NULL;
}
ps_release_base_fetch(ctx);
delete ctx;
}
// Set us up for processing a request. Creates a request context and determines
// which handler should deal with the request.
RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
if (ps_disabled(cfg_s)) {
// Not enabled for this server block.
return RequestRouting::kPagespeedDisabled;
}
if (ngx_terminate || ngx_exiting) {
return RequestRouting::kError;
}
if (r->err_status != 0) {
return RequestRouting::kErrorResponse;
}
GoogleString url_string = ps_determine_url(r);
GoogleUrl url(url_string);
if (!url.IsWebValid()) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "invalid url");
// Let nginx deal with the error however it wants; we will see a NULL ctx in
// the body filter or content handler and do nothing.
return RequestRouting::kInvalidUrl;
}
if (is_pagespeed_subrequest(r)) {
return RequestRouting::kPagespeedSubrequest;
} else if (
url.PathSansLeaf() == dynamic_cast<NgxRewriteDriverFactory*>(
cfg_s->server_context->factory())->static_asset_prefix()) {
return RequestRouting::kStaticContent;
}
const NgxRewriteOptions* global_options = cfg_s->server_context->config();
StringPiece path = url.PathSansQuery();
if (StringCaseEqual(path, global_options->statistics_path()) &&
global_options->StatisticsAccessAllowed(url)) {
return RequestRouting::kStatistics;
} else if (StringCaseEqual(path, global_options->global_statistics_path()) &&
global_options->GlobalStatisticsAccessAllowed(url)) {
return RequestRouting::kGlobalStatistics;
} else if (StringCaseEqual(path, global_options->console_path()) &&
global_options->ConsoleAccessAllowed(url)) {
return RequestRouting::kConsole;
} else if (StringCaseEqual(path, global_options->messages_path()) &&
global_options->MessagesAccessAllowed(url)) {
return RequestRouting::kMessages;
} else if (
// The admin handlers get everything under a path (/path/*) while all the
// other handlers only get exact matches (/path). So match all paths
// starting with the handler path.
!global_options->admin_path().empty() &&
StringCaseStartsWith(path, global_options->admin_path()) &&
global_options->AdminAccessAllowed(url)) {
return RequestRouting::kAdmin;
} else if (!global_options->global_admin_path().empty() &&
StringCaseStartsWith(path, global_options->global_admin_path()) &&
global_options->GlobalAdminAccessAllowed(url)) {
return RequestRouting::kGlobalAdmin;
} else if (global_options->enable_cache_purge() &&
!global_options->purge_method().empty() &&
(global_options->purge_method() ==
str_to_string_piece(r->method_name))) {
return RequestRouting::kCachePurge;
}
const GoogleString* beacon_url;
if (ps_is_https(r)) {
beacon_url = &(global_options->beacon_url().https);
} else {
beacon_url = &(global_options->beacon_url().http);
}
if (url.PathSansQuery() == StringPiece(*beacon_url)) {
return RequestRouting::kBeacon;
}
return RequestRouting::kResource;
}
ngx_int_t ps_resource_handler(ngx_http_request_t* r,
bool html_rewrite,
RequestRouting::Response response_category) {
if (r != r->main) {
return NGX_DECLINED;
}
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (ngx_terminate || ngx_exiting) {
cfg_s->server_context->message_handler()->Message(
kInfo, "ps_resource_handler declining: nginx worker is shutting down");
if (ctx == NULL) {
return NGX_DECLINED;
}
ps_release_base_fetch(ctx);
return NGX_DECLINED;
}
CHECK(!(html_rewrite && (ctx == NULL || ctx->html_rewrite == false)));
if (!html_rewrite &&
r->method != NGX_HTTP_GET &&
r->method != NGX_HTTP_HEAD &&
r->method != NGX_HTTP_POST &&
response_category != RequestRouting::kCachePurge) {
return NGX_DECLINED;
}
GoogleString url_string = ps_determine_url(r);
GoogleUrl url(url_string);
if (!url.IsWebValid()) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "invalid url");
return NGX_DECLINED;
}
scoped_ptr<RequestHeaders> request_headers(new RequestHeaders);
scoped_ptr<ResponseHeaders> response_headers(new ResponseHeaders);
copy_request_headers_from_ngx(r, request_headers.get());
copy_response_headers_from_ngx(r, response_headers.get());
RequestContextPtr request_context(
cfg_s->server_context->NewRequestContext(r));
GoogleString pagespeed_query_params;
GoogleString pagespeed_option_cookies;
RewriteOptions* options = ps_determine_remote_options(cfg_s);
if (!ps_determine_options(r, request_headers.get(), response_headers.get(),
&options, request_context, cfg_s, &url,
&pagespeed_query_params, &pagespeed_option_cookies,
html_rewrite)) {
return NGX_ERROR;
}
// Take ownership of custom_options.
scoped_ptr<RewriteOptions> custom_options(options);
if (options == NULL) {
options = cfg_s->server_context->global_options();
}
request_context->set_options(options->ComputeHttpOptions());
// ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
// parameters. Keep url_string in sync with url.
url.Spec().CopyToString(&url_string);
if (cfg_s->server_context->global_options()->respect_x_forwarded_proto()) {
bool modified_url = ps_apply_x_forwarded_proto(r, &url_string);
if (modified_url) {
url.Reset(url_string);
CHECK(url.IsWebValid()) << "The output of ps_apply_x_forwarded_proto"
<< " should always be a valid url because it only"
<< " changes the scheme between http and https.";
}
}
bool pagespeed_resource =
!html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
bool is_an_admin_handler =
response_category == RequestRouting::kStatistics ||
response_category == RequestRouting::kGlobalStatistics ||
response_category == RequestRouting::kConsole ||
response_category == RequestRouting::kAdmin ||
response_category == RequestRouting::kGlobalAdmin ||
response_category == RequestRouting::kCachePurge;
// Normally if we're disabled we won't handle any requests, but if we're in
// standby mode we do want to handle requests for .pagespeed. resources.
if (options->unplugged() ||
(!options->enabled() && !pagespeed_resource)) {
// Disabled via query params or request headers.
return NGX_DECLINED;
}
if (!html_rewrite) {
// create request ctx
CHECK(ctx == NULL);
ctx = new ps_request_ctx_t();
ctx->r = r;
ctx->html_rewrite = false;
ctx->in_place = false;
ctx->follow_flushes = options->follow_flushes();
ctx->preserve_caching_headers = kDontPreserveHeaders;
// See build_context_for_request() in mod_instaweb.cc
// TODO(jefftk): Is this the right place to be modifying caching headers for
// html fetches? Or should that be done later, in the headers flow for
// filter mode, rather than here in resource fetch mode?
if (!options->modify_caching_headers()) {
ctx->preserve_caching_headers = kPreserveAllCachingHeaders;
} else if (!options->IsDownstreamCacheIntegrationEnabled()) {
// Downstream cache integration is not enabled. Disable original
// Cache-Control headers.
ctx->preserve_caching_headers = kDontPreserveHeaders;
} else if (!pagespeed_resource && !is_an_admin_handler) {
ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
// Downstream cache integration is enabled. If a rebeaconing key has been
// configured and there is a ShouldBeacon header with the correct key,
// disable original Cache-Control headers so that the instrumented page is
// served out with no-cache.
StringPiece should_beacon(request_headers->Lookup1(kPsaShouldBeacon));
if (options->MatchesDownstreamCacheRebeaconingKey(should_beacon)) {
ctx->preserve_caching_headers = kDontPreserveHeaders;
}
}
ctx->recorder = NULL;
ctx->url_string = url_string;
ctx->location_field_set = false;
ctx->psol_vary_accept_only = false;
// Set up a cleanup handler on the request.
ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
if (cleanup == NULL) {
ps_release_request_context(ctx);
return NGX_ERROR;
}
cleanup->handler = ps_release_request_context;
cleanup->data = ctx;
ngx_http_set_ctx(r, ctx, ngx_pagespeed);
}
if (pagespeed_resource) {
// TODO(jefftk): Set using_spdy appropriately. See
// ProxyInterface::ProxyRequestCallback
ps_create_base_fetch(url.Spec(), ctx, request_context,
request_headers.release(), kPageSpeedResource,
options);
ResourceFetch::Start(
url,
custom_options.release() /* null if there aren't custom options */,
cfg_s->server_context, ctx->base_fetch);
return ps_async_wait_response(r);
} else if (is_an_admin_handler) {
ps_create_base_fetch(url.Spec(), ctx, request_context,
request_headers.release(), kAdminPage, options);
QueryParams query_params;
query_params.ParseFromUrl(url);
PosixTimer timer;
int64 now_ms = timer.NowMs();
ctx->base_fetch->response_headers()->SetDateAndCaching(
now_ms, 0 /* max-age */, ", no-cache");
if (response_category == RequestRouting::kStatistics ||
response_category == RequestRouting::kGlobalStatistics) {
cfg_s->server_context->StatisticsPage(
response_category == RequestRouting::kGlobalStatistics,
query_params,
cfg_s->server_context->config(),
ctx->base_fetch);
} else if (response_category == RequestRouting::kConsole) {
cfg_s->server_context->ConsoleHandler(
*cfg_s->server_context->config(),
AdminSite::kStatistics,
query_params,
ctx->base_fetch);
} else if (response_category == RequestRouting::kAdmin ||
response_category == RequestRouting::kGlobalAdmin) {
cfg_s->server_context->AdminPage(
response_category == RequestRouting::kGlobalAdmin,
url,
query_params,
custom_options == NULL ? cfg_s->server_context->config()
: custom_options.get(),
ctx->base_fetch);
} else if (response_category == RequestRouting::kCachePurge) {
AdminSite* admin_site = cfg_s->server_context->admin_site();
admin_site->PurgeHandler(url_string,
cfg_s->server_context->cache_path(),
ctx->base_fetch);
} else {
CHECK(false);
}
return ps_async_wait_response(r);
} else if (!html_rewrite && response_category == RequestRouting::kResource) {
bool is_proxy = false;
GoogleString mapped_url;
GoogleString host_header;
if (options->domain_lawyer()->MapOriginUrl(
url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
ps_create_base_fetch(url.Spec(), ctx, request_context,
request_headers.release(), kPageSpeedProxy, options);
RewriteDriver* driver;
if (custom_options.get() == NULL) {
driver = cfg_s->server_context->NewRewriteDriver(
ctx->base_fetch->request_context());
} else {
driver = cfg_s->server_context->NewCustomRewriteDriver(
custom_options.release(), ctx->base_fetch->request_context());
}
driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
driver->set_pagespeed_query_params(pagespeed_query_params);
driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
cfg_s->proxy_fetch_factory->StartNewProxyFetch(
mapped_url, ctx->base_fetch, driver, NULL /*property_callback*/,
NULL /*original_content_fetch*/);
return ps_async_wait_response(r);
}
}
if (html_rewrite && options->IsAllowed(url.Spec())) {
ps_create_base_fetch(url.Spec(), ctx, request_context,
request_headers.release(), kHtmlTransform, options);
// Do not store driver in request_context, it's not safe.
RewriteDriver* driver;
// If we don't have custom options we can use NewRewriteDriver which reuses
// rewrite drivers and so is faster because there's no wait to construct
// them. Otherwise we have to build a new one every time.
if (custom_options.get() == NULL) {
driver = cfg_s->server_context->NewRewriteDriver(
ctx->base_fetch->request_context());
} else {
// NewCustomRewriteDriver takes ownership of custom_options.
driver = cfg_s->server_context->NewCustomRewriteDriver(
custom_options.release(), ctx->base_fetch->request_context());
}
driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
driver->set_pagespeed_query_params(pagespeed_query_params);
driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
// TODO(jefftk): FlushEarlyFlow would go here.
ProxyFetchPropertyCallbackCollector* property_callback =
ProxyFetchFactory::InitiatePropertyCacheLookup(
!html_rewrite /* is_resource_fetch */,
url,
cfg_s->server_context,
options,
ctx->base_fetch);
// Will call StartParse etc. The rewrite driver will take care of deleting
// itself if necessary.
ctx->proxy_fetch = cfg_s->proxy_fetch_factory->CreateNewProxyFetch(
url_string, ctx->base_fetch, driver,
property_callback,
NULL /* original_content_fetch */);
ctx->proxy_fetch->set_trusted_input(true);
return NGX_OK;
}
if (options->in_place_rewriting_enabled() &&
options->enabled() &&
options->IsAllowed(url.Spec())) {
ps_create_base_fetch(url.Spec(), ctx, request_context,
request_headers.release(), kIproLookup, options);
// Do not store driver in request_context, it's not safe.
RewriteDriver* driver;
if (custom_options.get() == NULL) {
driver = cfg_s->server_context->NewRewriteDriver(
ctx->base_fetch->request_context());
} else {
// NewCustomRewriteDriver takes ownership of custom_options.
driver = cfg_s->server_context->NewCustomRewriteDriver(
custom_options.release(), ctx->base_fetch->request_context());
}
driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
ctx->driver = driver;
cfg_s->server_context->message_handler()->Message(
kInfo, "Trying to serve rewritten resource in-place: %s",
url_string.c_str());
ctx->in_place = true;
ctx->driver->FetchInPlaceResource(
url, false /* proxy_mode */, ctx->base_fetch);
return ps_async_wait_response(r);
}
// NOTE: We are using the below debug message as is for some of our system
// tests. So, be careful about test breakages caused by changing or
// removing this line.
ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0,
"Passing on content handling for non-pagespeed resource '%s'",
url_string.c_str());
CHECK(ctx->base_fetch == NULL);
// set html_rewrite flag.
ctx->html_rewrite = true;
return NGX_DECLINED;
}
// Send each buffer in the chain to the proxy_fetch for optimization.
// Eventually it will make it's way, optimized, to base_fetch.
void ps_send_to_pagespeed(ngx_http_request_t* r,
ps_request_ctx_t* ctx,
ps_srv_conf_t* cfg_s,
ngx_chain_t* in) {
ngx_chain_t* cur;
int last_buf = 0;
for (cur = in; cur != NULL; cur = cur->next) {
last_buf = cur->buf->last_buf;
// Buffers are not really the last buffer until they've been through
// pagespeed.
cur->buf->last_buf = 0;
CHECK(ctx->proxy_fetch != NULL);
if (ctx->inflater_ == NULL) {
ctx->proxy_fetch->Write(
StringPiece(reinterpret_cast<char*>(cur->buf->pos),
cur->buf->last - cur->buf->pos), cfg_s->handler);
} else {
char buf[kStackBufferSize];
ctx->inflater_->SetInput(reinterpret_cast<char*>(cur->buf->pos),
cur->buf->last - cur->buf->pos);
while (ctx->inflater_->HasUnconsumedInput()) {
int num_inflated_bytes = ctx->inflater_->InflateBytes(
buf, kStackBufferSize);
if (num_inflated_bytes < 0) {
cfg_s->handler->Message(kWarning, "Corrupted inflation");
} else if (num_inflated_bytes > 0) {
ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes),
cfg_s->handler);
}
}
}
if (cur->buf->flush && ctx->follow_flushes) {
// Calling ctx->proxy_fetch->Flush(cfg_s->handler) will be a no-op here,
// unless we have follow_flushes or flush_html enabled. Note that PSOL
// might aggregate multiple flushes into 1, and actually flush a little bit
// later due to html parser state and earlier scheduled operations.
// Also, unless we also set the flush flag on the nginx buffers we won't
// actually flush.
// Note that too many flushes could harm optimization over larger html
// fragments as PSOL gets less context to work with, e.g. it can't combine
// two css files if a flush happens in between.
ctx->proxy_fetch->Flush(cfg_s->handler);
}
// We're done with buffers as we pass them through, so mark them as sent as
// we go.
cur->buf->pos = cur->buf->last;
}
if (last_buf) {
ctx->proxy_fetch->Done(true /* success */);
ctx->proxy_fetch = NULL; // ProxyFetch deletes itself on Done().
}
}
#ifndef ngx_http_clear_etag
// The ngx_http_clear_etag(r) macro was added in 1.3.3. Backport it if it's not
// present.
#define ngx_http_clear_etag(r) \
if (r->headers_out.etag) { \
r->headers_out.etag->hash = 0; \
r->headers_out.etag = NULL; \
}
#endif
void ps_strip_html_headers(ngx_http_request_t* r) {
// We're modifying content, so switch to 'Transfer-Encoding: chunked' and
// calculate on the fly.
ngx_http_clear_content_length(r);
ngx_table_elt_t* header;
NgxListIterator it(&(r->headers_out.headers.part));
while ((header = it.Next()) != NULL) {
// We also need to strip:
// Accept-Ranges
// - won't work because our html changes
// Vary: Accept-Encoding
// - our gzip filter will add this later
if (STR_CASE_EQ_LITERAL(header->key, "Accept-Ranges") ||
(STR_CASE_EQ_LITERAL(header->key, "Vary") &&
STR_CASE_EQ_LITERAL(header->value, "Accept-Encoding"))) {
// Response headers with hash of 0 are excluded from the response.
header->hash = 0;
}
}
}
// Returns true, if the the response headers indicate there are multiple
// content encodings.
bool ps_has_stacked_content_encoding(ngx_http_request_t* r) {
ngx_uint_t i;
ngx_list_part_t* part = &(r->headers_out.headers.part);
ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
int field_count = 0;
for (i = 0 ; /* void */; i++) {
if (i >= part->nelts) {
if (part->next == NULL) {
break;
}
part = part->next;
header = static_cast<ngx_table_elt_t*>(part->elts);
i = 0;
}
// Inspect Content-Encoding headers, checking all value fields
// If an origin returns gzip,foo, that is what we will get here.
if (STR_CASE_EQ_LITERAL(header[i].key, "Content-Encoding")) {
if (header[i].value.data != NULL && header[i].value.len > 0) {
char* p = reinterpret_cast<char*>(header[i].value.data);
ngx_uint_t j;
for (j = 0; j < header[i].value.len; j++) {
if (p[j] == ',' || j == header[i].value.len - 1) {
field_count++;
}
}
if (field_count > 1) {
return true;
}
}
}
}
return false;
}
ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
u_char* etag = reinterpret_cast<u_char*>(
const_cast<char*>(kInternalEtagName));
ngx_table_elt_t* header;
NgxListIterator it(&(r->headers_out.headers.part));
while ((header = it.Next()) != NULL) {
if (header->key.len == strlen(kInternalEtagName) &&
!ngx_strncasecmp(header->key.data, etag, header->key.len)) {
header->key.data = reinterpret_cast<u_char*>(const_cast<char*>("ETag"));
header->key.len = 4;
r->headers_out.etag = header;
break;
}
}
ps_request_ctx_t* ctx = ps_get_request_context(r);
#if (NGX_HTTP_GZIP)
if (ctx && ctx->psol_vary_accept_only) {
r->gzip_vary = 0;
}
#endif
if (ctx && ctx->recorder) {
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
int s_maxage_sec =
cfg_s->server_context->global_options()->EffectiveInPlaceSMaxAgeSec();
if (s_maxage_sec != -1) {
GoogleString existing_cache_control;
bool cache_control_present = ps_get_cache_control(
r, &existing_cache_control);
GoogleString updated_cache_control;
if (ResponseHeaders::ApplySMaxAge(s_maxage_sec,
existing_cache_control,
&updated_cache_control)) {
// We're modifing the cache control header; save a copy first.
// NULL indicates that the header was not present.
ctx->recorder->SaveCacheControl(
cache_control_present ? existing_cache_control.c_str() : NULL);
// Replace the cache-control with our new s-maxage-including one.
ps_set_cache_control(r, string_piece_to_pool_string(
r->pool, updated_cache_control));
}
}
}
return ngx_http_ef_next_header_filter(r);
}
namespace html_rewrite {
ngx_http_output_header_filter_pt ngx_http_next_header_filter;
ngx_http_output_body_filter_pt ngx_http_next_body_filter;
// After pagespeed has had a chance to run, copy the headers it produced to
// nginx so it can send them out to the browser.
ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
if (ps_disabled(cfg_s)) {
return ngx_http_next_header_filter(r);
}
if (r != r->main) {
// Don't handle subrequests.
return ngx_http_next_header_filter(r);
}
// Poll for cache flush on every request (polls are rate-limited).
cfg_s->server_context->FlushCacheIfNecessary();
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (ctx == NULL || ctx->html_rewrite == false) {
return ngx_http_next_header_filter(r);
}
if (r->err_status != 0) {
ctx->html_rewrite = false;
return ngx_http_next_header_filter(r);
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http pagespeed html rewrite header filter \"%V\"", &r->uri);
// We don't know what this request is, but we only want to send html through
// to pagespeed. Check the content type header and find out.
const ContentType* content_type =
MimeTypeToContentType(
str_to_string_piece(r->headers_out.content_type));
if (content_type == NULL || !content_type->IsHtmlLike()) {
// Unknown or otherwise non-html content type: skip it.
ctx->html_rewrite = false;
return ngx_http_next_header_filter(r);
}
ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
RequestRouting::kResource);
if (rc != NGX_OK) {
ctx->html_rewrite = false;
return ngx_http_next_header_filter(r);
}
if (r->headers_out.content_encoding &&
r->headers_out.content_encoding->value.len) {
// headers_out.content_encoding will be set to the exact last
// Content-Encoding response header value that nginx receives. To
// check if there were multiple (aka stacked) encodings in the
// response headers, we must iterate them all.
if (!ps_has_stacked_content_encoding(r)) {
StringPiece content_encoding =
str_to_string_piece(r->headers_out.content_encoding->value);
GzipInflater::InflateType inflate_type = GzipInflater::kGzip;
bool is_encoded = false;
if (StringCaseEqual(content_encoding, "deflate")) {
is_encoded = true;
inflate_type = GzipInflater::kDeflate;
} else if (StringCaseEqual(content_encoding, "gzip")) {
is_encoded = true;
}
if (is_encoded) {
r->headers_out.content_encoding->hash = 0;
r->headers_out.content_encoding = NULL;
ctx->inflater_ = new GzipInflater(inflate_type);
ctx->inflater_->Init();
}
}
}
ps_strip_html_headers(r);
// See https://github.com/pagespeed/ngx_pagespeed/issues/819
ctx->location_field_set = r->headers_out.location != NULL;
// TODO(jefftk): is this thread safe?
copy_response_headers_from_ngx(r, ctx->base_fetch->response_headers());
ps_set_buffered(r, true);
r->filter_need_in_memory = 1;
return NGX_AGAIN;
}
ngx_int_t ps_html_rewrite_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
if (ps_disabled(cfg_s)) {
return ngx_http_next_body_filter(r, in);
}
if (r != r->main) {
// Don't handle subrequests.
return ngx_http_next_body_filter(r, in);
}
// Don't need to check for a cache flush; already did in
// ps_html_rewrite_header_filter.
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (ctx == NULL || ctx->html_rewrite == false) {
// ctx is null iff we've decided to pass through this request unchanged.
return ngx_http_next_body_filter(r, in);
}
// We don't want to handle requests with errors, but we should be dealing with
// that in the header filter and not initializing ctx.
CHECK(r->err_status == 0); // NOLINT
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http pagespeed html rewrite body filter \"%V\"", &r->uri);
if (in != NULL) {
// Send all input data to the proxy fetch.
ps_send_to_pagespeed(r, ctx, cfg_s, in);
}
return ngx_http_next_body_filter(r, NULL);
}
void ps_html_rewrite_filter_init() {
ngx_http_next_header_filter = ngx_http_top_header_filter;
ngx_http_top_header_filter = ps_html_rewrite_header_filter;
ngx_http_next_body_filter = ngx_http_top_body_filter;
ngx_http_top_body_filter = ps_html_rewrite_body_filter;
}
} // namespace html_rewrite
using html_rewrite::ps_html_rewrite_filter_init;
namespace in_place {
ngx_http_output_header_filter_pt ngx_http_next_header_filter;
ngx_http_output_body_filter_pt ngx_http_next_body_filter;
// Header filter to support IPRO.
//
// The control flow here is tricky, probably excessively so.
ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (ctx == NULL) {
return ngx_http_next_header_filter(r);
}
if (ctx->recorder != NULL) {
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ps in place check header filter recording: %V", &r->uri);
CHECK(!ctx->in_place);
// We didn't find this resource in cache originally, so we're recording it
// as it passes us by. At this point the headers from things that run
// before us are set but not things that run after us, which means here is
// where we need to check whether there's a "Content-Encoding: gzip". If we
// waited to do this in ps_in_place_body_filter we wouldn't be able to tell
// the difference between response headers that have "C-E: gz" because we're
// proxying for an upstream that gzipped the content and response headers
// that have it because the gzip filter (which runs after us) is going to
// produce gzipped output.
//
// The recorder will do this checking, so pass it the headers.
ResponseHeaders response_headers;
copy_response_headers_from_ngx(r, &response_headers);
ctx->recorder->ConsiderResponseHeaders(
InPlaceResourceRecorder::kPreliminaryHeaders, &response_headers);
return ngx_http_next_header_filter(r);
}
if (!ctx->in_place) {
return ngx_http_next_header_filter(r);
}
// If our request was finalized during the IPRO lookup, we decline.
if (r->connection->error) {
return ps_decline_request(r);
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ps in place check header filter initial: %V", &r->uri);
int status_code = r->headers_out.status;
bool status_ok = (status_code != 0) && (status_code < 400);
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
NgxServerContext* server_context = cfg_s->server_context;
MessageHandler* message_handler = cfg_s->handler;
GoogleString url = ps_determine_url(r);
// The URL we use for cache key is a bit different since it may
// have PageSpeed query params removed.
GoogleString cache_url = ctx->url_string;
// continue process
if (status_ok) {
ctx->in_place = false;
server_context->rewrite_stats()->ipro_served()->Add(1);
message_handler->Message(
kInfo, "Serving rewritten resource in-place: %s",
url.c_str());
return ngx_http_next_header_filter(r);
}
if (status_code == CacheUrlAsyncFetcher::kNotInCacheStatus &&
!r->header_only) {
server_context->rewrite_stats()->ipro_not_in_cache()->Add(1);
server_context->message_handler()->Message(
kInfo,
"Could not rewrite resource in-place "
"because URL is not in cache: %s",
cache_url.c_str());
const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
ctx->driver->options());
RequestContextPtr request_context(
cfg_s->server_context->NewRequestContext(r));
request_context->set_options(options->ComputeHttpOptions());
RequestHeaders request_headers;
copy_request_headers_from_ngx(r, &request_headers);
// This URL was not found in cache (neither the input resource nor
// a ResourceNotCacheable entry) so we need to get it into cache
// (or at least a note that it cannot be cached stored there).
// We do that using an Apache output filter.
ctx->recorder = new InPlaceResourceRecorder(
request_context,
cache_url,
ctx->driver->CacheFragment(),
request_headers.GetProperties(),
options->ipro_max_response_bytes(),
options->ipro_max_concurrent_recordings(),
server_context->http_cache(),
server_context->statistics(),
message_handler);
// set in memory flag for in_place_body_filter
r->filter_need_in_memory = 1;
// We don't have the response headers at all yet because we haven't yet gone
// to the backend.
} else {
server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
message_handler->Message(
kInfo, "Could not rewrite resource in-place: %s", url.c_str());
}
return ps_decline_request(r);
}
// If we've decided that we should record this response for future optimization
// with IPRO, then log the bytes as they come through
ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (ctx == NULL || ctx->recorder == NULL) {
return ngx_http_next_body_filter(r, in);
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ps in place body filter: %V", &r->uri);
InPlaceResourceRecorder* recorder = ctx->recorder;
for (ngx_chain_t* cl = in; cl; cl = cl->next) {
if (ngx_buf_size(cl->buf)) {
CHECK(ngx_buf_in_memory(cl->buf));
StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
ngx_buf_size(cl->buf));
recorder->Write(contents, recorder->handler());
}
if (cl->buf->flush) {
recorder->Flush(recorder->handler());
}
if (cl->buf->last_buf || recorder->failed()) {
ResponseHeaders response_headers;
copy_response_headers_from_ngx(r, &response_headers);
ctx->recorder->DoneAndSetHeaders(
&response_headers,
cl->buf->last_buf /* response is complete if last_buf is set */);
ctx->recorder = NULL;
break;
}
}
return ngx_http_next_body_filter(r, in);
}
void ps_in_place_filter_init() {
ngx_http_next_header_filter = ngx_http_top_header_filter;
ngx_http_top_header_filter = ps_in_place_check_header_filter;
ngx_http_next_body_filter = ngx_http_top_body_filter;
ngx_http_top_body_filter = ps_in_place_body_filter;
}
} // namespace in_place
using in_place::ps_in_place_filter_init;
ngx_int_t send_out_headers_and_body(
ngx_http_request_t* r,
const ResponseHeaders& response_headers,
const GoogleString& output) {
ngx_int_t rc = copy_response_headers_to_ngx(
r, response_headers, kDontPreserveHeaders);
if (rc != NGX_OK) {
return NGX_ERROR;
}
rc = ngx_http_send_header(r);
if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
return rc;
}
// Send the body.
ngx_chain_t* out;
rc = string_piece_to_buffer_chain(
r->pool, output, &out, true /* send_last_buf */, false);
if (rc == NGX_ERROR) {
return NGX_ERROR;
}
CHECK(rc == NGX_OK);
return ngx_http_output_filter(r, out);
}
// Handle responses where we have the content we need in memory and can just
// send it right out.
ngx_int_t ps_simple_handler(ngx_http_request_t* r,
NgxServerContext* server_context,
RequestRouting::Response response_category) {
NgxRewriteDriverFactory* factory =
static_cast<NgxRewriteDriverFactory*>(
server_context->factory());
NgxMessageHandler* message_handler = factory->ngx_message_handler();
StringPiece request_uri_path = str_to_string_piece(r->uri);
GoogleString url_string = ps_determine_url(r);
GoogleUrl url(url_string);
QueryParams query_params;
if (url.IsWebValid()) {
query_params.ParseFromUrl(url);
}
GoogleString output;
StringWriter writer(&output);
HttpStatus::Code status = HttpStatus::kOK;
ContentType content_type = kContentTypeHtml;
StringPiece cache_control = HttpAttributes::kNoCache;
const char* error_message = NULL;
switch (response_category) {
case RequestRouting::kStaticContent: {
StringPiece file_contents;
if (!server_context->static_asset_manager()->GetAsset(
request_uri_path.substr(factory->static_asset_prefix().length()),
&file_contents, &content_type, &cache_control)) {
return NGX_DECLINED;
}
file_contents.CopyToString(&output);
break;
}
case RequestRouting::kMessages: {
GoogleString log;
StringWriter log_writer(&log);
if (!message_handler->Dump(&log_writer)) {
writer.Write("Writing to ngx_pagespeed_message failed. \n"
"Please check if it's enabled in pagespeed.conf.\n",
message_handler);
} else {
HtmlKeywords::WritePre(log, "", &writer, message_handler);
}
break;
}
default:
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
"ps_simple_handler: unknown RequestRouting.");
return NGX_ERROR;
}
if (error_message != NULL) {
status = HttpStatus::kNotFound;
content_type = kContentTypeHtml;
output = error_message;
}
ResponseHeaders response_headers;
response_headers.SetStatusAndReason(status);
response_headers.set_major_version(1);
response_headers.set_minor_version(1);
response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
// http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx
// Script and styleSheet elements will reject responses with
// incorrect MIME types if the server sends the response header
// "X-Content-Type-Options: nosniff". This is a security feature
// that helps prevent attacks based on MIME-type confusion.
response_headers.Add("X-Content-Type-Options", "nosniff");
int64 now_ms = factory->timer()->NowMs();
response_headers.SetDate(now_ms);
response_headers.SetLastModified(now_ms);
response_headers.Add(HttpAttributes::kCacheControl, cache_control);
char* cache_control_s = string_piece_to_pool_string(r->pool, cache_control);
if (cache_control_s != NULL) {
if (FindIgnoreCase(cache_control, "private") == StringPiece::npos) {
response_headers.Add(HttpAttributes::kEtag, "W/\"0\"");
}
}
return send_out_headers_and_body(r, response_headers, output);
}
void ps_beacon_handler_helper(ngx_http_request_t* r,
StringPiece beacon_data) {
ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ps_beacon_handler_helper: beacon[%d] %*s",
beacon_data.size(), beacon_data.size(),
beacon_data.data());
StringPiece user_agent;
if (r->headers_in.user_agent != NULL) {
user_agent = str_to_string_piece(r->headers_in.user_agent->value);
}
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
CHECK(cfg_s != NULL);
RequestContextPtr request_context(
cfg_s->server_context->NewRequestContext(r));
// TODO(sligocki): Do we want custom options here? It probably doesn't matter
// for beacons.
request_context->set_options(
cfg_s->server_context->global_options()->ComputeHttpOptions());
cfg_s->server_context->HandleBeacon(beacon_data,
user_agent,
request_context);
ps_set_cache_control(r, const_cast<char*>("max-age=0, no-cache"));
// TODO(jefftk): figure out how to insert Content-Length:0 as a response
// header so wget doesn't hang.
}
// Load the request body into out. ngx_http_read_client_request_body must
// already have been called. Return false on failure, true on success.
bool ps_request_body_to_string_piece(
ngx_http_request_t* r, StringPiece* out) {
if (r->request_body == NULL || r->request_body->bufs == NULL) {
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
"ps_request_body_to_string_piece: "
"empty request body.");
return false;
}
if (r->request_body->temp_file) {
u_char buf[POST_BUF_READ_SIZE];
int count = 0;
ssize_t ret;
GoogleString tmp;
// Note that we depend on nginx to impose sensible limits on post data.
while ((ret = ngx_read_file(&r->request_body->temp_file->file,
buf, POST_BUF_READ_SIZE, count)) > 0) {
tmp.append(reinterpret_cast<char*>(buf), ret);
count += ret;
}
if (ret < 0) {
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
"ps_request_body_to_string_piece: "
"error reading post body.");
return false;
}
// We have read the complete post body, copy it to a buffer
// from the request's pool.
u_char* data = reinterpret_cast<u_char*>(ngx_palloc(r->pool, count));
memcpy(data, tmp.c_str(), count);
*out = StringPiece(reinterpret_cast<char*>(data), count);
return true;
} else if (r->request_body->bufs->next == NULL) {
// There's just one buffer, so we can simply return a StringPiece pointing
// to this buffer.
ngx_buf_t* buffer = r->request_body->bufs->buf;
CHECK(!buffer->in_file);
int len = buffer->last - buffer->pos;
ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ngx_pagespeed beacon: single buffer of %d", len);
*out = StringPiece(reinterpret_cast<char*>(buffer->pos), len);
return true;
} else {
// There are multiple buffers, so we need to allocate memory for a string to
// hold the whole result. This should only happen when the POST is sent
// with "Transfer-Encoding: Chunked".
// First determine how much data there is.
int len = 0;
int buffers = 0;
ngx_chain_t* chain_link;
for (chain_link = r->request_body->bufs;
chain_link != NULL;
chain_link = chain_link->next) {
len += chain_link->buf->last - chain_link->buf->pos;
buffers++;
}
ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"ngx_pagespeed beacon: %d buffers totalling %d", len);
// Allocate a string to store the combined result.
u_char* s = static_cast<u_char*>(ngx_palloc(r->pool, len));
if (s == NULL) {
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
"ps_request_body_to_string_piece: "
"failed to allocate memory");
return false;
}
// Copy the data into the combined string.
u_char* current_position = s;
int i;
for (chain_link = r->request_body->bufs, i = 0;
chain_link != NULL;
chain_link = chain_link->next, i++) {
ngx_buf_t* buffer = chain_link->buf;
CHECK(!buffer->in_file);
current_position = ngx_copy(current_position, buffer->pos,
buffer->last - buffer->pos);
}
CHECK_EQ(current_position, s + len);
*out = StringPiece(reinterpret_cast<char*>(s), len);
return true;
}
}
// Parses out query params from the request.
void ps_query_params_handler(ngx_http_request_t* r, StringPiece* data) {
StringPiece unparsed_uri = str_to_string_piece(r->unparsed_uri);
stringpiece_ssize_type question_mark_index = unparsed_uri.find("?");
if (question_mark_index == StringPiece::npos) {
*data = "";
} else {
*data = unparsed_uri.substr(
question_mark_index+1, unparsed_uri.size() - (question_mark_index+1));
}
}
// Called after nginx reads the request body from the client. For another
// example processing request buffers, see ngx_http_form_input_module.c
void ps_beacon_body_handler(ngx_http_request_t* r) {
// Even if the beacon is a POST, the originating url should be in the query
// params, not the POST body.
StringPiece query_param_beacon_data;
ps_query_params_handler(r, &query_param_beacon_data);
StringPiece request_body;
bool ok = ps_request_body_to_string_piece(r, &request_body);
GoogleString beacon_data = StrCat(
query_param_beacon_data, "&", request_body);
if (ok) {
ps_beacon_handler_helper(r, beacon_data.c_str());
ngx_http_finalize_request(r, NGX_HTTP_NO_CONTENT);
} else {
ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
}
}
// We need to get the beacon data to ps_beacon_body_handler so it can pass it
// along to SystemServerContext::HandleBeacon, but it might have been POSTed.
// If it's posted we need to make an async call to get the data, otherwise just
// read it out of the query params.
ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
if (r->method == NGX_HTTP_POST) {
// Use post body. Handler functions are called before the request body has
// been read from the client, so we need to ask nginx to read it from the
// client and then call us back. Control flow continues in
// ps_beacon_body_handler unless there's an error reading the request body.
//
// See: http://forum.nginx.org/read.php?2,31312,31312
ngx_int_t rc = ngx_http_read_client_request_body(r, ps_beacon_body_handler);
if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {
return rc;
}
return NGX_DONE;
} else {
// Use query params.
StringPiece query_param_beacon_data;
ps_query_params_handler(r, &query_param_beacon_data);
ps_beacon_handler_helper(r, query_param_beacon_data);
return NGX_HTTP_NO_CONTENT;
}
}
// Some things pagespeed filters on the way past (html) and other things it
// actually handles, like requests for resources
// (example.css.pagespeed.ce.LyfcM6Wulf.css) and static content
// (/ngx_pagespeed_static/js_defer.q1EBmcgYOC.js). This is called once we know
// we need to handle a resource, and it figures out which particular handler can
// supply it to the user.
ngx_int_t ps_content_handler(ngx_http_request_t* r) {
ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
if (ps_disabled(cfg_s)) {
// Pagespeed is not on for this server block.
return NGX_DECLINED;
}
// Poll for cache flush on every request (polls are rate-limited).
cfg_s->server_context->FlushCacheIfNecessary();
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http pagespeed handler \"%V\"", &r->uri);
RequestRouting::Response response_category =
ps_route_request(r);
switch (response_category) {
case RequestRouting::kError:
return NGX_ERROR;
case RequestRouting::kPagespeedDisabled:
case RequestRouting::kInvalidUrl:
case RequestRouting::kPagespeedSubrequest:
case RequestRouting::kErrorResponse:
return NGX_DECLINED;
case RequestRouting::kBeacon:
return ps_beacon_handler(r);
case RequestRouting::kStaticContent:
case RequestRouting::kMessages:
return ps_simple_handler(r, cfg_s->server_context, response_category);
case RequestRouting::kStatistics:
case RequestRouting::kGlobalStatistics:
case RequestRouting::kConsole:
case RequestRouting::kAdmin:
case RequestRouting::kGlobalAdmin:
case RequestRouting::kCachePurge:
case RequestRouting::kResource:
return ps_resource_handler(
r, false /* html rewrite */, response_category);
}
CHECK(0);
return NGX_ERROR;
}
ngx_int_t ps_phase_handler(ngx_http_request_t* r,
ngx_http_phase_handler_t* ph) {
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"pagespeed phase: %ui", r->phase_handler);
r->write_event_handler = ngx_http_request_empty_handler;
ngx_int_t rc = ps_content_handler(r);
// Warning: this requires ps_content_handler to always return NGX_DECLINED
// directly if it's not going to handle the request. It is not ok for
// ps_content_handler to asynchronously determine whether to handle the
// request, returning NGX_DONE here.
if (rc == NGX_DECLINED) {
r->write_event_handler = ngx_http_core_run_phases;
r->phase_handler++;
return NGX_AGAIN;
}
ngx_http_finalize_request(r, rc);
return NGX_OK;
}
namespace fix_headers {
ngx_http_output_header_filter_pt ngx_http_next_header_filter;
// Clear a few headers from html responses.
ngx_int_t ps_html_rewrite_fix_headers_filter(ngx_http_request_t* r) {
ps_request_ctx_t* ctx = ps_get_request_context(r);
if (r != r->main || ctx == NULL || !ctx->html_rewrite ||
ctx->preserve_caching_headers == kPreserveAllCachingHeaders) {
return ngx_http_next_header_filter(r);
}
if (ctx->preserve_caching_headers == kDontPreserveHeaders) {
// Don't cache html. See mod_instaweb:instaweb_fix_headers_filter.
NgxCachingHeaders caching_headers(r);
ps_set_cache_control(r, string_piece_to_pool_string(
r->pool, caching_headers.GenerateDisabledCacheControl()));
}
// Pagespeed html doesn't need etags: it should never be cached.
ngx_http_clear_etag(r);
// An html page may change without the underlying file changing, because of
// how resources are included. Pagespeed adds cache control headers for
// resources instead of using the last modified header.
ngx_http_clear_last_modified(r);
// Clear expires
if (r->headers_out.expires) {
r->headers_out.expires->hash = 0;
r->headers_out.expires = NULL;
}
return ngx_http_next_header_filter(r);
}
void ps_html_rewrite_fix_headers_filter_init() {
ngx_http_next_header_filter = ngx_http_top_header_filter;
ngx_http_top_header_filter = ps_html_rewrite_fix_headers_filter;
}
} // namespace fix_headers
using fix_headers::ps_html_rewrite_fix_headers_filter_init;
// preaccess_handler should be at generic phase before try_files
ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
ngx_http_core_main_conf_t* cmcf;
ngx_http_phase_handler_t* ph;
ngx_uint_t i;
cmcf = static_cast<ngx_http_core_main_conf_t*>(
ngx_http_get_module_main_conf(r, ngx_http_core_module));
ph = cmcf->phase_engine.handlers;
i = r->phase_handler;
// move handlers before try_files && content phase
// As of nginx 1.13.4 we will be right before the try_files module
#if (nginx_version < NGINX_1_13_4)
while (ph[i + 1].checker != ngx_http_core_try_files_phase &&
ph[i + 1].checker != ngx_http_core_content_phase) {
ph[i] = ph[i + 1];
ph[i].next--;
i++;
}
#endif
// insert ps phase handler
ph[i].checker = ps_phase_handler;
ph[i].handler = NULL;
ph[i].next = i + 1;
// next preaccess handler
r->phase_handler--;
return NGX_DECLINED;
}
ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
ngx_http_ef_next_header_filter = ngx_http_top_header_filter;
ngx_http_top_header_filter = ps_etag_header_filter;
}
return NGX_OK;
}
// Called before configuration.
ngx_int_t ps_pre_init(ngx_conf_t* cf) {
// Setup an intervention setter for gzip configuration and check
// gzip configuration command signatures.
g_gzip_setter.Init(cf);
return NGX_OK;
}
ngx_int_t ps_init(ngx_conf_t* cf) {
// Only put register pagespeed code to run if there was a "pagespeed"
// configuration option set in the config file. With "pagespeed off" we
// consider every request and choose not to do anything, while with no
// "pagespeed" directives we won't have any effect after nginx is done loading
// its configuration.
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
// The driver factory is on the main config and is non-NULL iff there is a
// pagespeed configuration option in the main config or a server block. Note
// that if any server block has pagespeed 'on' then our header filter, body
// filter, and content handler will run in every server block. This is ok,
// because they will notice that the server context is NULL and do nothing.
if (cfg_m->driver_factory != NULL) {
// The filter init order is important.
ps_in_place_filter_init();
ps_html_rewrite_fix_headers_filter_init();
ps_base_fetch::ps_base_fetch_filter_init();
ps_html_rewrite_filter_init();
ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module));
int phase = NGX_HTTP_PREACCESS_PHASE;
// As of nginx 1.13.4, try_files has changed.
// https://github.com/nginx/nginx/commit/129b06dc5dfab7b4513a4f274b3778cd9b8a6a22
#if (nginx_version >= NGINX_1_13_4)
phase = NGX_HTTP_PRECONTENT_PHASE;
#endif
ngx_http_handler_pt* h = static_cast<ngx_http_handler_pt*>(
ngx_array_push(&cmcf->phases[phase].handlers));
if (h == NULL) {
return NGX_ERROR;
}
*h = ps_preaccess_handler;
}
return NGX_OK;
}
ngx_http_module_t ps_etag_filter_module = {
NULL, // preconfiguration
ps_etag_filter_init, // postconfiguration
NULL,
NULL, // initialize main configuration
NULL,
NULL,
NULL,
NULL
};
ngx_http_module_t ps_module = {
ps_pre_init, // preconfiguration
ps_init, // postconfiguration
ps_create_main_conf,
NULL, // initialize main configuration
ps_create_srv_conf,
ps_merge_srv_conf,
ps_create_loc_conf,
ps_merge_loc_conf
};
// called after configuration is complete, but before nginx starts forking
ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
// See https://github.com/pagespeed/ngx_pagespeed/issues/1220
if (cfg_m == NULL) {
return NGX_OK;
}
ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
cmcf->servers.elts);
ngx_uint_t s;
std::vector<SystemServerContext*> server_contexts;
// Iterate over all configured server{} blocks to collect the server contexts.
for (s = 0; s < cmcf->servers.nelts; s++) {
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
cscfp[s]->ctx->srv_conf[ngx_pagespeed.ctx_index]);
if (cfg_s->server_context != NULL) {
server_contexts.push_back(cfg_s->server_context);
}
}
GoogleString error_message;
int error_index = -1;
Statistics* global_statistics = NULL;
cfg_m->driver_factory->PostConfig(
server_contexts, &error_message, &error_index, &global_statistics);
if (error_index != -1) {
server_contexts[error_index]->message_handler()->Message(
kError, "ngx_pagespeed is enabled. %s", error_message.c_str());
return NGX_ERROR;
}
if (!server_contexts.empty()) {
// TODO(oschaaf): this ignores sigpipe messages from memcached.
// however, it would be better to not have those signals generated
// in the first place, as suppressing them this way may interfere
// with other modules that actually are interested in these signals
ps_ignore_sigpipe();
// If no shared-mem statistics are enabled, then init using the default
// NullStatistics.
if (global_statistics == NULL) {
NgxRewriteDriverFactory::InitStats(cfg_m->driver_factory->statistics());
}
ngx_http_core_loc_conf_t* clcf = static_cast<ngx_http_core_loc_conf_t*>(
ngx_http_conf_get_module_loc_conf((*cscfp), ngx_http_core_module));
cfg_m->driver_factory->set_resolver(clcf->resolver);
cfg_m->driver_factory->set_resolver_timeout(clcf->resolver_timeout);
if (!cfg_m->driver_factory->CheckResolver()) {
cfg_m->handler->Message(
kError,
"UseNativeFetcher is on, please configure a resolver.");
return NGX_ERROR;
}
cfg_m->driver_factory->LoggingInit(cycle->log, true);
cfg_m->driver_factory->RootInit();
} else {
delete cfg_m->driver_factory;
cfg_m->driver_factory = NULL;
active_driver_factory = NULL;
}
return NGX_OK;
}
void ps_exit_child_process(ngx_cycle_t* cycle) {
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
NgxBaseFetch::Terminate();
if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
cfg_m->driver_factory->ShutDown();
}
}
// Called when nginx forks worker processes. No threads should be started
// before this.
ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
if (cfg_m == NULL || cfg_m->driver_factory == NULL) {
return NGX_OK;
}
if (!NgxBaseFetch::Initialize(cycle)) {
return NGX_ERROR;
}
// ChildInit() will initialise all ServerContexts, which we need to
// create ProxyFetchFactories below
cfg_m->driver_factory->LoggingInit(cycle->log, true);
cfg_m->driver_factory->ChildInit();
ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
cmcf->servers.elts);
ngx_uint_t s;
// Iterate over all configured server{} blocks, and find our context in it,
// so we can create and set a ProxyFetchFactory for it.
for (s = 0; s < cmcf->servers.nelts; s++) {
ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
cscfp[s]->ctx->srv_conf[ngx_pagespeed.ctx_index]);
// Some server{} blocks may not have a ServerContext in that case we must
// not instantiate a ProxyFetchFactory.
if (cfg_s->server_context != NULL) {
cfg_s->proxy_fetch_factory = new ProxyFetchFactory(cfg_s->server_context);
ngx_http_core_loc_conf_t* clcf = static_cast<ngx_http_core_loc_conf_t*>(
cscfp[s]->ctx->loc_conf[ngx_http_core_module.ctx_index]);
cfg_m->driver_factory->SetServerContextMessageHandler(
cfg_s->server_context, clcf->error_log);
}
}
cfg_m->driver_factory->StartThreads();
return NGX_OK;
}
} // namespace
} // namespace net_instaweb
ngx_module_t ngx_pagespeed_etag_filter = {
NGX_MODULE_V1,
&net_instaweb::ps_etag_filter_module,
NULL,
NGX_HTTP_MODULE,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NGX_MODULE_V1_PADDING
};
ngx_module_t ngx_pagespeed = {
NGX_MODULE_V1,
&net_instaweb::ps_module,
net_instaweb::ps_commands,
NGX_HTTP_MODULE,
NULL,
net_instaweb::ps_init_module,
net_instaweb::ps_init_child_process,
NULL,
NULL,
net_instaweb::ps_exit_child_process,
NULL,
NGX_MODULE_V1_PADDING
};
lib/ngx_pagespeed/src/ngx_pagespeed.h
-138
View File
@@ -1,138 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#ifndef NGX_PAGESPEED_H_
#define NGX_PAGESPEED_H_
// We might be compiled with syslog.h, which #defines LOG_INFO and LOG_WARNING
// as ints. But logging.h assumes they're usable as names, within their
// namespace, so we need to #undef them before including logging.h
#ifdef LOG_INFO
#undef LOG_INFO
#endif
#ifdef LOG_WARNING
#undef LOG_WARNING
#endif
extern "C" {
#include <ngx_http.h>
}
#include "base/logging.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/http/response_headers.h"
namespace net_instaweb {
class GzipInflater;
class NgxBaseFetch;
class ProxyFetch;
class RewriteDriver;
class RequestHeaders;
class ResponseHeaders;
class InPlaceResourceRecorder;
// Allocate chain links and buffers from the supplied pool, and copy over the
// data from the string piece. If the string piece is empty, return
// NGX_DECLINED immediately unless send_last_buf.
ngx_int_t string_piece_to_buffer_chain(
ngx_pool_t* pool, StringPiece sp,
ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
StringPiece str_to_string_piece(ngx_str_t s);
// s1: ngx_str_t, s2: string literal
// true if they're equal, false otherwise
#define STR_EQ_LITERAL(s1, s2) \
((s1).len == (sizeof(s2)-1) && \
ngx_strncmp((s1).data, (s2), (sizeof(s2)-1)) == 0)
// s1: ngx_str_t, s2: string literal
// true if they're equal ignoring case, false otherwise
#define STR_CASE_EQ_LITERAL(s1, s2) \
((s1).len == (sizeof(s2)-1) && \
ngx_strncasecmp((s1).data, ( \
reinterpret_cast<u_char*>( \
const_cast<char*>(s2))), \
(sizeof(s2)-1)) == 0)
// Allocate memory out of the pool for the string piece, and copy the contents
// over. Returns NULL if we can't get memory.
char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp);
enum PreserveCachingHeaders {
kPreserveAllCachingHeaders, // Cache-Control, ETag, Last-Modified, etc
kPreserveOnlyCacheControl, // Only Cache-Control.
kDontPreserveHeaders,
};
typedef struct {
NgxBaseFetch* base_fetch;
ngx_http_request_t* r;
bool html_rewrite;
bool in_place;
PreserveCachingHeaders preserve_caching_headers;
// for html rewrite
ProxyFetch* proxy_fetch;
GzipInflater* inflater_;
// for in place resource
RewriteDriver* driver;
InPlaceResourceRecorder* recorder;
ResponseHeaders* ipro_response_headers;
// We need to remember the URL here as well since we may modify what NGX
// gets by stripping our special query params and honoring X-Forwarded-Proto.
GoogleString url_string;
// We need to remember if the upstream had headers_out->location set, because
// we should mirror that when we write it back. nginx may absolutify
// Location: headers that start with '/' without regarding X-Forwarded-Proto.
bool location_field_set;
bool psol_vary_accept_only;
bool follow_flushes;
} ps_request_ctx_t;
ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);
void copy_request_headers_from_ngx(const ngx_http_request_t* r,
RequestHeaders* headers);
void copy_response_headers_from_ngx(const ngx_http_request_t* r,
ResponseHeaders* headers);
ngx_int_t copy_response_headers_to_ngx(
ngx_http_request_t* r,
const ResponseHeaders& pagespeed_headers,
PreserveCachingHeaders preserve_caching_headers);
StringPiece ps_determine_host(ngx_http_request_t* r);
namespace ps_base_fetch {
ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
} // namespace ps_base_fetch
} // namespace net_instaweb
#endif // NGX_PAGESPEED_H_
lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.cc
-288
View File
@@ -1,288 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#include "ngx_rewrite_driver_factory.h"
#include <cstdio>
#include "log_message_handler.h"
#include "ngx_message_handler.h"
#include "ngx_rewrite_options.h"
#include "ngx_server_context.h"
#include "ngx_url_async_fetcher.h"
#include "net/instaweb/http/public/rate_controller.h"
#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
#include "net/instaweb/http/public/wget_url_fetcher.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/util/public/property_cache.h"
#include "pagespeed/kernel/base/google_message_handler.h"
#include "pagespeed/kernel/base/null_shared_mem.h"
#include "pagespeed/kernel/base/posix_timer.h"
#include "pagespeed/kernel/base/stdio_file_system.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/thread_system.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
#include "pagespeed/kernel/thread/pthread_shared_mem.h"
#include "pagespeed/kernel/thread/scheduler_thread.h"
#include "pagespeed/kernel/thread/slow_worker.h"
#include "pagespeed/system/in_place_resource_recorder.h"
#include "pagespeed/system/serf_url_async_fetcher.h"
#include "pagespeed/system/system_caches.h"
#include "pagespeed/system/system_rewrite_options.h"
namespace net_instaweb {
class FileSystem;
class Hasher;
class MessageHandler;
class Statistics;
class Timer;
class UrlAsyncFetcher;
class UrlFetcher;
class Writer;
class SharedCircularBuffer;
NgxRewriteDriverFactory::NgxRewriteDriverFactory(
const ProcessContext& process_context,
SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
: SystemRewriteDriverFactory(process_context, system_thread_system,
NULL /* default shared memory runtime */, hostname, port),
threads_started_(false),
ngx_message_handler_(
new NgxMessageHandler(timer(), thread_system()->NewMutex())),
ngx_html_parse_message_handler_(
new NgxMessageHandler(timer(), thread_system()->NewMutex())),
log_(NULL),
resolver_timeout_(NGX_CONF_UNSET_MSEC),
use_native_fetcher_(false),
// 100 Aligns to nginx's server-side default.
native_fetcher_max_keepalive_requests_(100),
ngx_shared_circular_buffer_(NULL),
hostname_(hostname.as_string()),
port_(port),
process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
process_script_variables_set_(false),
shut_down_(false) {
InitializeDefaultOptions();
default_options()->set_beacon_url("/ngx_pagespeed_beacon");
SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
default_options());
system_options->set_file_cache_clean_inode_limit(500000);
system_options->set_avoid_renaming_introspective_javascript(true);
set_message_handler(ngx_message_handler_);
set_html_parse_message_handler(ngx_html_parse_message_handler_);
}
NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
ShutDown();
ngx_shared_circular_buffer_ = NULL;
STLDeleteElements(&uninitialized_server_contexts_);
}
Hasher* NgxRewriteDriverFactory::NewHasher() {
return new MD5Hasher;
}
UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
SystemRewriteOptions* config) {
if (use_native_fetcher_) {
NgxUrlAsyncFetcher* fetcher = new NgxUrlAsyncFetcher(
config->fetcher_proxy().c_str(),
log_,
resolver_timeout_,
config->blocking_fetch_timeout_ms(),
resolver_,
native_fetcher_max_keepalive_requests_,
thread_system(),
message_handler());
ngx_url_async_fetchers_.push_back(fetcher);
return fetcher;
} else {
return SystemRewriteDriverFactory::AllocateFetcher(config);
}
}
MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
return ngx_html_parse_message_handler_;
}
MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
return ngx_message_handler_;
}
FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
return new StdioFileSystem();
}
Timer* NgxRewriteDriverFactory::DefaultTimer() {
return new PosixTimer;
}
NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
CHECK(false);
return NULL;
}
RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
// TODO(jefftk): figure out why using SetDefaultRewriteLevel like
// mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
// -- if you use that instead then ngx_pagespeed doesn't actually end up
// defaulting CoreFilters.
// See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
options->SetRewriteLevel(RewriteOptions::kCoreFilters);
return options;
}
RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
return new NgxRewriteOptions(thread_system());
}
bool NgxRewriteDriverFactory::CheckResolver() {
if (use_native_fetcher_ && resolver_ == NULL) {
return false;
}
return true;
}
NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
StringPiece hostname, int port) {
NgxServerContext* server_context = new NgxServerContext(this, hostname, port);
uninitialized_server_contexts_.insert(server_context);
return server_context;
}
ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
ServerContext* sc = new NgxServerContext(this, hostname_, port_);
InitStubDecodingServerContext(sc);
return sc;
}
ServerContext* NgxRewriteDriverFactory::NewServerContext() {
LOG(DFATAL) << "MakeNgxServerContext should be used instead";
return NULL;
}
void NgxRewriteDriverFactory::ShutDown() {
if (!shut_down_) {
shut_down_ = true;
SystemRewriteDriverFactory::ShutDown();
}
}
void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
ngx_message_handler_->set_buffer(NULL);
ngx_html_parse_message_handler_->set_buffer(NULL);
for (NgxMessageHandlerSet::iterator p =
server_context_message_handlers_.begin();
p != server_context_message_handlers_.end(); ++p) {
(*p)->set_buffer(NULL);
}
server_context_message_handlers_.clear();
}
void NgxRewriteDriverFactory::StartThreads() {
if (threads_started_) {
return;
}
// TODO(jefftk): use a native nginx timer instead of running our own thread.
// See issue #111.
SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
bool ok = thread->Start();
CHECK(ok) << "Unable to start scheduler thread";
defer_cleanup(thread->MakeDeleter());
threads_started_ = true;
}
void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
// Propagate process-scope options from the copy we had during nginx option
// parsing to our own.
if (main_options != NULL) {
default_options()->MergeOnlyProcessScopeOptions(*main_options);
}
}
void NgxRewriteDriverFactory::LoggingInit(
ngx_log_t* log, bool may_install_crash_handler) {
log_ = log;
net_instaweb::log_message_handler::Install(log);
if (may_install_crash_handler && install_crash_handler()) {
NgxMessageHandler::InstallCrashHandler(log);
}
ngx_message_handler_->set_log(log);
ngx_html_parse_message_handler_->set_log(log);
}
void NgxRewriteDriverFactory::SetCircularBuffer(
SharedCircularBuffer* buffer) {
ngx_shared_circular_buffer_ = buffer;
ngx_message_handler_->set_buffer(buffer);
ngx_html_parse_message_handler_->set_buffer(buffer);
}
void NgxRewriteDriverFactory::SetServerContextMessageHandler(
ServerContext* server_context, ngx_log_t* log) {
NgxMessageHandler* handler = new NgxMessageHandler(
timer(), thread_system()->NewMutex());
handler->set_log(log);
// The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
// been raised from its default of 0.
handler->set_buffer(ngx_shared_circular_buffer_);
server_context_message_handlers_.insert(handler);
defer_cleanup(new Deleter<NgxMessageHandler>(handler));
server_context->set_message_handler(handler);
}
void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
// Init standard PSOL stats.
SystemRewriteDriverFactory::InitStats(statistics);
RewriteDriverFactory::InitStats(statistics);
RateController::InitStats(statistics);
// Init Ngx-specific stats.
NgxServerContext::InitStats(statistics);
InPlaceResourceRecorder::InitStats(statistics);
}
void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
ngx_pid = ngx_getpid(); // Needed for logging to have the right PIDs.
SystemRewriteDriverFactory::PrepareForkedProcess(name);
}
void NgxRewriteDriverFactory::NameProcess(const char* name) {
SystemRewriteDriverFactory::NameProcess(name);
// Superclass set status with prctl. Nginx has a helper function for setting
// argv[0] as well, so let's use that. We'll show up as:
//
// nginx: pagespeed $name
char name_for_setproctitle[32];
snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
"pagespeed %s", name);
ngx_setproctitle(name_for_setproctitle);
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.h
-176
View File
@@ -1,176 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#ifndef NGX_REWRITE_DRIVER_FACTORY_H_
#define NGX_REWRITE_DRIVER_FACTORY_H_
extern "C" {
#include <ngx_auto_config.h>
#if (NGX_THREADS)
#include <ngx_thread.h>
#endif
#include <ngx_core.h>
#include <ngx_http.h>
#include <ngx_config.h>
#include <ngx_log.h>
}
#include <set>
#include "pagespeed/kernel/base/md5_hasher.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/system/system_rewrite_driver_factory.h"
namespace net_instaweb {
class NgxMessageHandler;
class NgxRewriteOptions;
class NgxServerContext;
class NgxUrlAsyncFetcher;
class SharedCircularBuffer;
class SharedMemRefererStatistics;
class SlowWorker;
class Statistics;
class SystemThreadSystem;
enum ProcessScriptVariablesMode {
kOff,
kLegacyRestricted,
kAll
};
class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
public:
// We take ownership of the thread system.
explicit NgxRewriteDriverFactory(
const ProcessContext& process_context,
SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
virtual ~NgxRewriteDriverFactory();
virtual Hasher* NewHasher();
virtual UrlAsyncFetcher* AllocateFetcher(SystemRewriteOptions* config);
virtual MessageHandler* DefaultHtmlParseMessageHandler();
virtual MessageHandler* DefaultMessageHandler();
virtual FileSystem* DefaultFileSystem();
virtual Timer* DefaultTimer();
virtual NamedLockManager* DefaultLockManager();
// Create a new RewriteOptions. In this implementation it will be an
// NgxRewriteOptions, and it will have CoreFilters explicitly set.
virtual RewriteOptions* NewRewriteOptions();
virtual RewriteOptions* NewRewriteOptionsForQuery();
virtual ServerContext* NewDecodingServerContext();
// Check resolver configured or not.
bool CheckResolver();
// Initializes all the statistics objects created transitively by
// NgxRewriteDriverFactory, including nginx-specific and
// platform-independent statistics.
static void InitStats(Statistics* statistics);
NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
virtual ServerContext* NewServerContext();
virtual void ShutDown();
// Starts pagespeed threads if they've not been started already. Must be
// called after the caller has finished any forking it intends to do.
void StartThreads();
void SetServerContextMessageHandler(ServerContext* server_context,
ngx_log_t* log);
NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
virtual void NonStaticInitStats(Statistics* statistics) {
InitStats(statistics);
}
void SetMainConf(NgxRewriteOptions* main_conf);
void set_resolver(ngx_resolver_t* resolver) {
resolver_ = resolver;
}
void set_resolver_timeout(ngx_msec_t resolver_timeout) {
resolver_timeout_ = resolver_timeout == NGX_CONF_UNSET_MSEC ?
1000 : resolver_timeout;
}
bool use_native_fetcher() {
return use_native_fetcher_;
}
void set_use_native_fetcher(bool x) {
use_native_fetcher_ = x;
}
int native_fetcher_max_keepalive_requests() {
return native_fetcher_max_keepalive_requests_;
}
void set_native_fetcher_max_keepalive_requests(int x) {
native_fetcher_max_keepalive_requests_ = x;
}
ProcessScriptVariablesMode process_script_variables() {
return process_script_variables_mode_;
}
void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
virtual void ShutDownMessageHandlers();
virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
if (!process_script_variables_set_) {
process_script_variables_mode_ = mode;
process_script_variables_set_ = true;
return true;
}
return false;
}
virtual void PrepareForkedProcess(const char* name);
virtual void NameProcess(const char* name);
private:
Timer* timer_;
bool threads_started_;
NgxMessageHandler* ngx_message_handler_;
NgxMessageHandler* ngx_html_parse_message_handler_;
std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
ngx_log_t* log_;
ngx_msec_t resolver_timeout_;
ngx_resolver_t* resolver_;
bool use_native_fetcher_;
int native_fetcher_max_keepalive_requests_;
typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
NgxMessageHandlerSet server_context_message_handlers_;
// Owned by the superclass.
// TODO(jefftk): merge the nginx and apache ways of doing this.
SharedCircularBuffer* ngx_shared_circular_buffer_;
GoogleString hostname_;
int port_;
ProcessScriptVariablesMode process_script_variables_mode_;
bool process_script_variables_set_;
bool shut_down_;
DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
};
} // namespace net_instaweb
#endif // NGX_REWRITE_DRIVER_FACTORY_H_
lib/ngx_pagespeed/src/ngx_rewrite_options.cc
-555
View File
@@ -1,555 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#include "ngx_rewrite_options.h"
extern "C" {
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
}
#include "ngx_pagespeed.h"
#include "ngx_rewrite_driver_factory.h"
#include "net/instaweb/public/version.h"
#include "net/instaweb/rewriter/public/file_load_policy.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/system/system_caches.h"
namespace net_instaweb {
namespace {
const char kStatisticsPath[] = "StatisticsPath";
const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
const char kConsolePath[] = "ConsolePath";
const char kMessagesPath[] = "MessagesPath";
const char kAdminPath[] = "AdminPath";
const char kGlobalAdminPath[] = "GlobalAdminPath";
// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
// indicates that they can not be set at the directory/location level. They set
// options in the RewriteDriverFactory, so they're entirely global and do not
// appear in RewriteOptions. They are not alphabetized on purpose, but rather
// left in the same order as in mod_instaweb.cc in case we end up needing to
// compare.
// TODO(oschaaf): this duplication is a short term solution.
const char* const server_only_options[] = {
"FetcherTimeoutMs",
"FetchProxy",
"ForceCaching",
"GeneratedFilePrefix",
"ImgMaxRewritesAtOnce",
"InheritVHostConfig",
"InstallCrashHandler",
"MessageBufferSize",
"NumRewriteThreads",
"NumExpensiveRewriteThreads",
"StaticAssetPrefix",
"TrackOriginalContentLength",
"UsePerVHostStatistics", // TODO(anupama): What to do about "No longer used"
"BlockingRewriteRefererUrls",
"CreateSharedMemoryMetadataCache",
"LoadFromFile",
"LoadFromFileMatch",
"LoadFromFileRule",
"LoadFromFileRuleMatch",
"UseNativeFetcher",
"NativeFetcherMaxKeepaliveRequests"
};
// Options that can only be used in the main (http) option scope.
const char* const main_only_options[] = {
"UseNativeFetcher",
"NativeFetcherMaxKeepaliveRequests"
};
} // namespace
RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
NgxRewriteOptions::NgxRewriteOptions(const StringPiece& description,
ThreadSystem* thread_system)
: SystemRewriteOptions(description, thread_system) {
Init();
}
NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
: SystemRewriteOptions(thread_system) {
Init();
}
void NgxRewriteOptions::Init() {
DCHECK(ngx_properties_ != NULL)
<< "Call NgxRewriteOptions::Initialize() before construction";
clear_inherited_scripts_ = false;
InitializeOptions(ngx_properties_);
}
void NgxRewriteOptions::AddProperties() {
// Nginx-specific options.
add_ngx_option(
"", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
false);
add_ngx_option(
"", &NgxRewriteOptions::global_statistics_path_, "ngsp",
kGlobalStatisticsPath, kProcessScopeStrict,
"Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
false);
add_ngx_option(
"", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
"Set the console path. Ex: /pagespeed_console", false);
add_ngx_option(
"", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
kServerScope, "Set the messages path. Ex: /ngx_pagespeed_message",
false);
add_ngx_option(
"", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
kServerScope, "Set the admin path. Ex: /pagespeed_admin", false);
add_ngx_option(
"", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
kProcessScopeStrict,
"Set the global admin path. Ex: /pagespeed_global_admin",
false);
MergeSubclassProperties(ngx_properties_);
// Default properties are global but to set them the current API requires
// a RewriteOptions instance and we're in a static method.
NgxRewriteOptions dummy_config(NULL);
dummy_config.set_default_x_header_value(kModPagespeedVersion);
}
void NgxRewriteOptions::Initialize() {
if (Properties::Initialize(&ngx_properties_)) {
SystemRewriteOptions::Initialize();
AddProperties();
}
}
void NgxRewriteOptions::Terminate() {
if (Properties::Terminate(&ngx_properties_)) {
SystemRewriteOptions::Terminate();
}
}
bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
StringPiece compare_directive) {
return StringCaseEqual(config_directive, compare_directive);
}
RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
StringPiece option_name) {
ngx_uint_t i;
ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
for (i = 0; i < size; i++) {
if (StringCaseEqual(main_only_options[i], option_name)) {
return kProcessScopeStrict;
}
}
size = sizeof(server_only_options) / sizeof(char*);
for (i = 0; i < size; i++) {
if (StringCaseEqual(server_only_options[i], option_name)) {
return kServerScope;
}
}
// This could be made more efficient if RewriteOptions provided a map allowing
// access of options by their name. It's not too much of a worry at present
// since this is just during initialization.
for (OptionBaseVector::const_iterator it = all_options().begin();
it != all_options().end(); ++it) {
RewriteOptions::OptionBase* option = *it;
if (option->option_name() == option_name) {
// We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
// if an option is out of place.
return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
: option->scope();
}
}
return kDirectoryScope;
}
RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
StringPiece directive, GoogleString* msg, MessageHandler* handler) {
EnabledEnum enabled;
if (!ParseFromString(directive, &enabled)) {
return RewriteOptions::kOptionNameUnknown;
}
if (enabled == RewriteOptions::kEnabledOff) {
// In ngx_pagespeed, for historical reasons, we treat "off" as "unplugged".
// Also, "off" is deprecated and people should be using "standby" or
// "unplugged" now depending on which sense they want. See comment on
// RewriteOptions::EnabledEnum.
enabled = RewriteOptions::kEnabledUnplugged;
}
set_enabled(enabled);
return RewriteOptions::kOptionOk;
}
RewriteOptions::OptionSettingResult
NgxRewriteOptions::ParseAndSetOptionFromName1(
StringPiece name, StringPiece arg,
GoogleString* msg, MessageHandler* handler) {
// FileCachePath needs error checking.
if (StringCaseEqual(name, kFileCachePath)) {
if (!StringCaseStartsWith(arg, "/")) {
*msg = "must start with a slash";
return RewriteOptions::kOptionValueInvalid;
}
}
return SystemRewriteOptions::ParseAndSetOptionFromName1(
name, arg, msg, handler);
}
template <class DriverFactoryT>
RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
StringPiece option_value,
DriverFactoryT* driver_factory,
void (DriverFactoryT::*set_option_method)(bool)) {
bool parsed_value;
if (StringCaseEqual(option_value, "on") ||
StringCaseEqual(option_value, "true")) {
parsed_value = true;
} else if (StringCaseEqual(option_value, "off") ||
StringCaseEqual(option_value, "false")) {
parsed_value = false;
} else {
return RewriteOptions::kOptionValueInvalid;
}
(driver_factory->*set_option_method)(parsed_value);
return RewriteOptions::kOptionOk;
}
namespace {
const char* ps_error_string_for_option(
ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
GoogleString msg =
StrCat("\"", directive, "\" ", warning);
char* s = string_piece_to_pool_string(pool, msg);
if (s == NULL) {
return "failed to allocate memory";
}
return s;
}
} // namespace
// Very similar to apache/mod_instaweb::ParseDirective.
const char* NgxRewriteOptions::ParseAndSetOptions(
StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
NgxRewriteDriverFactory* driver_factory,
RewriteOptions::OptionScope scope, ngx_conf_t* cf,
ProcessScriptVariablesMode script_mode) {
CHECK_GE(n_args, 1);
StringPiece directive = args[0];
// Remove initial "ModPagespeed" if there is one.
StringPiece mod_pagespeed("ModPagespeed");
if (StringCaseStartsWith(directive, mod_pagespeed)) {
directive.remove_prefix(mod_pagespeed.size());
}
if (GetOptionScope(directive) > scope) {
return ps_error_string_for_option(
pool, directive, "cannot be set at this scope.");
}
bool compile_scripts = false;
if (script_mode != ProcessScriptVariablesMode::kOff) {
// In the old mode we only allowed a few, so restrict to those.
compile_scripts =
StringCaseStartsWith(directive, "LoadFromFile") ||
StringCaseEqual(directive, "EnableFilters") ||
StringCaseEqual(directive, "DisableFilters") ||
StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
StringCaseEqual(directive,
"DownstreamCacheRewrittenPercentageThreshold") ||
StringCaseEqual(directive, "ShardDomain");
// In the new behaviour we also allow scripting of query- and directory-
// scoped options.
compile_scripts |=
script_mode == ProcessScriptVariablesMode::kAll &&
(GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
(StringCaseEqual(directive, "Allow") ||
StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
StringCaseEqual(directive, "Disallow") ||
StringCaseEqual(directive, "DistributableFilters") ||
StringCaseEqual(directive, "Domain") ||
StringCaseEqual(directive, "ExperimentVariable") ||
StringCaseEqual(directive, "ExperimentSpec") ||
StringCaseEqual(directive, "ForbidFilters") ||
StringCaseEqual(directive, "RetainComment") ||
StringCaseEqual(directive, "CustomFetchHeader") ||
StringCaseEqual(directive, "MapOriginDomain") ||
StringCaseEqual(directive, "MapProxyDomain") ||
StringCaseEqual(directive, "MapRewriteDomain") ||
StringCaseEqual(directive, "UrlValuedAttribute") ||
StringCaseEqual(directive, "Library")));
}
ScriptLine* script_line;
script_line = NULL;
if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
clear_inherited_scripts_ = true;
return NGX_CONF_OK;
}
if (compile_scripts) {
CHECK(cf != NULL);
int i;
// Skip the first arg which is always 'pagespeed'
for (i = 1; i < n_args; i++) {
ngx_str_t script_source;
script_source.len = args[i].as_string().length();
std::string tmp = args[i].as_string();
script_source.data = reinterpret_cast<u_char*>(
const_cast<char*>(tmp.c_str()));
if (ngx_http_script_variables_count(&script_source) > 0) {
ngx_http_script_compile_t* sc =
reinterpret_cast<ngx_http_script_compile_t*>(
ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
sc->cf = cf;
sc->source = &script_source;
sc->lengths = reinterpret_cast<ngx_array_t**>(
ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
sc->values = reinterpret_cast<ngx_array_t**>(
ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
sc->variables = 1;
sc->complete_lengths = 1;
sc->complete_values = 1;
if (ngx_http_script_compile(sc) != NGX_OK) {
return ps_error_string_for_option(
pool, directive, "Failed to compile script variables");
} else {
if (script_line == NULL) {
script_line = new ScriptLine(args, n_args, scope);
}
script_line->AddScriptAndArgIndex(sc, i);
}
}
}
if (script_line != NULL) {
script_lines_.push_back(RefCountedPtr<ScriptLine>(script_line));
// We have found script variables in the current configuration line, and
// prepared the associated rewriteoptions for that.
// We will defer parsing, validation and processing of this line to
// request time. That means we are done handling this configuration line.
return NGX_CONF_OK;
}
}
GoogleString msg;
OptionSettingResult result;
if (n_args == 1) {
result = ParseAndSetOptions0(directive, &msg, handler);
} else if (n_args == 2) {
StringPiece arg = args[1];
if (IsDirective(directive, "UseNativeFetcher")) {
result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
arg, driver_factory,
&NgxRewriteDriverFactory::set_use_native_fetcher);
} else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
int max_keepalive_requests;
if (StringToInt(arg, &max_keepalive_requests) &&
max_keepalive_requests > 0) {
driver_factory->set_native_fetcher_max_keepalive_requests(
max_keepalive_requests);
result = RewriteOptions::kOptionOk;
} else {
result = RewriteOptions::kOptionValueInvalid;
}
} else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
if (scope == RewriteOptions::kProcessScopeStrict) {
ProcessScriptVariablesMode mode;
if (StringCaseEqual(arg, "all")) {
mode = ProcessScriptVariablesMode::kAll;
} else if (StringCaseEqual(arg, "on")) {
mode = ProcessScriptVariablesMode::kLegacyRestricted;
} else if (StringCaseEqual(arg, "off")) {
mode = ProcessScriptVariablesMode::kOff;
} else {
return const_cast<char*>(
"pagespeed ProcessScriptVariables: invalid value");
}
if (driver_factory->SetProcessScriptVariables(mode)) {
result = RewriteOptions::kOptionOk;
} else {
return const_cast<char*>(
"pagespeed ProcessScriptVariables: can only be set once");
}
} else {
return const_cast<char*>(
"ProcessScriptVariables is only allowed at the top level");
}
} else {
result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
if (result == RewriteOptions::kOptionNameUnknown) {
result = driver_factory->ParseAndSetOption1(
directive,
arg,
scope >= RewriteOptions::kLegacyProcessScope,
&msg,
handler);
}
}
} else if (n_args == 3) {
result = ParseAndSetOptionFromName2(directive, args[1], args[2],
&msg, handler);
if (result == RewriteOptions::kOptionNameUnknown) {
result = driver_factory->ParseAndSetOption2(
directive,
args[1],
args[2],
scope >= RewriteOptions::kLegacyProcessScope,
&msg,
handler);
}
} else if (n_args == 4) {
result = ParseAndSetOptionFromName3(
directive, args[1], args[2], args[3], &msg, handler);
} else {
result = RewriteOptions::kOptionNameUnknown;
}
switch (result) {
case RewriteOptions::kOptionOk:
return NGX_CONF_OK;
case RewriteOptions::kOptionNameUnknown:
return ps_error_string_for_option(
pool, directive, "not recognized or too many arguments");
case RewriteOptions::kOptionValueInvalid: {
GoogleString full_directive;
for (int i = 0 ; i < n_args ; i++) {
StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
}
return ps_error_string_for_option(pool, full_directive, msg);
}
}
CHECK(false);
return NULL;
}
// Execute all entries in the script_lines vector, and hand the result off to
// ParseAndSetOptions to obtain the final option values.
bool NgxRewriteOptions::ExecuteScriptVariables(
ngx_http_request_t* r, MessageHandler* handler,
NgxRewriteDriverFactory* driver_factory) {
bool script_error = false;
if (script_lines_.size() > 0) {
std::vector<RefCountedPtr<ScriptLine> >::iterator it;
for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
ScriptLine* script_line = it->get();
StringPiece args[NGX_PAGESPEED_MAX_ARGS];
std::vector<ScriptArgIndex*>::iterator cs_it;
int i;
for (i = 0; i < script_line->n_args(); i++) {
args[i] = script_line->args()[i];
}
for (cs_it = script_line->data().begin();
cs_it != script_line->data().end(); cs_it++) {
ngx_http_script_compile_t* script;
ngx_array_t* values;
ngx_array_t* lengths;
ngx_str_t value;
script = (*cs_it)->script();
lengths = *script->lengths;
values = *script->values;
if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
== NULL) {
handler->Message(kError, "ngx_http_script_run error");
script_error = true;
break;
} else {
args[(*cs_it)->index()] = str_to_string_piece(value);
}
}
const char* status = ParseAndSetOptions(args, script_line->n_args(),
r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
ProcessScriptVariablesMode::kOff);
if (status != NULL) {
script_error = true;
handler->Message(kWarning,
"Error setting option value from script: '%s'", status);
break;
}
}
}
if (script_error) {
handler->Message(kWarning,
"Script error(s) in configuration, disabling optimization");
set_enabled(RewriteOptions::kEnabledOff);
return false;
}
return true;
}
void NgxRewriteOptions::CopyScriptLinesTo(
NgxRewriteOptions* destination) const {
destination->script_lines_ = script_lines_;
}
void NgxRewriteOptions::AppendScriptLinesTo(
NgxRewriteOptions* destination) const {
destination->script_lines_.insert(destination->script_lines_.end(),
script_lines_.begin(), script_lines_.end());
}
NgxRewriteOptions* NgxRewriteOptions::Clone() const {
NgxRewriteOptions* options = new NgxRewriteOptions(
StrCat("cloned from ", description()), thread_system());
this->CopyScriptLinesTo(options);
options->Merge(*this);
return options;
}
const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
const RewriteOptions* instance) {
return dynamic_cast<const NgxRewriteOptions*>(instance);
}
NgxRewriteOptions* NgxRewriteOptions::DynamicCast(RewriteOptions* instance) {
return dynamic_cast<NgxRewriteOptions*>(instance);
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_rewrite_options.h
-248
View File
@@ -1,248 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
// Manage configuration for pagespeed. Compare to ApacheConfig.
#ifndef NGX_REWRITE_OPTIONS_H_
#define NGX_REWRITE_OPTIONS_H_
extern "C" {
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
}
#include <vector>
#include "ngx_rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/ref_counted_ptr.h"
#include "pagespeed/kernel/base/stl_util.h" // for STLDeleteElements
#include "pagespeed/system/system_rewrite_options.h"
#define NGX_PAGESPEED_MAX_ARGS 10
namespace net_instaweb {
class NgxRewriteDriverFactory;
class ScriptArgIndex {
public:
explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
: script_(script), index_(index) {
CHECK(script != NULL);
CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
}
virtual ~ScriptArgIndex() {}
ngx_http_script_compile_t* script() { return script_; }
int index() { return index_; }
private:
// Not owned.
ngx_http_script_compile_t* script_;
int index_;
};
// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
// different rewriteoptions.
class ScriptLine : public RefCounted<ScriptLine> {
public:
explicit ScriptLine(StringPiece* args, int n_args,
RewriteOptions::OptionScope scope)
: n_args_(n_args),
scope_(scope) {
for (int i = 0; i < n_args; i++) {
args_[i] = args[i];
}
}
virtual ~ScriptLine() {
STLDeleteElements(&data_);
data_.clear();
}
void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
int script_index) {
CHECK(script != NULL);
CHECK(script_index < NGX_PAGESPEED_MAX_ARGS);
data_.push_back(new ScriptArgIndex(script, script_index));
}
int n_args() { return n_args_;}
StringPiece* args() { return args_;}
RewriteOptions::OptionScope scope() { return scope_; }
std::vector<ScriptArgIndex*>& data() {
return data_;
}
private:
StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
int n_args_;
RewriteOptions::OptionScope scope_;
std::vector<ScriptArgIndex*> data_;
DISALLOW_COPY_AND_ASSIGN(ScriptLine);
};
class NgxRewriteOptions : public SystemRewriteOptions {
public:
// See rewrite_options::Initialize and ::Terminate
static void Initialize();
static void Terminate();
NgxRewriteOptions(const StringPiece& description,
ThreadSystem* thread_system);
explicit NgxRewriteOptions(ThreadSystem* thread_system);
virtual ~NgxRewriteOptions() { }
// args is an array of n_args StringPieces together representing a directive.
// For example:
// ["RewriteLevel", "PassThrough"]
// or
// ["EnableFilters", "combine_css,extend_cache,rewrite_images"]
// or
// ["ShardDomain", "example.com", "s1.example.com,s2.example.com"]
// Apply the directive, returning NGX_CONF_OK on success or an error message
// on failure.
//
// pool is a memory pool for allocating error strings.
// cf is only required when compile_scripts is true
// when compile_scripts is true, the rewrite_options will be prepared
// for replacing any script $variables encountered in args. when false,
// script variables will be substituted using the prepared rewrite options.
const char* ParseAndSetOptions(
StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
NgxRewriteDriverFactory* driver_factory, OptionScope scope,
ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
bool ExecuteScriptVariables(
ngx_http_request_t* r, MessageHandler* handler,
NgxRewriteDriverFactory* driver_factory);
void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
void AppendScriptLinesTo(NgxRewriteOptions* destination) const;
// Make an identical copy of these options and return it.
virtual NgxRewriteOptions* Clone() const;
// Returns a suitably down cast version of 'instance' if it is an instance
// of this class, NULL if not.
static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
const GoogleString& statistics_path() const {
return statistics_path_.value();
}
const GoogleString& global_statistics_path() const {
return global_statistics_path_.value();
}
const GoogleString& console_path() const {
return console_path_.value();
}
const GoogleString& messages_path() const {
return messages_path_.value();
}
const GoogleString& admin_path() const {
return admin_path_.value();
}
const GoogleString& global_admin_path() const {
return global_admin_path_.value();
}
const std::vector<RefCountedPtr<ScriptLine> >& script_lines() const {
return script_lines_;
}
const bool& clear_inherited_scripts() const {
return clear_inherited_scripts_;
}
private:
// Helper methods for ParseAndSetOptions(). Each can:
// - return kOptionNameUnknown and not set msg:
// - directive not handled; continue on with other possible
// interpretations.
// - return kOptionOk and not set msg:
// - directive handled, all's well.
// - return kOptionValueInvalid and set msg:
// - directive handled with an error; return the error to the user.
//
// msg will be shown to the user on kOptionValueInvalid. While it would be
// nice to always use msg and never use the MessageHandler, some option
// parsing code in RewriteOptions expects to write to a MessageHandler. If
// that happens we put a summary on msg so the user sees something, and the
// detailed message goes to their log via handler.
OptionSettingResult ParseAndSetOptions0(
StringPiece directive, GoogleString* msg, MessageHandler* handler);
virtual OptionSettingResult ParseAndSetOptionFromName1(
StringPiece name, StringPiece arg,
GoogleString* msg, MessageHandler* handler);
// We may want to override 2- and 3-argument versions as well in the future,
// but they are not needed yet.
// Keeps the properties added by this subclass. These are merged into
// RewriteOptions::all_properties_ during Initialize().
//
// RewriteOptions uses static initialization to reduce memory usage and
// construction time. All NgxRewriteOptions instances will have the same
// Properties, so we can build the list when we initialize the first one.
static Properties* ngx_properties_;
static void AddProperties();
void Init();
// Add an option to ngx_properties_
template<class OptionClass>
static void add_ngx_option(typename OptionClass::ValueType default_value,
OptionClass NgxRewriteOptions::*offset,
const char* id,
StringPiece option_name,
OptionScope scope,
const char* help,
bool safe_to_print) {
AddProperty(default_value, offset, id, option_name, scope, help,
safe_to_print, ngx_properties_);
}
Option<GoogleString> statistics_path_;
Option<GoogleString> global_statistics_path_;
Option<GoogleString> console_path_;
Option<GoogleString> messages_path_;
Option<GoogleString> admin_path_;
Option<GoogleString> global_admin_path_;
bool clear_inherited_scripts_;
std::vector<RefCountedPtr<ScriptLine> > script_lines_;
// Helper for ParseAndSetOptions. Returns whether the two directives equal,
// ignoring case.
bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
// Returns a given option's scope.
RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
// TODO(jefftk): support fetch proxy in server and location blocks.
DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);
};
} // namespace net_instaweb
#endif // NGX_REWRITE_OPTIONS_H_
lib/ngx_pagespeed/src/ngx_server_context.cc
-98
View File
@@ -1,98 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
#include "ngx_server_context.h"
extern "C" {
#include <ngx_http.h>
}
#include "ngx_pagespeed.h"
#include "ngx_message_handler.h"
#include "ngx_rewrite_driver_factory.h"
#include "ngx_rewrite_options.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "pagespeed/system/add_headers_fetcher.h"
#include "pagespeed/system/loopback_route_fetcher.h"
#include "pagespeed/system/system_request_context.h"
namespace net_instaweb {
NgxServerContext::NgxServerContext(
NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
: SystemServerContext(factory, hostname, port),
ngx_http2_variable_index_(NGX_ERROR) {
}
NgxServerContext::~NgxServerContext() { }
NgxRewriteOptions* NgxServerContext::config() {
return NgxRewriteOptions::DynamicCast(global_options());
}
SystemRequestContext* NgxServerContext::NewRequestContext(
ngx_http_request_t* r) {
// Based on ngx_http_variable_server_port.
bool port_set = false;
int local_port = 0;
#if (NGX_HAVE_INET6)
if (r->connection->local_sockaddr->sa_family == AF_INET6) {
local_port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
r->connection->local_sockaddr)->sin6_port);
port_set = true;
}
#endif
if (!port_set) {
local_port = ntohs(reinterpret_cast<struct sockaddr_in*>(
r->connection->local_sockaddr)->sin_port);
}
ngx_str_t local_ip;
u_char addr[NGX_SOCKADDR_STRLEN];
local_ip.len = NGX_SOCKADDR_STRLEN;
local_ip.data = addr;
ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &local_ip, 0);
if (rc != NGX_OK) {
local_ip.len = 0;
}
SystemRequestContext* ctx = new SystemRequestContext(
thread_system()->NewMutex(), timer(),
ps_determine_host(r), local_port, str_to_string_piece(local_ip));
// See if http2 is in use.
if (ngx_http2_variable_index_ >= 0) {
ngx_http_variable_value_t* val =
ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
if (val != NULL && val->valid) {
StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
if (str_val == "h2" || str_val == "h2c") {
ctx->set_using_http2(true);
}
}
}
return ctx;
}
GoogleString NgxServerContext::FormatOption(StringPiece option_name,
StringPiece args) {
return StrCat("pagespeed ", option_name, " ", args, ";");
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_server_context.h
-79
View File
@@ -1,79 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: jefftk@google.com (Jeff Kaufman)
// Manage pagespeed state across requests. Compare to ApacheResourceManager.
#ifndef NGX_SERVER_CONTEXT_H_
#define NGX_SERVER_CONTEXT_H_
#include "ngx_message_handler.h"
#include "pagespeed/system/system_server_context.h"
extern "C" {
#include <ngx_http.h>
}
namespace net_instaweb {
class NgxRewriteDriverFactory;
class NgxRewriteOptions;
class SystemRequestContext;
class NgxServerContext : public SystemServerContext {
public:
NgxServerContext(
NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
virtual ~NgxServerContext();
// We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
// set_trusted_input() on any ProxyFetches we use to transform internal HTML.
virtual bool ProxiesHtml() const { return false; }
// Call only when you need an NgxRewriteOptions. If you don't need
// nginx-specific behavior, call global_options() instead which doesn't
// downcast.
NgxRewriteOptions* config();
NgxRewriteDriverFactory* ngx_rewrite_driver_factory() { return ngx_factory_; }
SystemRequestContext* NewRequestContext(ngx_http_request_t* r);
NgxMessageHandler* ngx_message_handler() {
return dynamic_cast<NgxMessageHandler*>(message_handler());
}
virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
void set_ngx_http2_variable_index(ngx_int_t idx) {
ngx_http2_variable_index_ = idx;
}
ngx_int_t ngx_http2_variable_index() const {
return ngx_http2_variable_index_;
}
private:
NgxRewriteDriverFactory* ngx_factory_;
// what index the "http2" var is, or NGX_ERROR.
ngx_int_t ngx_http2_variable_index_;
DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
};
} // namespace net_instaweb
#endif // NGX_SERVER_CONTEXT_H_
lib/ngx_pagespeed/src/ngx_url_async_fetcher.cc
-299
View File
@@ -1,299 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: x.dinic@gmail.com (Junmin Xiong)
extern "C" {
#include <ngx_http.h>
#include <ngx_core.h>
}
#include "ngx_url_async_fetcher.h"
#include "ngx_fetch.h"
#include <vector>
#include <algorithm>
#include <string>
#include <list>
#include <map>
#include <set>
#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/inflating_fetch.h"
#include "net/instaweb/public/version.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/condvar.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/pool.h"
#include "pagespeed/kernel/base/pool_element.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/thread_system.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/base/writer.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/response_headers_parser.h"
namespace net_instaweb {
NgxUrlAsyncFetcher::NgxUrlAsyncFetcher(const char* proxy,
ngx_log_t* log,
ngx_msec_t resolver_timeout,
ngx_msec_t fetch_timeout,
ngx_resolver_t* resolver,
int max_keepalive_requests,
ThreadSystem* thread_system,
MessageHandler* handler)
: fetchers_count_(0),
shutdown_(false),
track_original_content_length_(false),
byte_count_(0),
thread_system_(thread_system),
message_handler_(handler),
mutex_(NULL),
max_keepalive_requests_(max_keepalive_requests),
event_connection_(NULL) {
resolver_timeout_ = resolver_timeout;
fetch_timeout_ = fetch_timeout;
ngx_memzero(&proxy_, sizeof(proxy_));
if (proxy != NULL && *proxy != '\0') {
proxy_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
proxy_.url.len = ngx_strlen(proxy);
}
mutex_ = thread_system_->NewMutex();
log_ = log;
pool_ = NULL;
resolver_ = resolver;
// If init fails, set shutdown_ so no fetches will be attempted.
if (!Init(const_cast<ngx_cycle_t*>(ngx_cycle))) {
shutdown_ = true;
message_handler_->Message(
kError, "NgxUrlAsyncFetcher failed to init, fetching disabled.");
}
}
NgxUrlAsyncFetcher::~NgxUrlAsyncFetcher() {
DCHECK(shutdown_) << "Shut down before destructing NgxUrlAsyncFetcher.";
message_handler_->Message(
kInfo,
"Destruct NgxUrlAsyncFetcher with [%d] active fetchers",
ApproximateNumActiveFetches());
CancelActiveFetches();
active_fetches_.DeleteAll();
NgxConnection::Terminate();
if (pool_ != NULL) {
ngx_destroy_pool(pool_);
pool_ = NULL;
}
if (mutex_ != NULL) {
delete mutex_;
mutex_ = NULL;
}
}
bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) {
size_t scheme_offset;
u_short port;
if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
const_cast<char*>("http://")), 7) == 0) {
scheme_offset = 7;
port = 80;
} else if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
const_cast<char*>("https://")), 8) == 0) {
scheme_offset = 8;
port = 443;
} else {
scheme_offset = 0;
port = 80;
}
url->url.data += scheme_offset;
url->url.len -= scheme_offset;
url->default_port = port;
// See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
url->no_resolve = 0;
url->uri_part = 1;
if (ngx_parse_url(pool, url) == NGX_OK) {
return true;
}
return false;
}
// If there are still active requests, cancel them.
void NgxUrlAsyncFetcher::CancelActiveFetches() {
// TODO(oschaaf): this seems tricky, this may end up calling
// FetchComplete, modifying the active fetches while we are looping
// it
for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
e = active_fetches_.end(); p != e; ++p) {
NgxFetch* fetch = *p;
fetch->CallbackDone(false);
}
}
// Create the pool for fetcher, create the pipe, add the read event for main
// thread. It should be called in the worker process.
bool NgxUrlAsyncFetcher::Init(ngx_cycle_t* cycle) {
log_ = cycle->log;
CHECK(event_connection_ == NULL) << "event connection already set";
event_connection_ = new NgxEventConnection(ReadCallback);
if (!event_connection_->Init(cycle)) {
return false;
}
if (pool_ == NULL) {
pool_ = ngx_create_pool(4096, log_);
if (pool_ == NULL) {
ngx_log_error(NGX_LOG_ERR, log_, 0,
"NgxUrlAsyncFetcher::Init create pool failed");
return false;
}
}
if (proxy_.url.len == 0) {
return true;
}
// TODO(oschaaf): shouldn't we do this earlier? Do we need to clean
// up when parsing the url failed?
if (!ParseUrl(&proxy_, pool_)) {
ngx_log_error(NGX_LOG_ERR, log_, 0,
"NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url);
return false;
}
return true;
}
void NgxUrlAsyncFetcher::ShutDown() {
shutdown_ = true;
if (!pending_fetches_.empty()) {
for (Pool<NgxFetch>::iterator p = pending_fetches_.begin(),
e = pending_fetches_.end(); p != e; p++) {
NgxFetch* fetch = *p;
fetch->CallbackDone(false);
}
pending_fetches_.DeleteAll();
}
if (!active_fetches_.empty()) {
for (Pool<NgxFetch>::iterator p = active_fetches_.begin(),
e = active_fetches_.end(); p != e; p++) {
NgxFetch* fetch = *p;
fetch->CallbackDone(false);
}
active_fetches_.Clear();
}
if (event_connection_ != NULL) {
event_connection_->Shutdown();
delete event_connection_;
event_connection_ = NULL;
}
}
// It's called in the rewrite thread. All the fetches are started at
// this function. It will notify the main thread to start the fetch job.
void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* async_fetch) {
// Don't accept new fetches when shut down. This flow is also entered when
// we did not initialize properly in ::Init().
if (shutdown_) {
async_fetch->Done(false);
return;
}
async_fetch = EnableInflation(async_fetch);
NgxFetch* fetch = new NgxFetch(url, async_fetch,
message_handler, log_);
ScopedMutex lock(mutex_);
pending_fetches_.Add(fetch);
// TODO(oschaaf): thread safety on written vs shutdown.
// It is possible that shutdown() is called after writing an event? In that
// case, this could (rarely) fail when it shouldn't.
bool written = event_connection_->WriteEvent(this);
CHECK(written || shutdown_) << "NgxUrlAsyncFetcher: event write failure";
}
// This is the read event which is called in the main thread.
// It will do the real work. Add the work event and start the fetch.
void NgxUrlAsyncFetcher::ReadCallback(const ps_event_data& data) {
std::vector<NgxFetch*> to_start;
NgxUrlAsyncFetcher* fetcher = reinterpret_cast<NgxUrlAsyncFetcher*>(
data.sender);
fetcher->mutex_->Lock();
fetcher->completed_fetches_.DeleteAll();
for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
e = fetcher->pending_fetches_.end(); p != e; p++) {
NgxFetch* fetch = *p;
to_start.push_back(fetch);
}
fetcher->pending_fetches_.Clear();
fetcher->mutex_->Unlock();
for (size_t i = 0; i < to_start.size(); i++) {
fetcher->StartFetch(to_start[i]);
}
return;
}
// TODO(oschaaf): return value is ignored.
bool NgxUrlAsyncFetcher::StartFetch(NgxFetch* fetch) {
mutex_->Lock();
active_fetches_.Add(fetch);
fetchers_count_++;
mutex_->Unlock();
// Don't initiate the fetch when we are shutting down
if (shutdown_) {
fetch->CallbackDone(false);
return false;
}
bool started = fetch->Start(this);
if (!started) {
message_handler_->Message(kWarning, "Fetch failed to start: %s",
fetch->str_url());
fetch->CallbackDone(false);
}
return started;
}
void NgxUrlAsyncFetcher::FetchComplete(NgxFetch* fetch) {
ScopedMutex lock(mutex_);
byte_count_ += fetch->bytes_received();
fetchers_count_--;
active_fetches_.Remove(fetch);
completed_fetches_.Add(fetch);
}
void NgxUrlAsyncFetcher::PrintActiveFetches(MessageHandler* handler) const {
for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
e = active_fetches_.end(); p != e; ++p) {
NgxFetch* fetch = *p;
handler->Message(kInfo, "Active fetch: %s", fetch->str_url());
}
}
} // namespace net_instaweb
lib/ngx_pagespeed/src/ngx_url_async_fetcher.h
-154
View File
@@ -1,154 +0,0 @@
/*
* Copyright 2012 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Author: x.dinic@gmail.com(Junmin Xiong)
//
// Fetch the resources asynchronously in Nginx. The fetcher is called in
// the rewrite thread.
//
// It can communicate with Nginx by pipe. One pipe for one fetcher.
// When new url fetch comes, Fetcher will add it to the pending queue and
// notify the Nginx thread to start the Fetch event. All the events are hooked
// in the main thread's epoll structure.
#ifndef NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
#define NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
extern "C" {
#include <ngx_config.h>
#include <ngx_core.h>
}
#include <vector>
#include "ngx_event_connection.h"
#include "net/instaweb/http/public/url_async_fetcher.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/pool.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/thread_system.h"
namespace net_instaweb {
class AsyncFetch;
class MessageHandler;
class Statistics;
class NgxFetch;
class Variable;
class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
public:
NgxUrlAsyncFetcher(
const char* proxy, ngx_log_t* log, ngx_msec_t resolver_timeout,
ngx_msec_t fetch_timeout, ngx_resolver_t* resolver,
int max_keepalive_requests, ThreadSystem* thread_system,
MessageHandler* handler);
~NgxUrlAsyncFetcher();
// It should be called in the module init_process callback function. Do some
// intializations which can't be done in the master process
bool Init(ngx_cycle_t* cycle);
// shutdown all the fetches.
virtual void ShutDown();
// the read handler in the main thread
static void ReadCallback(const ps_event_data& data);
virtual bool SupportsHttps() const { return false; }
virtual void Fetch(const GoogleString& url,
MessageHandler* message_handler,
AsyncFetch* callback);
bool StartFetch(NgxFetch* fetch);
// Remove the completed fetch from the active fetch set, and put it into a
// completed fetch list to be cleaned up.
void FetchComplete(NgxFetch* fetch);
void PrintActiveFetches(MessageHandler* handler) const;
// Indicates that it should track the original content length for
// fetched resources.
bool track_original_content_length() {
return track_original_content_length_;
}
void set_track_original_content_length(bool x) {
track_original_content_length_ = x;
}
typedef Pool<NgxFetch> NgxFetchPool;
// AnyPendingFetches is accurate only at the time of call; this is
// used conservatively during shutdown. It counts fetches that have been
// requested by some thread, and can include fetches for which no action
// has yet been taken (ie fetches that are not active).
virtual bool AnyPendingFetches() {
return !active_fetches_.empty();
}
// ApproximateNumActiveFetches can under- or over-count and is used only for
// error reporting.
int ApproximateNumActiveFetches() {
return active_fetches_.size();
}
void CancelActiveFetches();
// These must be accessed with mutex_ held.
bool shutdown() const { return shutdown_; }
void set_shutdown(bool s) { shutdown_ = s; }
private:
static void TimeoutHandler(ngx_event_t* tev);
static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool);
friend class NgxFetch;
NgxFetchPool active_fetches_;
// Add the pending task to this list
NgxFetchPool pending_fetches_;
NgxFetchPool completed_fetches_;
ngx_url_t proxy_;
int fetchers_count_;
bool shutdown_;
bool track_original_content_length_;
int64 byte_count_;
ThreadSystem* thread_system_;
MessageHandler* message_handler_;
// Protect the member variable in this class
// active_fetches, pending_fetches
ThreadSystem::CondvarCapableMutex* mutex_;
ngx_pool_t* pool_;
ngx_log_t* log_;
ngx_resolver_t* resolver_;
int max_keepalive_requests_;
ngx_msec_t resolver_timeout_;
ngx_msec_t fetch_timeout_;
NgxEventConnection* event_connection_;
DISALLOW_COPY_AND_ASSIGN(NgxUrlAsyncFetcher);
};
} // namespace net_instaweb
#endif // NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_$
lib/ngx_pagespeed/test/nginx_system_test.sh
-1464
View File
@@ -1,1464 +0,0 @@
#!/bin/bash
#
# Copyright 2012 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: jefftk@google.com (Jeff Kaufman)
#
#
# Runs pagespeed's generic system test and nginx-specific system tests. Not
# intended to be run on it's own; use run_tests.sh instead.
#
# Exits with status 0 if all tests pass.
# Exits with status 1 immediately if any test fails.
# Exits with status 2 if command line args are wrong.
# Exits with status 3 if all failures were expected.
# Exits with status 4 if instructed not to run any tests.
# Inherits the following from environment variables:
: ${USE_VALGRIND:?"Set USE_VALGRIND to true or false"}
: ${NATIVE_FETCHER:?"Set NATIVE_FETCHER to off or on"}
: ${PRIMARY_PORT:?"Set PRIMARY_PORT"}
: ${SECONDARY_PORT:?"Set SECONDARY_PORT"}
: ${CONTROLLER_PORT:?"Set CONTROLLER_PORT"}
: ${RCPORT:?"Set RCPORT"}
: ${MOD_PAGESPEED_DIR:?"Set MOD_PAGESPEED_DIR"}
: ${NGINX_EXECUTABLE:?"Set NGINX_EXECUTABLE"}
: ${PAGESPEED_TEST_HOST:?"Set PAGESPEED_TEST_HOST"}
POSITION_AUX="${POSITION_AUX:-unset}"
RUN_CONTROLLER_TEST="${RUN_CONTROLLER_TEST:-off}"
PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT"
this_dir="$( cd $(dirname "$0") && pwd)"
echo this_dir=$this_dir
TEST_TMP="$this_dir/tmp"
rm -rf "$TEST_TMP"
mkdir -p "$TEST_TMP"
echo TEST_TMP=$TEST_TMP
if [ -d "$MOD_PAGESPEED_DIR/src" ]; then
MOD_PAGESPEED_DIR+="/src"
fi
APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/install/"
SERVER_ROOT="$TEST_TMP/root"
echo SERVER_ROOT=$SERVER_ROOT
rm -rf "$SERVER_ROOT"
mkdir -p "$SERVER_ROOT"
export APACHE_DOC_ROOT="$SERVER_ROOT"
mkdir -p "$APACHE_DOC_ROOT"
make -f "$APACHE_DOC_SRC/Makefile.tests" setup_doc_root \
INSTALL_DATA_DIR="$APACHE_DOC_SRC"
# We need check and check_not before we source SYSTEM_TEST_FILE that provides
# them.
function handle_failure_simple() {
echo "FAIL"
exit 1
}
function check_simple() {
echo " check" "$@"
"$@" || handle_failure_simple
}
function check_not_simple() {
echo " check_not" "$@"
"$@" && handle_failure_simple
}
# Argument list:
# host_name, path, post-data
# Runs 5 keepalive requests both with and without gzip for a few times.
# Curl will use keepalive when running multiple request with one command.
# When post-data is empty, a get request will be executed.
function keepalive_test() {
HOST_NAME=$1
URL="$SECONDARY_HOSTNAME$2"
CURL_LOG_FILE="$1.curl.log"
NGX_LOG_FILE="$1.error.log"
POST_DATA=$3
for ((i=0; i < 10; i++)); do
for accept_encoding in "" "gzip"; do
if [ -z "$POST_DATA" ]; then
curl -m 2 -S -s -v -H "Accept-Encoding: $accept_encoding" \
-H "Host: $HOST_NAME" $URL $URL $URL $URL $URL > /dev/null \
2>>"$TEST_TMP/$CURL_LOG_FILE" || true
else
curl -X POST --data "$POST_DATA" -m 2 -S -s -v \
-H "Accept-Encoding: $accept_encoding" -H "Host: $HOST_NAME"\
$URL $URL $URL $URL $URL > /dev/null \
2>>"$TEST_TMP/$CURL_LOG_FILE" || true
fi
done
done
# Filter the curl output from unimportant messages
OUT=$(cat "$TEST_TMP/$CURL_LOG_FILE"\
| grep -v "^[<>]"\
| grep -v "^{ \\[data not shown"\
| grep -v "^\\* About to connect"\
| grep -v "^\\* Closing"\
| grep -v "^\\* Connected to"\
| grep -v "^\\* Re-using"\
| grep -v "^\\* Connection.*left intact"\
| grep -v "^} \\[data not shown"\
| grep -v "^\\* upload completely sent off"\
| grep -v "^\\* Found bundle for host"\
| grep -v "^\\* connected"\
| grep -v "^\\* Found bundle for host"\
| grep -v "^\\* Adding handle"\
| grep -v "^\\* Curl_addHandleToPipeline"\
| grep -v "^\\* - Conn "\
| grep -v "^\\* Server "\
| grep -v "^\\* Trying.*\\.\\.\\."\
| grep -v "^\\* Hostname was NOT found in DNS cache" \
|| true)
# Nothing should remain after that.
check [ -z "$OUT" ]
# Filter the nginx log from our vhost from unimportant messages.
OUT=$(cat "$TEST_TMP/$NGX_LOG_FILE"\
| grep -v "closed keepalive connection$" \
| grep -v ".*Cache Flush.*" \
|| true)
# Nothing should remain after that.
check [ -z "$OUT" ]
}
function fire_ab_load() {
AB_PID="0"
if hash ab 2>/dev/null; then
ab -n 10000 -k -c 100 http://$PRIMARY_HOSTNAME/ &>/dev/null & AB_PID=$!
# Sleep to allow some queueing up of requests
else
echo "ab is not available, not able to test stressed shutdown and reload."
fi
sleep 2
}
# stop nginx/valgrind
killall -s KILL nginx
killall -s KILL memcheck-amd64-
killall -s KILL memcheck-x86-
SECONDS=0
while pgrep -x nginx >/dev/null || pgrep memcheck >/dev/null;do
if [ $SECONDS -gt 20 ]; then
echo "Old processes won't die" >&2
exit 1
fi
sleep 1
done
check_simple mkdir -p "$TEST_TMP"
PROXY_CACHE="$TEST_TMP/proxycache"
TMP_PROXY_CACHE="$TEST_TMP/tmpproxycache"
ERROR_LOG="$TEST_TMP/error.log"
ACCESS_LOG="$TEST_TMP/access.log"
# Check that we do ok with directories that already exist.
FILE_CACHE="$TEST_TMP/file-cache"
check_simple mkdir "$FILE_CACHE"
# And directories that don't.
SECONDARY_CACHE="$TEST_TMP/file-cache/secondary/"
IPRO_CACHE="$TEST_TMP/file-cache/ipro/"
SHM_CACHE="$TEST_TMP/file-cache/intermediate/directories/with_shm/"
VALGRIND_OPTIONS=""
if $USE_VALGRIND; then
DAEMON=off
else
DAEMON=on
fi
if [ "$NATIVE_FETCHER" = "on" ]; then
RESOLVER="resolver 8.8.8.8;"
else
RESOLVER=""
fi
if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
CONTROLLER="pagespeed ExperimentalCentralControllerPort $CONTROLLER_PORT;"
else
CONTROLLER=""
fi
# set up the config file for the test
PAGESPEED_CONF="$TEST_TMP/pagespeed_test.conf"
PAGESPEED_CONF_TEMPLATE="$this_dir/pagespeed_test.conf.template"
# check for config file template
check_simple test -e "$PAGESPEED_CONF_TEMPLATE"
# create PAGESPEED_CONF by substituting on PAGESPEED_CONF_TEMPLATE
echo > $PAGESPEED_CONF <<EOF
This file is automatically generated from $PAGESPEED_CONF_TEMPLATE"
by nginx_system_test.sh; don't edit here."
EOF
cat $PAGESPEED_CONF_TEMPLATE \
| sed 's#@@DAEMON@@#'"$DAEMON"'#' \
| sed 's#@@TEST_TMP@@#'"$TEST_TMP/"'#' \
| sed 's#@@PROXY_CACHE@@#'"$PROXY_CACHE/"'#' \
| sed 's#@@TMP_PROXY_CACHE@@#'"$TMP_PROXY_CACHE/"'#' \
| sed 's#@@ERROR_LOG@@#'"$ERROR_LOG"'#' \
| sed 's#@@ACCESS_LOG@@#'"$ACCESS_LOG"'#' \
| sed 's#@@FILE_CACHE@@#'"$FILE_CACHE/"'#' \
| sed 's#@@SECONDARY_CACHE@@#'"$SECONDARY_CACHE/"'#' \
| sed 's#@@IPRO_CACHE@@#'"$IPRO_CACHE/"'#' \
| sed 's#@@SHM_CACHE@@#'"$SHM_CACHE/"'#' \
| sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
| sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
| sed 's#@@SECONDARY_PORT@@#'"$SECONDARY_PORT"'#' \
| sed 's#@@CONTROLLER@@#'"$CONTROLLER"'#' \
| sed 's#@@NATIVE_FETCHER@@#'"$NATIVE_FETCHER"'#' \
| sed 's#@@RESOLVER@@#'"$RESOLVER"'#' \
| sed 's#@@RCPORT@@#'"$RCPORT"'#' \
| sed 's#@@PAGESPEED_TEST_HOST@@#'"$PAGESPEED_TEST_HOST"'#' \
>> $PAGESPEED_CONF
# make sure we substituted all the variables
check_not_simple grep @@ $PAGESPEED_CONF
# start nginx with new config
if $USE_VALGRIND; then
(valgrind -q --leak-check=full --gen-suppressions=all \
--show-possibly-lost=no --log-file=$TEST_TMP/valgrind.log \
--suppressions="$this_dir/valgrind.sup" \
$NGINX_EXECUTABLE -c $PAGESPEED_CONF) & VALGRIND_PID=$!
trap "echo 'terminating valgrind!' && kill -s TERM $VALGRIND_PID" EXIT
echo "Wait until nginx is ready to accept connections"
while ! curl -I "http://$PRIMARY_HOSTNAME/mod_pagespeed_example/" 2>/dev/null; do
sleep 0.1;
done
echo "Valgrind (pid:$VALGRIND_PID) is logging to $TEST_TMP/valgrind.log"
else
TRACE_FILE="$TEST_TMP/conf_loading_trace"
$NGINX_EXECUTABLE -c $PAGESPEED_CONF >& "$TRACE_FILE"
if [[ $? -ne 0 ]]; then
echo "FAIL"
cat $TRACE_FILE
if [[ $(grep -c "unknown directive \"proxy_cache_purge\"" $TRACE_FILE) == 1 ]]; then
echo "This test requires proxy_cache_purge. One way to do this:"
echo "Run git clone https://github.com/FRiCKLE/ngx_cache_purge.git"
echo "And compile nginx with the additional ngx_cache_purge module."
fi
rm $TRACE_FILE
exit 1
fi
fi
# Helper methods used by downstream caching tests.
# Helper method that does a wget and verifies that the rewriting status matches
# the $1 argument that is passed to this method.
check_rewriting_status() {
$WGET $WGET_ARGS $CACHABLE_HTML_LOC > $OUT_CONTENTS_FILE
if $1; then
check zgrep -q "pagespeed.ic" $OUT_CONTENTS_FILE
else
check_not zgrep -q "pagespeed.ic" $OUT_CONTENTS_FILE
fi
}
# Helper method that obtains a gzipped response and verifies that rewriting
# has happened. Also takes an extra parameter that identifies extra headers
# to be added during wget.
check_for_rewriting() {
WGET_ARGS="$GZIP_WGET_ARGS $1" check_rewriting_status true
}
# Helper method that obtains a gzipped response and verifies that no rewriting
# has happened. Also takes an extra parameter that identifies extra headers
# to be added during wget.
check_for_no_rewriting() {
WGET_ARGS="$GZIP_WGET_ARGS $1" check_rewriting_status false
}
if $RUN_TESTS; then
echo "Starting tests"
else
if $USE_VALGRIND; then
# Clear valgrind trap
trap - EXIT
echo "To end valgrind, run 'kill -s TERM $VALGRIND_PID'"
fi
echo "Not running tests; commence manual testing"
exit 4
fi
# check_stat in system_test_helpers.sh needs to know whether statstistics are
# enabled, which is always the case for ngx_pagespeed.
statistics_enabled=1
CACHE_FLUSH_TEST="on"
CACHE_PURGE_METHODS="PURGE GET"
SERVER_NAME=nginx
# TODO(cheesy): add an nginx controller test variant. Actually requires some
# thought as we don't currently have a multi-phase test flow in ngx_pagespeeed,
# as far as I can tell, and we really have to update the config and restart
# since the controller is a global setting.
RUN_CONTROLLER_TEST=${RUN_CONTROLLER_TEST:-off}
# run generic system tests
PAGESPEED_DIR="$MOD_PAGESPEED_DIR/pagespeed"
SYSTEM_TEST_FILE="$PAGESPEED_DIR/system/system_test.sh"
REMOTE_CONFIG_TEST_FILE="$PAGESPEED_DIR/system/remote_config_test.sh"
if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
echo "Not finding $SYSTEM_TEST_FILE -- is mod_pagespeed not in a parallel"
echo "directory to ngx_pagespeed?"
exit 2
fi
PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
BEACON_HANDLER="ngx_pagespeed_beacon"
STATISTICS_HANDLER="ngx_pagespeed_statistics"
GLOBAL_STATISTICS_HANDLER="ngx_pagespeed_global_statistics"
MESSAGES_HANDLER="ngx_pagespeed_message"
STATISTICS_URL=http://$PRIMARY_HOSTNAME/$STATISTICS_HANDLER
GLOBAL_STATISTICS_URL=http://$PRIMARY_HOSTNAME/$GLOBAL_STATISTICS_HANDLER
# An expected failure can be indicated like: "~In-place resource optimization~"
PAGESPEED_EXPECTED_FAILURES="
~Cache purging with PageSpeed off in vhost, but on in directory.~
~2-pass ipro with long ModPagespeedInPlaceRewriteDeadline~
~3-pass ipro with short ModPagespeedInPlaceRewriteDeadline~
"
if [ "$POSITION_AUX" = "true" ] ; then
PAGESPEED_EXPECTED_FAILURES+="
~server-side includes~
"
fi
# Some tests are flakey under valgrind. For now, add them to the expected
# failures when running under valgrind.
#
# TODO(sligicki): When the prioritize critical css race condition is fixed, the
# two prioritize_critical_css tests no longer need to be listed here.
# TODO(oschaaf): Now that we wait after we send a SIGHUP for the new worker
# process to handle requests, check if we can remove more from the expected
# failures here under valgrind.
if $USE_VALGRIND; then
PAGESPEED_EXPECTED_FAILURES+="
~combine_css Maximum size of combined CSS.~
~prioritize_critical_css~
~prioritize_critical_css Able to read POST data from temp file.~
~IPRO flow doesn't copy uncacheable resources multiple times.~
~inline_unauthorized_resources allows unauthorized css selectors~
"
fi
# The existing system test takes its arguments as positional parameters, and
# wants different ones than we want, so we need to reset our positional args.
set -- "$PRIMARY_HOSTNAME"
source $SYSTEM_TEST_FILE
# If we were only asked to run a single test, it ran in SYSTEM_TEST_FILE, so
# we're done now and should exit.
if [ -n ${TEST_TO_RUN+x} ]; then
check_failures_and_exit
fi
source $REMOTE_CONFIG_TEST_FILE
# nginx-specific system tests
start_test Test pagespeed directive inside if block inside location block.
URL="http://if-in-location.example.com/"
URL+="mod_pagespeed_example/inline_javascript.html"
# When we specify the X-Custom-Header-Inline-Js that triggers an if block in the
# config which turns on inline_javascript.
WGET_ARGS="--header=X-Custom-Header-Inline-Js:Yes"
http_proxy=$SECONDARY_HOSTNAME \
fetch_until $URL 'grep -c document.write' 1
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $WGET_ARGS $URL)
check_from "$OUT" fgrep "X-Inline-Javascript: Yes"
check_not_from "$OUT" fgrep "inline_javascript.js"
# Without that custom header we don't trigger the if block, and shouldn't get
# any inline javascript.
WGET_ARGS=""
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $WGET_ARGS $URL)
check_from "$OUT" fgrep "X-Inline-Javascript: No"
check_from "$OUT" fgrep "inline_javascript.js"
check_not_from "$OUT" fgrep "document.write"
# Tests related to rewritten response (downstream) caching.
if [ "$NATIVE_FETCHER" = "on" ]; then
echo "Native fetcher doesn't support PURGE requests and so we can't use or"
echo "test downstream caching."
else
CACHABLE_HTML_LOC="http://${SECONDARY_HOSTNAME}/mod_pagespeed_test/cachable_rewritten_html"
CACHABLE_HTML_LOC+="/downstream_caching.html"
TMP_LOG_LINE="proxy_cache.example.com GET /purge/mod_pagespeed_test/cachable_rewritten_"
PURGE_REQUEST_IN_ACCESS_LOG=$TMP_LOG_LINE"html/downstream_caching.html.*(200)"
OUT_CONTENTS_FILE="$OUTDIR/gzipped.html"
OUT_HEADERS_FILE="$OUTDIR/headers.html"
GZIP_WGET_ARGS="-q -S --header=Accept-Encoding:gzip -o $OUT_HEADERS_FILE -O - "
# Number of downstream cache purges should be 0 here.
CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
check_from "$CURRENT_STATS" egrep -q \
"downstream_cache_purge_attempts:[[:space:]]*0"
# The 1st request results in a cache miss, non-rewritten response
# produced by pagespeed code and a subsequent purge request.
start_test Check for case where rewritten cache should get purged.
check_for_no_rewriting "--header=Host:proxy_cache.example.com"
check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
fetch_until $STATISTICS_URL \
'grep -c successful_downstream_cache_purges:[[:space:]]*1' 1
check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
# The 2nd request results in a cache miss (because of the previous purge),
# rewritten response produced by pagespeed code and no new purge requests.
start_test Check for case where rewritten cache should not get purged.
check_for_rewriting "--header=Host:proxy_cache.example.com \
--header=X-PSA-Blocking-Rewrite:psatest"
check egrep -q "X-Cache: MISS" $OUT_HEADERS_FILE
CURRENT_STATS=$($WGET_DUMP $STATISTICS_URL)
check_from "$CURRENT_STATS" egrep -q \
"downstream_cache_purge_attempts:[[:space:]]*1"
check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
# The 3rd request results in a cache hit (because the previous response is
# now present in cache), rewritten response served out from cache and not
# by pagespeed code and no new purge requests.
start_test Check for case where there is a rewritten cache hit.
check_for_rewriting "--header=Host:proxy_cache.example.com"
check egrep -q "X-Cache: HIT" $OUT_HEADERS_FILE
fetch_until $STATISTICS_URL \
'grep -c downstream_cache_purge_attempts:[[:space:]]*1' 1
check [ $(grep -ce "$PURGE_REQUEST_IN_ACCESS_LOG" $ACCESS_LOG) = 1 ];
# Enable one of the beaconing dependent filters and verify interaction
# between beaconing and downstream caching logic, by verifying that
# whenever beaconing code is present in the rewritten page, the
# output is also marked as a cache-miss, indicating that the instrumentation
# was done by the backend.
start_test Check whether beaconing is accompanied by a BYPASS always.
WGET_ARGS="-S --header=Host:proxy_cache.example.com \
--header=X-Allow-Beacon:yes"
CACHABLE_HTML_LOC+="?PageSpeedFilters=lazyload_images"
fetch_until -gzip $CACHABLE_HTML_LOC \
"zgrep -c \"pagespeed\.CriticalImages\.Run\"" 1
check egrep -q 'X-Cache: BYPASS' $WGET_OUTPUT
check fgrep -q 'Cache-Control: no-cache, max-age=0' $WGET_OUTPUT
fi
start_test "Custom statistics paths in server block"
# Served on normal paths by default.
URL="inherit-paths.example.com/ngx_pagespeed_statistics"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q cache_time_us
URL="inherit-paths.example.com/ngx_pagespeed_message"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q Info
URL="inherit-paths.example.com/pagespeed_console"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q console_div
URL="inherit-paths.example.com/pagespeed_admin/"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q Admin
# Not served on normal paths when overriden.
URL="custom-paths.example.com/ngx_pagespeed_statistics"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
check_not_from "$OUT" fgrep -q cache_time_us
URL="custom-paths.example.com/ngx_pagespeed_message"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
check_not_from "$OUT" fgrep -q Info
URL="custom-paths.example.com/pagespeed_console"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
check_not_from "$OUT" fgrep -q console_div
URL="custom-paths.example.com/pagespeed_admin/"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET_DUMP $URL)
check_not_from "$OUT" fgrep -q Admin
# Served on custom paths when overriden
URL="custom-paths.example.com/custom_pagespeed_statistics"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q cache_time_us
URL="custom-paths.example.com/custom_pagespeed_message"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q Info
URL="custom-paths.example.com/custom_pagespeed_console"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q console_div
URL="custom-paths.example.com/custom_pagespeed_admin/"
OUT=$(http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP $URL)
check_from "$OUT" fgrep -q Admin
function gunzip_grep_0ff() {
gunzip - | fgrep -q "color:#00f"
echo $?
}
start_test ipro with mod_deflate
CSS_FILE="http://compressed-css.example.com/"
CSS_FILE+="mod_pagespeed_test/ipro/mod_deflate/big.css"
http_proxy=$SECONDARY_HOSTNAME fetch_until -gzip $CSS_FILE gunzip_grep_0ff 0
start_test ipro with reverse proxy of compressed content
http_proxy=$SECONDARY_HOSTNAME \
fetch_until -gzip http://ipro-proxy.example.com/big.css \
gunzip_grep_0ff 0
# Also test the .pagespeed. version, to make sure we didn't accidentally gunzip
# stuff above when we shouldn't have.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -q -O - \
http://ipro-proxy.example.com/A.big.css.pagespeed.cf.0.css)
check_from "$OUT" fgrep -q "big{color:#00f}"
start_test Respect X-Forwarded-Proto when told to
FETCHED=$OUTDIR/x_forwarded_proto
URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?PageSpeedFilters=add_base_tag
HEADERS="--header=X-Forwarded-Proto:https --header=Host:xfp.example.com"
check $WGET_DUMP -O $FETCHED $HEADERS $URL
# When enabled, we respect X-Forwarded-Proto and thus list base as https.
check fgrep -q '<base href="https://' $FETCHED
start_test Relative redirects starting with a forward slash survive.
URL=http://xfp.example.com/redirect
# wget seems a bit hairy here, when it comes to handling (relative) redirects.
# I could not get this test going with wget, and that is why curl is used here.
# TODO(oschaaf): debug wget some more and swap out curl here.
OUT=$(curl -v --proxy $SECONDARY_HOSTNAME $URL 2>&1)
check_from "$OUT" egrep -q '301 Moved Permanently'
# The important part is that we don't end up with an absolute location here.
check_from "$OUT" grep -q 'Location: /mod_pagespeed_example'
check_not_from "$OUT" grep -q 'Location: http'
# Test that loopback route fetcher works with vhosts not listening on
# 127.0.0.1
start_test IP choice for loopback fetches.
HOST_NAME="loopbackfetch.example.com"
URL="$HOST_NAME/mod_pagespeed_example/rewrite_images.html"
http_proxy=127.0.0.2:$SECONDARY_PORT \
fetch_until $URL 'grep -c .pagespeed.ic' 2
# When we allow ourself to fetch a resource because the Host header tells us
# that it is one of our resources, we should be fetching it from ourself.
start_test "Loopback fetches go to local IPs without DNS lookup"
# If we're properly fetching from ourself we will issue loopback fetches for
# /mod_pagespeed_example/combine_javascriptN.js, which will succeed, so
# combining will work. If we're taking 'Host:www.google.com' to mean that we
# should fetch from www.google.com then those fetches will fail because
# google.com won't have /mod_pagespeed_example/combine_javascriptN.js and so
# we'll not rewrite any resources.
URL="$HOSTNAME/mod_pagespeed_example/combine_javascript.html"
URL+="?PageSpeed=on&PageSpeedFilters=combine_javascript"
fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
# If this accepts the Host header and fetches from google.com it will fail with
# a 404. Instead it should use a loopback fetch and succeed.
URL="$HOSTNAME/mod_pagespeed_example/styles/big.css.pagespeed.ce.8CfGBvwDhH.css"
check wget -O /dev/null --header=Host:www.google.com "$URL"
start_test statistics load
OUT=$($WGET_DUMP $STATISTICS_URL)
check_from "$OUT" grep 'PageSpeed Statistics'
start_test statistics handler full-featured
OUT=$($WGET_DUMP $STATISTICS_URL?config)
check_from "$OUT" grep "InPlaceResourceOptimization (ipro)"
start_test statistics handler properly sets JSON content-type
OUT=$($WGET_DUMP $STATISTICS_URL?json)
check_from "$OUT" grep "Content-Type: application/javascript"
start_test scrape stats works
# This needs to be before reload, when we clear the stats.
check test $(scrape_stat image_rewrite_total_original_bytes) -ge 10000
# Test that ngx_pagespeed keeps working after nginx gets a signal to reload the
# configuration. This is in the middle of tests so that significant work
# happens both before and after.
start_test "Reload config"
function find_exactly_once {
test $(grep -c "$1") -eq 1
}
if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
function check_process_names() {
if ! $USE_VALGRIND; then
# There should be one babysitter and controller running. Under valgrind
# process labels are confused, so skip the check then.
running=$(ps auxww | grep 'ngin[x]')
check_from "$running" find_exactly_once "nginx: pagespeed babysitter"
check_from "$running" find_exactly_once "nginx: pagespeed controller"
fi
}
check_process_names
fi
# Fire up some heavy load if ab is available to test a stressed reload.
# TODO(oschaaf): make sure we wait for the new worker to get ready to accept
# requests.
fire_ab_load
URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
check wget "$URL" -O /dev/null
check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
# Wait for the new worker process with the new configuration to get ready, or
# else the sudden reset of the shared mem statistics/cache might catch upcoming
# tests unaware.
function wait_for_new_worker() {
while [ $(scrape_stat image_rewrite_total_original_bytes) -gt 0 ]; do
echo "Waiting for new worker to get ready..."
sleep .1
done
}
wait_for_new_worker
check wget "$URL" -O /dev/null
if [ "$AB_PID" != "0" ]; then
echo "Kill ab (pid: $AB_PID)"
kill -s KILL $AB_PID &>/dev/null || true
fi
# There should still be just one babysitter and controller running.
if [ "$RUN_CONTROLLER_TEST" = "on" ]; then
check_process_names
check grep "Writing a byte to a pipe to tell the old controller to exit." \
$ERROR_LOG
check grep "Root process is starting a new controller; shutting down." \
$ERROR_LOG
fi
start_test "Shared memory checkpointing"
# We do two tests here:
# 1. Metadata cache: check that an IPRO'd image is still fully optimized after a
# restart.
# 2. Property cache: prioritize critical CSS persists beacon results across
# restarts.
IPRO_URL="$EXAMPLE_ROOT/images/Cuppa.png"
# Checkpoint beacon results to disk if we haven't already.
check wget "$IPRO_URL" -O /dev/null
test_prioritize_critical_css
sleep 2
check wget "$IPRO_URL" -O /dev/null
test_prioritize_critical_css
# Reload nginx again.
check_simple "$NGINX_EXECUTABLE" -s reload -c "$PAGESPEED_CONF"
wait_for_new_worker
# The image should be fully optimized.
OUT=$($CURL -sS -D- -o/dev/null "$IPRO_URL")
check_from "$OUT" grep ^X-Original-Content-Length:
check_from "$OUT" grep ^Content-Length:
content_length=$(echo "$OUT" | grep ^Content-Length: | grep -o [0-9]*)
original_content_length=$(echo "$OUT" | grep ^X-Original-Content-Length: \
| grep -o [0-9]*)
check [ "$original_content_length" -gt "$content_length" ]
# The beacon responses are stored in the metadata cache, so this can only pass
# if we persisted the metadata across restarts.
test_prioritize_critical_css_final
# This is dependent upon having a beacon handler.
test_filter add_instrumentation beacons load.
# Nginx won't sent a Content-Length header on a 204, and while this is correct
# per rfc 2616 wget hangs. Adding --no-http-keep-alive fixes that, as wget will.
# send 'Connection: close' in its request headers, which will make nginx
# respond with that as well. Check that we got a 204.
BEACON_URL="http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
OUT=$(wget -q --save-headers -O - --no-http-keep-alive \
"$PRIMARY_SERVER/$BEACON_HANDLER?ets=load:13&url=$BEACON_URL")
check_from "$OUT" grep '^HTTP/1.1 204'
# The $'...' tells bash to interpret c-style escapes, \r in this case.
check_from "$OUT" grep $'^Cache-Control: max-age=0, no-cache\r$'
# Several cache flushing tests.
start_test Cache flushing works by touching cache.flush in cache directory.
# If we write fixed values into the css file here, there is a risk that
# we will end up seeing the 'right' value because an old process hasn't
# invalidated things yet, rather than because it updated to what we expect
# in the first run followed by what we expect in the second run.
# So, we incorporate the timestamp into RGB colors, using hours
# prefixed with 1 (as 0-123 fits the 0-255 range) to get a second value.
# A one-second precision is good enough since there is a sleep 2 below.
COLOR_SUFFIX=`date +%H,%M,%S\)`
COLOR0=rgb\($COLOR_SUFFIX
COLOR1=rgb\(1$COLOR_SUFFIX
# We test on three different cache setups:
#
# 1. A virtual host using the normal FileCachePath.
# 2. Another virtual host with a different FileCachePath.
# 3. Another virtual host with a different CacheFlushFilename.
#
# This means we need to repeat many of the steps three times.
echo "Clear out our existing state before we begin the test."
check touch "$FILE_CACHE/cache.flush"
check touch "$FILE_CACHE/othercache.flush"
check touch "$SECONDARY_CACHE/cache.flush"
check touch "$IPRO_CACHE/cache.flush"
sleep 1
CACHE_TESTING_DIR="$SERVER_ROOT/cache_flush/"
CACHE_TESTING_TMPDIR="$CACHE_TESTING_DIR/$$"
mkdir "$CACHE_TESTING_TMPDIR"
cp "$CACHE_TESTING_DIR/cache_flush_test.html" "$CACHE_TESTING_TMPDIR/"
CSS_FILE="$CACHE_TESTING_TMPDIR/update.css"
echo ".class myclass { color: $COLOR0; }" > "$CSS_FILE"
URL_PATH="cache_flush/$$/cache_flush_test.html"
URL="$SECONDARY_HOSTNAME/$URL_PATH"
CACHE_A="--header=Host:cache_a.example.com"
fetch_until $URL "grep -c $COLOR0" 1 $CACHE_A
CACHE_B="--header=Host:cache_b.example.com"
fetch_until $URL "grep -c $COLOR0" 1 $CACHE_B
CACHE_C="--header=Host:cache_c.example.com"
fetch_until $URL "grep -c $COLOR0" 1 $CACHE_C
# All three caches are now populated.
# Track how many flushes were noticed by pagespeed processes up till this point
# in time. Note that each process/vhost separately detects the 'flush'.
# A helper function just used here to look up the cache flush count for each
# cache.
function cache_flush_count_scraper {
CACHE_LETTER=$1 # a, b, or c
URL="$SECONDARY_HOSTNAME/ngx_pagespeed_statistics"
HOST="--header=Host:cache_${CACHE_LETTER}.example.com"
$WGET_DUMP $HOST $URL | egrep "^cache_flush_count:? " | awk '{print $2}'
}
NUM_INITIAL_FLUSHES_A=$(cache_flush_count_scraper a)
NUM_INITIAL_FLUSHES_B=$(cache_flush_count_scraper b)
NUM_INITIAL_FLUSHES_C=$(cache_flush_count_scraper c)
# Now change the file to $COLOR1.
echo ".class myclass { color: $COLOR1; }" > "$CSS_FILE"
# We expect to have a stale cache for 5 seconds, so the result should stay
# $COLOR0. This only works because we have only one worker process. If we had
# more than one then the worker process handling this request might be different
# than the one that got the previous one, and it wouldn't be in cache.
OUT="$($WGET_DUMP $CACHE_A "$URL")"
check_from "$OUT" fgrep $COLOR0
OUT="$($WGET_DUMP $CACHE_B "$URL")"
check_from "$OUT" fgrep $COLOR0
OUT="$($WGET_DUMP $CACHE_C "$URL")"
check_from "$OUT" fgrep $COLOR0
# Flush the cache by touching a special file in the cache directory. Now
# css gets re-read and we get $COLOR1 in the output. Sleep here to avoid
# a race due to 1-second granularity of file-system timestamp checks. For
# the test to pass we need to see time pass from the previous 'touch'.
#
# The three vhosts here all have CacheFlushPollIntervalSec set to 1.
sleep 2
check touch "$FILE_CACHE/cache.flush"
sleep 1
# Check that CACHE_A flushed properly.
fetch_until $URL "grep -c $COLOR1" 1 $CACHE_A
# Cache was just flushed, so it should see see exactly one flush and the other
# two should see none.
NUM_MEDIAL_FLUSHES_A=$(cache_flush_count_scraper a)
NUM_MEDIAL_FLUSHES_B=$(cache_flush_count_scraper b)
NUM_MEDIAL_FLUSHES_C=$(cache_flush_count_scraper c)
check [ $(($NUM_MEDIAL_FLUSHES_A - $NUM_INITIAL_FLUSHES_A)) -eq 1 ]
check [ $NUM_MEDIAL_FLUSHES_B -eq $NUM_INITIAL_FLUSHES_B ]
check [ $NUM_MEDIAL_FLUSHES_C -eq $NUM_INITIAL_FLUSHES_C ]
start_test Flushing one cache does not flush all caches.
# Check that CACHE_B and CACHE_C are still serving a stale version.
OUT="$($WGET_DUMP $CACHE_B "$URL")"
check_from "$OUT" fgrep $COLOR0
OUT="$($WGET_DUMP $CACHE_C "$URL")"
check_from "$OUT" fgrep $COLOR0
start_test Secondary caches also flush.
# Now flush the other two files so they can see the color change.
check touch "$FILE_CACHE/othercache.flush"
check touch "$SECONDARY_CACHE/cache.flush"
sleep 1
# Check that CACHE_B and C flushed properly.
fetch_until $URL "grep -c $COLOR1" 1 $CACHE_B
fetch_until $URL "grep -c $COLOR1" 1 $CACHE_C
# Now cache A should see no flush while caches B and C should each see a flush.
NUM_FINAL_FLUSHES_A=$(cache_flush_count_scraper a)
NUM_FINAL_FLUSHES_B=$(cache_flush_count_scraper b)
NUM_FINAL_FLUSHES_C=$(cache_flush_count_scraper c)
check [ $NUM_FINAL_FLUSHES_A -eq $NUM_MEDIAL_FLUSHES_A ]
check [ $(($NUM_FINAL_FLUSHES_B - $NUM_MEDIAL_FLUSHES_B)) -eq 1 ]
check [ $(($NUM_FINAL_FLUSHES_C - $NUM_MEDIAL_FLUSHES_C)) -eq 1 ]
# Clean up so we don't leave behind a stray file not under source control.
rm -rf "$CACHE_TESTING_TMPDIR"
# connection_refused.html references modpagespeed.com:1023/someimage.png.
# Pagespeed will attempt to connect to that host and port to fetch the input
# resource using serf. We expect the connection to be refused. Relies on
# "pagespeed Domain modpagespeed.com:1023" in the config. Also relies on
# running after a cache-flush to avoid bypassing the serf fetch, since pagespeed
# remembers fetch-failures in its cache for 5 minutes.
start_test Connection refused handling
# Monitor the log starting now. tail -F will catch log rotations.
FETCHER_REFUSED_PATH=$TESTTMP/instaweb_fetcher_refused
rm -f $FETCHER_REFUSED_PATH
LOG="$TEST_TMP/error.log"
echo LOG = $LOG
tail --sleep-interval=0.1 -F $LOG > $FETCHER_REFUSED_PATH &
TAIL_PID=$!
# Wait for tail to start.
echo -n "Waiting for tail to start..."
while [ ! -s $FETCHER_REFUSED_PATH ]; do
sleep 0.1
echo -n "."
done
echo "done!"
# Actually kick off the request.
echo $WGET_DUMP $TEST_ROOT/connection_refused.html
echo checking...
check $WGET_DUMP $TEST_ROOT/connection_refused.html > /dev/null
echo check done
# If we are spewing errors, this gives time to spew lots of them.
sleep 1
# Wait up to 10 seconds for the background fetch of someimage.png to fail.
if [ "$NATIVE_FETCHER" = "on" ]; then
EXPECTED="111: Connection refused"
else
EXPECTED="Serf status 111"
fi
for i in {1..100}; do
ERRS=$(grep -c "$EXPECTED" $FETCHER_REFUSED_PATH || true)
if [ $ERRS -ge 1 ]; then
break;
fi;
echo -n "."
sleep 0.1
done;
echo "."
# Kill the log monitor silently.
kill $TAIL_PID
wait $TAIL_PID 2> /dev/null || true
check [ $ERRS -ge 1 ]
# TODO(jefftk): when we support ListOutstandingUrlsOnError uncomment the below
#
## Make sure we have the URL detail we expect because ListOutstandingUrlsOnError
## is on in the config file.
#echo Check that ListOutstandingUrlsOnError works
#check grep "URL http://modpagespeed.com:1023/someimage.png active for " \
# $FETCHER_REFUSED_PATH
start_test Blocking rewrite enabled.
# We assume that blocking_rewrite_test_dont_reuse_1.jpg will not be
# rewritten on the first request since it takes significantly more time to
# rewrite than the rewrite deadline and it is not already accessed by
# another request earlier.
BLOCKING_REWRITE_URL="$TEST_ROOT/blocking_rewrite.html"
BLOCKING_REWRITE_URL+="?PageSpeedFilters=rewrite_images"
OUTFILE=$WGET_DIR/blocking_rewrite.out.html
OLDSTATS=$WGET_DIR/blocking_rewrite_stats.old
NEWSTATS=$WGET_DIR/blocking_rewrite_stats.new
$WGET_DUMP $STATISTICS_URL > $OLDSTATS
check $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest'\
$BLOCKING_REWRITE_URL -O $OUTFILE
$WGET_DUMP $STATISTICS_URL > $NEWSTATS
check_stat $OLDSTATS $NEWSTATS image_rewrites 1
check_stat $OLDSTATS $NEWSTATS cache_hits 0
check_stat $OLDSTATS $NEWSTATS cache_misses 2
check_stat $OLDSTATS $NEWSTATS cache_inserts 3
# TODO(sligocki): There is no stat num_rewrites_executed. Fix.
#check_stat $OLDSTATS $NEWSTATS num_rewrites_executed 1
start_test Blocking rewrite enabled using wrong key.
URL="blocking.example.com/mod_pagespeed_test/blocking_rewrite_another.html"
OUTFILE=$WGET_DIR/blocking_rewrite.out.html
http_proxy=$SECONDARY_HOSTNAME check $WGET_DUMP \
--header 'X-PSA-Blocking-Rewrite: junk' \
$URL > $OUTFILE
check [ $(grep -c "[.]pagespeed[.]" $OUTFILE) -lt 1 ]
http_proxy=$SECONDARY_HOSTNAME fetch_until $URL \
'grep -c [.]pagespeed[.]' 1
run_post_cache_flush
# Test ForbidAllDisabledFilters, which is set in the config for
# /mod_pagespeed_test/forbid_all_disabled/disabled/ where we've disabled
# remove_quotes, remove_comments, and collapse_whitespace. We fetch 3 times
# trying to circumvent the forbidden flag: a normal fetch, a fetch using a query
# parameter to try to enable the forbidden filters, and a fetch using a request
# header to try to enable the forbidden filters.
function test_forbid_all_disabled() {
QUERYP="$1"
HEADER="$2"
if [ -n "$QUERYP" ]; then
INLINE_CSS=",-inline_css"
else
INLINE_CSS="?PageSpeedFilters=-inline_css"
fi
WGET_ARGS="--header=X-PSA-Blocking-Rewrite:psatest"
URL=$TEST_ROOT/forbid_all_disabled/disabled/forbidden.html
OUTFILE="$TESTTMP/test_forbid_all_disabled"
# Fetch testing that forbidden filters stay disabled.
echo $WGET $HEADER $URL$QUERYP$INLINE_CSS
$WGET $WGET_ARGS -q -O $OUTFILE $HEADER $URL$QUERYP$INLINE_CSS
check egrep -q '<link rel="stylesheet' $OUTFILE
check egrep -q '<!--' $OUTFILE
check egrep -q ' <li>' $OUTFILE
# Fetch testing that enabling inline_css works.
echo $WGET $HEADER $URL
$WGET $WGET_ARGS -q -O $OUTFILE $HEADER $URL
check egrep -q '<style>.yellow' $OUTFILE
rm -f $OUTFILE
}
start_test ForbidAllDisabledFilters baseline check.
test_forbid_all_disabled "" ""
start_test ForbidAllDisabledFilters query parameters check.
QUERYP="?PageSpeedFilters="
QUERYP="${QUERYP}+remove_quotes,+remove_comments,+collapse_whitespace"
test_forbid_all_disabled $QUERYP ""
start_test ForbidAllDisabledFilters request headers check.
HEADER="--header=PageSpeedFilters:"
HEADER="${HEADER}+remove_quotes,+remove_comments,+collapse_whitespace"
test_forbid_all_disabled "" $HEADER
# Test the experiment framework (Furious).
start_test PageSpeedExperiment cookie is set.
EXP_EXAMPLE="http://experiment.example.com/mod_pagespeed_example"
EXP_EXTEND_CACHE="$EXP_EXAMPLE/extend_cache.html"
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $EXP_EXTEND_CACHE)
check_from "$OUT" fgrep "PageSpeedExperiment="
MATCHES=$(echo "$OUT" | grep -c "PageSpeedExperiment=")
check [ $MATCHES -eq 1 ]
start_test PageSpeedFilters query param should disable experiments.
URL="$EXP_EXTEND_CACHE?PageSpeed=on&PageSpeedFilters=rewrite_css"
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
check_not_from "$OUT" fgrep 'PageSpeedExperiment='
start_test experiment assignment can be forced
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
"$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=2")
check_from "$OUT" fgrep 'PageSpeedExperiment=2'
start_test experiment assignment can be forced to a 0% experiment
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
"$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=3")
check_from "$OUT" fgrep 'PageSpeedExperiment=3'
start_test experiment assignment can be forced even if already assigned
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
--header Cookie:PageSpeedExperiment=7 \
"$EXP_EXTEND_CACHE?PageSpeedEnrollExperiment=2")
check_from "$OUT" fgrep 'PageSpeedExperiment=2'
start_test If the user is already assigned, no need to assign them again.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
--header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE)
check_not_from "$OUT" fgrep 'PageSpeedExperiment='
start_test The beacon should include the experiment id.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
--header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE)
BEACON_CODE="pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load',"
BEACON_CODE+=" '&exptid=2', 'http://experiment.example.com/"
BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
check_from "$OUT" grep "$BEACON_CODE"
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
$EXP_EXTEND_CACHE)
BEACON_CODE="pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load',"
BEACON_CODE+=" '&exptid=7', 'http://experiment.example.com/"
BEACON_CODE+="mod_pagespeed_example/extend_cache.html');"
check_from "$OUT" grep "$BEACON_CODE"
start_test The no-experiment group beacon should not include an experiment id.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
--header='Cookie: PageSpeedExperiment=0' $EXP_EXTEND_CACHE)
check_not_from "$OUT" grep 'pagespeed_beacon.*exptid'
# We expect id=7 to be index=a and id=2 to be index=b because that's the
# order they're defined in the config file.
start_test Resource urls are rewritten to include experiment indexes.
http_proxy=$SECONDARY_HOSTNAME \
fetch_until $EXP_EXTEND_CACHE \
"fgrep -c .pagespeed.a.ic." 1 --header=Cookie:PageSpeedExperiment=7
http_proxy=$SECONDARY_HOSTNAME \
fetch_until $EXP_EXTEND_CACHE \
"fgrep -c .pagespeed.b.ic." 1 --header=Cookie:PageSpeedExperiment=2
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
$EXP_EXTEND_CACHE)
check_from "$OUT" fgrep ".pagespeed.a.ic."
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
$EXP_EXTEND_CACHE)
check_from "$OUT" fgrep ".pagespeed.b.ic."
start_test Images are different when the url specifies different experiments.
# While the images are the same, image B should be smaller because in the config
# file we enable convert_jpeg_to_progressive only for id=2 (side B). Ideally we
# would check that it was actually progressive, by checking whether "identify
# -verbose filename" produced "Interlace: JPEG" or "Interlace: None", but that
# would introduce a dependency on imagemagick. This is just as accurate, but
# more brittle (because changes to our compression code would change the
# computed file sizes).
IMG_A="$EXP_EXAMPLE/images/xPuzzle.jpg.pagespeed.a.ic.fakehash.jpg"
IMG_B="$EXP_EXAMPLE/images/xPuzzle.jpg.pagespeed.b.ic.fakehash.jpg"
http_proxy=$SECONDARY_HOSTNAME fetch_until $IMG_A 'wc -c' 102902 "" -le
http_proxy=$SECONDARY_HOSTNAME fetch_until $IMG_B 'wc -c' 98276 "" -le
start_test Analytics javascript is added for the experimental group.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
$EXP_EXTEND_CACHE)
check_from "$OUT" fgrep -q 'Experiment: 2'
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
$EXP_EXTEND_CACHE)
check_from "$OUT" fgrep -q 'Experiment: 7'
start_test Analytics javascript is not added for the no-experiment group.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
$EXP_EXTEND_CACHE)
check_not_from "$OUT" fgrep -q 'Experiment:'
start_test Analytics javascript is not added for any group with Analytics off.
EXP_NO_GA_EXTEND_CACHE="http://experiment.noga.example.com"
EXP_NO_GA_EXTEND_CACHE+="/mod_pagespeed_example/extend_cache.html"
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=2' \
$EXP_NO_GA_EXTEND_CACHE)
check_not_from "$OUT" fgrep -q 'Experiment:'
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \
$EXP_NO_GA_EXTEND_CACHE)
check_not_from "$OUT" fgrep -q 'Experiment:'
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=0' \
$EXP_NO_GA_EXTEND_CACHE)
check_not_from "$OUT" fgrep -q 'Experiment:'
# Repeat the critical_css_filter test on a host that processes post data via
# temp files to test that ngx_pagespeed specific code path.
filter_spec_method="headers"
test_filter prioritize_critical_css Able to read POST data from temp file.
URL="http://beacon-post-temp-file.example.com/mod_pagespeed_example/prioritize_critical_css.html"
http_proxy=$SECONDARY_HOSTNAME\
fetch_until -save $URL 'fgrep -c pagespeed.criticalCssBeaconInit' 1
check [ $(fgrep -o ".very_large_class_name_" $FETCH_FILE | wc -l) -eq 36 ]
CALL_PAT=".*criticalCssBeaconInit("
SKIP_ARG="[^,]*,"
CAPTURE_ARG="'\([^']*\)'.*"
BEACON_PATH=$(sed -n "s/${CALL_PAT}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
ESCAPED_URL=$( \
sed -n "s/${CALL_PAT}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
OPTIONS_HASH=$( \
sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE)
NONCE=$( \
sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" \
$FETCH_FILE)
BEACON_URL="http://${SECONDARY_HOSTNAME}${BEACON_PATH}?url=${ESCAPED_URL}"
BEACON_DATA="oh=${OPTIONS_HASH}&n=${NONCE}&cs=.big,.blue,.bold,.foo"
OUT=$(wget -q --save-headers -O - --no-http-keep-alive \
--post-data "$BEACON_DATA" "$BEACON_URL" \
--header "Host:beacon-post-temp-file.example.com")
check_from "$OUT" grep '^HTTP/1.1 204'
# Now make sure we see the correct critical css rules.
http_proxy=$SECONDARY_HOSTNAME\
fetch_until $URL \
'grep -c <style>[.]blue{[^}]*}</style>' 1
http_proxy=$SECONDARY_HOSTNAME\
fetch_until $URL \
'grep -c <style>[.]big{[^}]*}</style>' 1
http_proxy=$SECONDARY_HOSTNAME\
fetch_until $URL \
'grep -c <style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' 1
http_proxy=$SECONDARY_HOSTNAME\
fetch_until -save $URL \
'grep -c <style>[.]foo{[^}]*}</style>' 1
# The last one should also have the other 3, too.
check [ `grep -c '<style>[.]blue{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
check [ `grep -c '<style>[.]big{[^}]*}</style>' $FETCH_UNTIL_OUTFILE` = 1 ]
check [ `grep -c '<style>[.]blue{[^}]*}[.]bold{[^}]*}</style>' \
$FETCH_UNTIL_OUTFILE` = 1 ]
start_test keepalive with html rewriting
keepalive_test "keepalive-html.example.com"\
"/mod_pagespeed_example/rewrite_images.html" ""
start_test keepalive with serving resources
keepalive_test "keepalive-resource.example.com"\
"/mod_pagespeed_example/combine_javascript2.js+combine_javascript1.js+combine_javascript2.js.pagespeed.jc.0.js"\
""
BEACON_URL="http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
start_test keepalive with beacon get requests
keepalive_test "keepalive-beacon-get.example.com"\
"/$BEACON_HANDLER?ets=load:13&url=$BEACON_URL" ""
BEACON_DATA="url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
BEACON_DATA+="image_rewriting%2Frewrite_images.html"
BEACON_DATA+="&oh=$OPTIONS_HASH&ci=2932493096"
start_test keepalive with beacon post requests
keepalive_test "keepalive-beacon-post.example.com" "/$BEACON_HANDLER"\
"$BEACON_DATA"
start_test keepalive with static resources
keepalive_test "keepalive-static.example.com"\
"/pagespeed_custom_static/js_defer.0.js" ""
start_test pagespeed_custom_static defer js served with correct headers.
# First, determine which hash js_defer is served with. We need a correct hash
# to get it served up with an Etag, which is one of the things we want to test.
URL="$HOSTNAME/mod_pagespeed_example/defer_javascript.html?PageSpeed=on&PageSpeedFilters=defer_javascript"
OUT=$($WGET_DUMP $URL)
HASH=$(echo $OUT \
| grep --only-matching "/js_defer\\.*\([^.]\)*.js" | cut -d '.' -f 2)
start_test JS gzip headers
JS_URL="$HOSTNAME/pagespeed_custom_static/js_defer.$HASH.js"
JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
$JS_URL 2>&1)
check_200_http_response "$JS_HEADERS"
check_from "$JS_HEADERS" fgrep -qi 'Content-Encoding: gzip'
check_from "$JS_HEADERS" fgrep -qi 'Vary: Accept-Encoding'
# Nginx's gzip module clears etags, which we don't want. Make sure we have it.
check_from "$JS_HEADERS" egrep -qi 'Etag: W/"0"'
check_from "$JS_HEADERS" fgrep -qi 'Last-Modified:'
start_test PageSpeedFilters response headers is interpreted
URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/
OUT=$($WGET_DUMP --header=Host:response-header-filters.example.com $URL)
check_from "$OUT" egrep -qi 'addInstrumentationInit'
OUT=$($WGET_DUMP --header=Host:response-header-disable.example.com $URL)
check_not_from "$OUT" egrep -qi 'addInstrumentationInit'
# TODO(jmaessen, jefftk): Port proxying tests, which rely on pointing a
# MapProxyDomain construct at a static server. Perhaps $HOSTNAME will
# serve, but the tests need to use different urls then. For mod_pagespeed these
# tests immediately precede the "scrape_secondary_stat" definition in
# system_test.sh.
start_test messages load
OUT=$($WGET_DUMP "$HOSTNAME/ngx_pagespeed_message")
check_not_from "$OUT" grep "Writing to ngx_pagespeed_message failed."
check_from "$OUT" grep -q "/mod_pagespeed_example"
start_test Check keepalive after a 304 responses.
# '-m 2' specifies that the whole operation is allowed to take 2 seconds max.
check curl -vv -m 2 http://$PRIMARY_HOSTNAME/foo.css.pagespeed.ce.0.css \
-H 'If-Modified-Since: Z' http://$PRIMARY_HOSTNAME/foo
start_test Date response header set
OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html)
check_not_from "$OUT" egrep -q '^Date: Thu, 01 Jan 1970 00:00:00 GMT'
OUT=$($WGET_DUMP --header=Host:date.example.com \
http://$SECONDARY_HOSTNAME/mod_pagespeed_example/combine_css.html)
check_from "$OUT" egrep -q '^Date: Fri, 16 Oct 2009 23:05:07 GMT'
WGET_ARGS=
#very basic tests to test gzip nesting configuration
start_test Nested gzip gzip off
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/"
HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test1.example.com"
OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
check_not_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
check_not_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
start_test Nested gzip gzip on
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/styles/big.css"
HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test1.example.com"
OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
check_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
check_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
start_test Nested gzip pagespeed off
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/"
HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test2.example.com"
OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
check_not_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
check_not_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
start_test Nested gzip pagespeed on
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/styles/big.css"
HEADERS="--header=Accept-Encoding:gzip --header=Host:gzip-test2.example.com"
OUT=$($WGET_DUMP -O /dev/null -S $HEADERS $URL 2>&1)
check_from "$OUT" fgrep -qi 'Content-Encoding: gzip'
check_from "$OUT" fgrep -qi 'Vary: Accept-Encoding'
start_test Test that POST requests are rewritten.
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
HEADERS="--header=Host:proxy-post.example.com --post-data=abcdefgh"
OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
check_from "$OUT" fgrep -qi 'addInstrumentationInit'
start_test Test that we can modify filters via script variables.
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
HEADERS="--header=Host:script-filters.example.com --header=X-Script:1"
OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
check_from "$OUT" fgrep -qi 'addInstrumentationInit'
HEADERS="--header=Host:script-filters.example.com"
OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
check_not_from "$OUT" fgrep -qi 'addInstrumentationInit'
start_test Test that we can modify domain sharding via script variables.
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
HEADERS="--header=Host:script-filters.example.com"
OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
check_from "$OUT" fgrep "http://cdn1.example.com"
check_from "$OUT" fgrep "http://cdn2.example.com"
URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
HEADERS="--header=Host:script-filters.example.com --header=X-Script:1"
OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
check_not_from "$OUT" fgrep "http://cdn1.example.com"
check_not_from "$OUT" fgrep "http://cdn2.example.com"
if [ "$NATIVE_FETCHER" != "on" ]; then
start_test Test that we can rewrite an HTTPS resource.
fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
'grep -c /https_gstatic_dot_com/1.gif.pagespeed.ce' 1
fi
start_test Base config has purging disabled. Check error message syntax.
OUT=$($WGET_DUMP "$HOSTNAME/pagespeed_admin/cache?purge=*")
check_from "$OUT" fgrep -q "pagespeed EnableCachePurge on;"
start_test Default server header in html flow.
URL=http://headers.example.com/mod_pagespeed_example/
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
# '|| true' in the line below supresses the exit code from grep when there is no
# match in its input (1).
MATCHES=$(echo "$OUT" | grep -c "Server: nginx/") || true
check [ $MATCHES -eq 1 ]
start_test Default server header in resource flow.
URL=http://headers.example.com/mod_pagespeed_example/
URL+=combine_javascript2.js+combine_javascript1.js.pagespeed.jc.0.js
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
MATCHES=$(echo "$OUT" | grep -c "Server: nginx/") || true
check [ $MATCHES -eq 1 ]
start_test Default server header in IPRO flow.
URL=http://headers.example.com//mod_pagespeed_example/combine_javascript2.js
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
MATCHES=$(echo "$OUT" | grep -c "Server: nginx/") || true
check [ $MATCHES -eq 1 ]
start_test Override server header in html flow.
URL=http://headers.example.com/mod_pagespeed_test/whitespace.html
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
MATCHES=$(echo "$OUT" | grep -c "Server: override") || true
check [ $MATCHES -eq 1 ]
if [ "$POSITION_AUX" = "true" ] ; then
start_test Override server header in resource flow.
URL=http://headers.example.com/mod_pagespeed_test/
URL+=A.proxy_pass.css.pagespeed.cf.0.css
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
MATCHES=$(echo "$OUT" | grep -c "Server: override") || true
check [ $MATCHES -eq 1 ]
start_test Override server header in IPRO flow.
URL=http://headers.example.com/mod_pagespeed_test/proxy_pass.css
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
MATCHES=$(echo "$OUT" | grep -c "Server: override") || true
check [ $MATCHES -eq 1 ]
fi
start_test Conditional cache-control header override in resource flow.
URL=http://headers.example.com/mod_pagespeed_test/
URL+=A.doesnotexist.css.pagespeed.cf.0.css
# The 404 response makes wget exit with an error code, which we ignore.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || true
# We ignored the exit code, check if we got a 404 response.
check_from "$OUT" fgrep -qi '404'
MATCHES=$(echo "$OUT" | grep -c "Cache-Control: override") || true
check [ $MATCHES -eq 1 ]
start_test Custom 404 does not crash.
URL=http://custom404.example.com/mod_pagespeed_test/
URL+=A.doesnotexist.css.pagespeed.cf.0.css
# The 404 response makes wget exit with an error code, which we ignore.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || true
# We ignored the exit code, check if we got a 404 response.
check_from "$OUT" fgrep -qi '404'
start_test Single Vary: Accept-Encoding header in IPRO flow
URL=http://psol-vary.example.com/mod_pagespeed_example/styles/index_style.css
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
# First hit will be recorded and passed on untouched
MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
check [ $MATCHES -eq 1 ]
# Fetch until we get a fully optimized response
http_proxy=$SECONDARY_HOSTNAME \
fetch_until $URL "fgrep -c W/\"PSA" 1 --save-headers
# Test the optimized response.
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
check [ $MATCHES -eq 1 ]
start_test Follow flushes can be turned off.
echo "Check that FollowFlushes off outputs a single chunk"
check_flushing noflush 5.4 1
start_test Special responses from php are handled OK.
URL="http://special-response.example.com/A.foo.css.pagespeed.cf.0.css"
OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -S --content-on-error $URL 2>&1) || true
check_from "$OUT" fgrep -qi '404'
check_from "$OUT" fgrep -q "PHP with a call to flush"
start_test Shutting down.
# Fire up some heavy load if ab is available to test a stressed shutdown
fire_ab_load
if $USE_VALGRIND; then
# SIGQUIT requests a graceful shutdown.
kill -s QUIT $VALGRIND_PID
while pgrep memcheck > /dev/null; do sleep 1; done
# Clear the previously set trap, we don't need it anymore.
trap - EXIT
start_test No Valgrind complaints.
check_not [ -s "$TEST_TMP/valgrind.log" ]
else
check_simple "$NGINX_EXECUTABLE" -s quit -c "$PAGESPEED_CONF"
while pgrep -x nginx > /dev/null; do sleep 1; done
fi
if [ "$AB_PID" != "0" ]; then
echo "Kill ab (pid: $AB_PID)"
kill -s KILL $AB_PID &>/dev/null || true
fi
start_test Logged output looks healthy.
# TODO(oschaaf): Sanity check for all the warnings/errors here.
OUT=$(cat "$ERROR_LOG" \
| grep "\\[" \
| grep -v "\\[debug\\]" \
| grep -v "\\[info\\]" \
| grep -v "\\[notice\\]" \
| grep -v "\\[warn\\].*Cache Flush.*" \
| grep -v "\\[warn\\].*doesnotexist.css.*" \
| grep -v "\\[warn\\].*Invalid filter name: bogus.*" \
| grep -v "\\[warn\\].*You seem to have downstream caching.*" \
| grep -v "\\[warn\\].*Warning_trigger*" \
| grep -v "\\[warn\\].*Rewrite http://www.google.com/mod_pagespeed_example/ failed*" \
| grep -v "\\[warn\\].*A.bad:0:Resource*" \
| grep -v "\\[warn\\].*W.bad.pagespeed.cf.hash.css*" \
| grep -v "\\[warn\\].*BadName*" \
| grep -v "\\[warn\\].*CSS parsing error*" \
| grep -v "\\[warn\\].*Fetch failed for resource*" \
| grep -v "\\[warn\\].*Rewrite.*example.pdf failed*" \
| grep -v "\\[warn\\].*Rewrite.*hello.js failed*" \
| grep -v "\\[warn\\].*Resource based on.*ngx_pagespeed_statistics.*" \
| grep -v "\\[warn\\].*Canceling 1 functions on sequence Shutdown.*" \
| grep -v "\\[warn\\].*using uninitialized.*" \
| grep -v "\\[warn\\].*Controller process .* exited with wait status 9" \
| grep -v "\\[warn\\].*Controller process .* exited with wait status 15" \
| grep -v "\\[error\\].*BadName*" \
| grep -v "\\[error\\].*/mod_pagespeed/bad*" \
| grep -v "\\[error\\].*doesnotexist.css.*" \
| grep -v "\\[error\\].*is forbidden.*" \
| grep -v "\\[error\\].*access forbidden by rule.*" \
| grep -v "\\[error\\].*forbidden.example.com*" \
| grep -v "\\[error\\].*custom-paths.example.com*" \
| grep -v "\\[error\\].*bogus_format*" \
| grep -v "\\[error\\].*/install/foo*" \
| grep -v "\\[error\\].*recv() failed*" \
| grep -v "\\[error\\].*send() failed*" \
| grep -v "\\[error\\].*Invalid url requested: js_defer.js.*" \
| grep -v "\\[error\\].*/mod_pagespeed_example/styles/yellow.css+blue.css.pagespeed.cc..css.*" \
| grep -v "\\[error\\].*/mod_pagespeed_example/images/Puzzle.jpg.pagespeed.ce..jpg.*" \
| grep -v "\\[error\\].*/pagespeed_custom_static/js_defer.js.*" \
| grep -v "\\[error\\].*UH8L-zY4b4AAAAAAAAAA.*" \
| grep -v "\\[error\\].*UH8L-zY4b4.*" \
| grep -v "\\[error\\].*Serf status 111(Connection refused) polling.*" \
| grep -v "\\[error\\].*Failed to make directory*" \
| grep -v "\\[error\\].*Could not create directories*" \
| grep -v "\\[error\\].*opening temp file: No such file or directory.*" \
| grep -v "\\[error\\].*remote\.cfg.*" \
| grep -v "\\[error\\].*Slow read operation on file.*" \
| grep -v "\\[error\\].*Slow ReadFile operation on file.*" \
| grep -v "\\[error\\].*Slow write operation on file.*" \
| grep -v "\\[warn\\].*remote\.cfg.*" \
| grep -v "\\[warn\\].*end token not received.*" \
| grep -v "\\[warn\\].*failed to hook next event.*" \
| grep -v "\\[warn\\].*Fetch timed out:.*" \
| grep -v "\\[warn\\].*Controller process .* exited with status code.*" \
| grep -v "\\[warn\\].*Rewrite.*failed.*.pagespeed....0.foo.*" \
| grep -v "\\[warn\\].*A.blue.css.*but cannot access the original.*" \
| grep -v "\\[warn\\].*Adding function to sequence.*" \
| grep -v "\\[warn\\].*special-response.*foo.css.*but cannot access the original.*" \
|| true)
check [ -z "$OUT" ]
check_failures_and_exit
lib/ngx_pagespeed/test/pagespeed_test.conf.template
-2308
View File
@@ -1,2308 +0,0 @@
# nginx_system_test.sh makes a few substitutions to pagespeed_test.conf.template
# file to generate @@TEST_TMP@@/pagespeed_test.conf
# The cache flush test depends on worker_processes=1.
worker_processes 1;
daemon @@DAEMON@@;
master_process on;
error_log "@@ERROR_LOG@@" debug;
pid "@@TEST_TMP@@/nginx.pid";
events {
worker_connections 1024;
}
http {
server_names_hash_bucket_size 128;
log_format cache '$time_local '
'$upstream_cache_status '
'$http_host $request ($status) '
'"$http_user_agent"';
access_log "@@ACCESS_LOG@@" cache;
# Don't put entries in the error log for 403s and 404s.
log_not_found off;
proxy_cache_path "@@PROXY_CACHE@@" levels=1:2 keys_zone=htmlcache:60m inactive=90m max_size=50m;
proxy_temp_path "@@TMP_PROXY_CACHE@@";
pagespeed ProcessScriptVariables all;
pagespeed StatisticsPath /ngx_pagespeed_statistics;
pagespeed GlobalStatisticsPath /ngx_pagespeed_global_statistics;
pagespeed ConsolePath /pagespeed_console;
pagespeed MessagesPath /ngx_pagespeed_message;
pagespeed AdminPath /pagespeed_admin;
pagespeed GlobalAdminPath /pagespeed_global_admin;
pagespeed StaticAssetPrefix /pagespeed_custom_static/;
pagespeed MessageBufferSize 200000;
# Increase the default fetcher timeout to resolve sporadic flakeyness when
# the native fetcher uses 8.8.8.8 to resolve.
pagespeed FetcherTimeoutMs 10000;
pagespeed NativeFetcherMaxKeepaliveRequests 50;
root "@@SERVER_ROOT@@";
# Block 5a: Decide on Cache-Control header value to use for outgoing
# response.
# Map new_cache_control_header_val to "no-cache, max-age=0" if the
# content is html and use the original Cache-Control header value
# in all other cases.
map $upstream_http_content_type $new_cache_control_header_val {
default $upstream_http_cache_control;
"~*text/html" "no-cache, max-age=0";
}
pagespeed UsePerVHostStatistics on;
pagespeed InPlaceResourceOptimization on;
pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
pagespeed BlockingRewriteKey psatest;
# We need to checkpoint more frequently than usual so that our checkpointing
# test will definitly have seen one of these intervals and so not be flaky.
# This needs to have a 1s interval because the checkpointing test has a 2s
# sleep.
pagespeed ShmMetadataCacheCheckpointIntervalSec 1;
# CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
# With this option enabled, the inline image system test will currently fail.
# When critical image beaconing is enabled, only critical images are inlined.
# In our current system test, we aren't yet sending a beacon to identify the
# critical images to be inlined, so we just disable the option here.
pagespeed CriticalImagesBeaconEnabled false;
# By default, resources will not be used for inlining without explicit
# authorization. Supported values are off or a comma-separated list of strings
# from {Script,Stylesheet}.
pagespeed InlineResourcesWithoutExplicitAuthorization off;
pagespeed Statistics on;
pagespeed StatisticsLogging on;
pagespeed LogDir "@@TEST_TMP@@/logdir";
# Expanded to CentralControllerPort directive when RUN_CONTROLLER_TEST=on
@@CONTROLLER@@
server {
# This is the default vhost for @@SECONDARY_PORT@@
# Your request will land here when you pass in an unknown host.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name default.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel CoreFilters;
}
server {
# Sets up a logical home-page server on
# max-cacheable-content-length.example.com. This server is only used to
# test ModPagespeedMaxCacheableContentLength, i.e.,
# max_cacheable_response_content_length.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name max-cacheable-content-length.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_javascript;
pagespeed MaxCacheableContentLength 85;
}
# Test how load from file handles maximum sizes.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name lff-large-files.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed MaxCacheableContentLength 4096;
pagespeed LoadFromFile
"http://lff-large-files.example.com/"
"@@SERVER_ROOT@@/";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name lff-large-files-no-fallback.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed MaxCacheableContentLength 4096;
# http://lff-large-files-no-fallback.example.com/foo.css won't load, because
# we've configured a LoadFromFile pattern that doesn't match the path the
# webserver would take.
pagespeed LoadFromFile
"http://lff-large-files-no-fallback.example.com/"
"@@SERVER_ROOT@@/mod_pagespeed_example/styles/";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name pagespeed-off.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed off;
pagespeed EnableFilters collapse_whitespace;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name pagespeed-on.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed on;
pagespeed EnableFilters collapse_whitespace;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name pagespeed-standby.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed standby;
pagespeed EnableFilters collapse_whitespace;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name pagespeed-unplugged.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed unplugged;
pagespeed EnableFilters collapse_whitespace;
}
pagespeed UseNativeFetcher "@@NATIVE_FETCHER@@";
@@RESOLVER@@
server {
# Block 1: Basic port, server_name definitions.
# This server represents the external caching layer server which
# receives user requests and proxies them to the upstream server
# running on the PRIMARY_PORT when the response is not available in
# the cache. It also services purge requests from the upstream server.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name proxy_cache.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
# Disable PageSpeed on this server.
pagespeed off;
# Block 2: Define prefix for proxy_cache_key based on the UserAgent.
# Define placeholder PS-CapabilityList header values for large and small
# screens with no UA dependent optimizations. Note that these placeholder
# values should not contain any of ll, ii, dj, jw or ws, since these
# codes will end up representing optimizations to be supported for the
# request.
set $default_ps_capability_list_for_large_screens "LargeScreen.SkipUADependentOptimizations";
set $default_ps_capability_list_for_small_screens "TinyScreen.SkipUADependentOptimizations";
# As a fallback, the PS-CapabilityList header that is sent to the upstream
# PageSpeed server should be for a large screen device with no browser
# specific optimizations.
set $ps_capability_list $default_ps_capability_list_for_large_screens;
# Cache-fragment 1: Desktop User-Agents that support lazyload_images (ll),
# inline_images (ii) and defer_javascript (dj).
# Note: Wget is added for testing purposes only.
if ($http_user_agent ~* "Chrome/|Firefox/|MSIE |Safari|Wget") {
set $ps_capability_list "ll,ii,dj:";
}
# Cache-fragment 2: Desktop User-Agents that support lazyload_images (ll),
# inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
# (ws).
if ($http_user_agent ~*
"Chrome/[2][3-9]+\.|Chrome/[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
set $ps_capability_list "ll,ii,dj,jw,ws:";
}
# Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
# match fragments 1 or 2 but should not because they represent older
# versions of certain browsers or bots and (b) Tablet User-Agents that
# correspond to large screens. These will only get optimizations that work
# on all browsers and use image compression qualities applicable to large
# screens. Note that even Tablets that are capable of supporting inline or
# webp images, e.g. Android 4.1.2, will not get these advanced
# optimizations.
if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.|bot|Yahoo!|Ruby|RPT-HTTPClient|(Google \(\+https\:\/\/developers\.google\.com\/\+\/web\/snippet\/\))|Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
set $ps_capability_list $default_ps_capability_list_for_large_screens;
}
# Cache-fragment 4: Mobiles and small screen Tablets will use image compression
# qualities applicable to small screens, but all other optimizations will be
# those that work on all browsers.
if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo|Galaxy Nexus|GT-I9300|GT-N7100|HTC One|Nexus [4|7|S]|Xoom|XT907") {
set $ps_capability_list $default_ps_capability_list_for_small_screens;
}
# Block 3a: Bypass the cache for .pagespeed. resource. PageSpeed has its own
# cache for these, and these could bloat up the caching layer.
if ($uri ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+") {
set $bypass_cache "1";
}
# For testing purposes, we never generate values that will result in a beacon
# unless a test request it via "X-Allow-Beacon: yes" in its request header.
# This is for testing purposes, note that in a production environment,
# you want 'set_random $rand 0 100;' unconditionally!
set $rand 5;
if ($http_x_allow_beacon ~ "yes") {
set_random $rand 0 100;
}
set $should_beacon_header_val "";
if ($rand ~* "^[0-4]$") {
set $should_beacon_header_val "random_rebeaconing_key";
set $bypass_cache 1;
}
# Block 3b: Only cache responses to clients that support gzip. Most clients
# do, and the cache holds much more if it stores gzipped responses.
if ($http_accept_encoding !~* gzip) {
set $bypass_cache "1";
}
# Block 4: Location block for purge requests.
location ~ /purge(/.*) {
allow 127.0.0.1;
allow ::1;
deny all;
proxy_cache_purge htmlcache $ps_capability_list$1$is_args$args;
}
# Block 6: Location block with proxy_cache directives.
location /mod_pagespeed_test/cachable_rewritten_html/ {
# 1: Upstream PageSpeed server is running at localhost:@@PRIMARY_PORT@@.
proxy_pass http://localhost:@@PRIMARY_PORT@@;
# 2: Use htmlcache as the zone for caching.
proxy_cache htmlcache;
# 3: Bypass requests that correspond to .pagespeed. resources
# or clients that do not support gzip etc.
proxy_cache_bypass $bypass_cache;
# 4: Use the redefined proxy_cache_key and make sure the /purge/
# block uses the same key.
proxy_cache_key $ps_capability_list$uri$is_args$args;
# 5: Forward Host header to upstream server.
proxy_set_header Host $host;
# 6: Set the PS-CapabilityList header for PageSpeed server to respect.
proxy_set_header PS-CapabilityList $ps_capability_list;
add_header PS-CapabilityList $ps_capability_list;
# 7: Add a header for identifying cache hits/misses/expires. This is
# for debugging purposes only and can be commented out in production.
add_header X-Cache $upstream_cache_status;
# Block 5b: Override Cache-Control headers as needed.
# Hide the upstream cache control header.
proxy_hide_header Cache-Control;
# Add the inferred Cache-Control header.
add_header Cache-Control $new_cache_control_header_val;
proxy_set_header PS-ShouldBeacon $should_beacon_header_val;
proxy_hide_header PS-ShouldBeacon;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name if-in-server.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RewriteLevel PassThrough;
set $inline_javascript "No";
if ($http_x_custom_header_inline_js) {
# TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
# pagespeed directives inside of a server location block to be respected,
# then uncomment the following line and duplicate the if-in-location test
# for if-in-server.
#pagespeed EnableFilters inline_javascript;
set $inline_javascript "Yes";
}
add_header "X-Inline-Javascript" $inline_javascript;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name if-in-location.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
location / {
set $inline_javascript "No";
pagespeed RewriteLevel PassThrough;
if ($http_x_custom_header_inline_js) {
pagespeed EnableFilters inline_javascript;
set $inline_javascript "Yes";
}
add_header "X-Inline-Javascript" $inline_javascript;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name secondary.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed MapProxyDomain secondary.example.com/gstatic_images
http://www.gstatic.com/psa/static;
}
# These three vhosts are for testing the experiment framework (Furious).
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name experiment.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed InPlaceResourceOptimization off;
pagespeed RunExperiment on;
pagespeed AnalyticsID "123-45-6734";
pagespeed UseAnalyticsJs false;
pagespeed ExperimentVariable 2;
pagespeed ExperimentSpec
"id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
pagespeed ExperimentSpec "id=3;default;percent=0";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name contentexperiment.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed InPlaceResourceOptimization off;
pagespeed RunExperiment on;
pagespeed AnalyticsID "123-45-6734";
pagespeed UseAnalyticsJs false;
pagespeed ExperimentVariable 2;
pagespeed ExperimentSpec
"id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=111";
pagespeed ExperimentSpec
"id=2;enable=recompress_images;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=222";
pagespeed ExperimentSpec
"id=3;default;percent=0;options=ContentExperimentID=123,ContentExperimentVariantID=333";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name experiment.noga.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RunExperiment on;
pagespeed ExperimentVariable 2;
pagespeed ExperimentSpec
"id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
pagespeed ExperimentSpec "id=3;default;percent=0";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name experiment.devicematch.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RunExperiment on;
pagespeed ExperimentSpec
"id=1;percent=100;matches_device_type=mobile;enable=recompress_images";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name experiment.ajax.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel CoreFilters;
pagespeed DisableFilters add_instrumentation;
pagespeed JsInlineMaxBytes 1;
pagespeed RunExperiment on;
pagespeed UseAnalyticsJs false;
pagespeed ExperimentSpec "id=1;percent=100;level=CoreFilters;enable=collapse_whitespace;options=JsInlineMaxBytes=1";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name ajax.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel CoreFilters;
pagespeed EnableFilters add_instrumentation;
pagespeed JsInlineMaxBytes 1;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name preserveurls.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed EnableFilters inline_preview_images,lazyload_images;
pagespeed EnableFilters resize_mobile_images,rewrite_images,sprite_images;
pagespeed EnableFilters fallback_rewrite_css_urls,flatten_css_imports;
pagespeed EnableFilters inline_css,move_css_above_scripts,move_css_to_head;
pagespeed EnableFilters outline_css,rewrite_css,combine_css;
pagespeed EnableFilters combine_javascript,defer_javascript;
pagespeed EnableFilters inline_javascript;
pagespeed EnableFilters outline_javascript,rewrite_javascript;
location /mod_pagespeed_test/preserveurls/on/ {
pagespeed JsPreserveURLs On;
pagespeed ImagePreserveURLs On;
pagespeed CssPreserveURLs On;
}
location /mod_pagespeed_test/preserveurls/off/ {
pagespeed JsPreserveURLs Off;
pagespeed ImagePreserveURLs Off;
pagespeed CssPreserveURLs Off;
}
}
server {
# Setup a vhost with the critical image beacon and lazyload filter enabled
# to make sure that critical images are not lazyloaded.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name imagebeacon.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters lazyload_images;
pagespeed CriticalImagesBeaconEnabled true;
}
server {
# Setup a vhost with the critical image beacon enabled to make sure that
# downstream caches and rebeaconing interact correctly.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name downstreamcacherebeacon.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed CriticalImagesBeaconEnabled true;
# Enable the downstream caching feature and specify a rebeaconing key.
pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
pagespeed DownstreamCacheRebeaconingKey random_rebeaconing_key;
location ~ .*[.]html {
add_header Cache-Control "private, max-age=3000";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name downstreamcacheresource.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_images;
pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
}
# Set up a reverse proxy (rproxy.) and origin (origin.) as vhosts for
# showing that we can configure PageSpeed via response headers.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name rproxy.rmcomments.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed DisableFilters add_instrumentation,remove_comments;
# Note that we don't enable remove_comments here; that setting comes from
# the response headers from origin.rmcomments.example.com
location / {
proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/;
proxy_set_header "Host" "origin.rmcomments.example.com";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name origin.rmcomments.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed off;
location / {
add_header PageSpeedFilters remove_comments;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name renderedimagebeacon.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters resize_rendered_image_dimensions;
pagespeed CriticalImagesBeaconEnabled true;
}
# Build a configuration hierarchy where at the root we have turned on
# OptimizeForBandwidth, and in various subdirectories we override settings
# to make them more aggressive.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name optimizeforbandwidth.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel OptimizeForBandwidth;
pagespeed DisableFilters add_instrumentation;
location /mod_pagespeed_test/optimize_for_bandwidth/inline_css {
pagespeed EnableFilters inline_css;
}
location /mod_pagespeed_test/optimize_for_bandwidth/css_urls {
pagespeed CssPreserveURLs off;
}
location /mod_pagespeed_test/optimize_for_bandwidth/image_urls {
pagespeed ImagePreserveURLs off;
}
location /mod_pagespeed_test/optimize_for_bandwidth/core_filters {
pagespeed RewriteLevel CoreFilters;
}
}
server {
# For testing with a custom origin header. In this VirtualHost,
# /mod_pagespeed_test is included in our DocumentRoot and thus does
# not need to be in any resource URL paths. This helps us verify that
# we are looping back to the corect VirtualHost -- if we hit the wrong
# one it will not work. Also we don't have a VirtualHost for
# sharedcdn.example.com, so the default Host header used for
# origin-mapping won't work either. Instead, we want origin-fetches
# to go back to this VirtualHost so we rely on the new third optional
# argument to MapOriginDomain.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name customhostheader.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@_test";
root "@@SERVER_ROOT@@/mod_pagespeed_test";
pagespeed on;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_images;
# Don't use localhost, as ngx_pagespeed's native fetcher cannot resolve it
pagespeed MapOriginDomain 127.0.0.1:@@SECONDARY_PORT@@/customhostheader
sharedcdn.example.com/test customhostheader.example.com;
pagespeed JpegRecompressionQuality 50;
pagespeed CriticalImagesBeaconEnabled false;
}
server {
# Sets up a virtual host where we can specify forbidden filters without
# affecting any other hosts.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name forbidden.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
# Start with all core filters enabled ...
pagespeed RewriteLevel CoreFilters;
# ... then forbid these filters ...
pagespeed ForbidFilters remove_quotes,remove_comments,collapse_whitespace;
pagespeed ForbidFilters rewrite_css,resize_images;
# ... and disable but not forbid this one (to ensure we retain its URL).
pagespeed DisableFilters inline_css;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name unauthorizedresources.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed InlineResourcesWithoutExplicitAuthorization Script,Stylesheet;
pagespeed CssInlineMaxBytes 1000000;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name client-domain-rewrite.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
# Don't actually try to rewrite any resources; the ones in
# rewrite_domains.html don't actually exist.
pagespeed RewriteLevel PassThrough;
pagespeed MapRewriteDomain http://client-domain-rewrite.example.com
http://src.example.com;
pagespeed ClientDomainRewrite true;
pagespeed EnableFilters rewrite_domains;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name url-attribute.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
# Don't actually try to rewrite any resources; the ones in
# rewrite_domains.html don't actually exist.
pagespeed RewriteLevel PassThrough;
# This is used for testing dynamically defined url-valued
# attributes
pagespeed UrlValuedAttribute span src Hyperlink;
pagespeed UrlValuedAttribute hr imgsrc Image;
pagespeed DomainRewriteHyperlinks on;
pagespeed MapRewriteDomain http://dst.example.com http://src.example.com;
pagespeed EnableFilters rewrite_domains;
pagespeed UrlValuedAttribute custom a Image;
pagespeed UrlValuedAttribute custom b otherResource;
pagespeed UrlValuedAttribute custom c hyperlink;
pagespeed UrlValuedAttribute img alt-src Image;
pagespeed UrlValuedAttribute video alt-a Image;
pagespeed UrlValuedAttribute video alt-b Image;
pagespeed UrlValuedAttribute link data-stylesheet Stylesheet;
pagespeed UrlValuedAttribute span data-stylesheet-a Stylesheet;
pagespeed UrlValuedAttribute span data-stylesheet-b Stylesheet;
pagespeed UrlValuedAttribute span data-stylesheet-c Stylesheet;
# Also test that we can redefine spec-defined attributes.
pagespeed UrlValuedAttribute blockquote cite Image;
}
server {
listen 127.0.0.2:@@SECONDARY_PORT@@;
server_name loopbackfetch.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name domain-hyperlinks-on.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
# Don't actually try to rewrite any resources; the ones in
# rewrite_domains.html don't actually exist.
pagespeed RewriteLevel PassThrough;
pagespeed DomainRewriteHyperlinks on;
pagespeed MapRewriteDomain http://dst.example.com http://src.example.com;
pagespeed EnableFilters rewrite_domains;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name domain-hyperlinks-off.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
# Don't actually try to rewrite any resources; the ones in
# rewrite_domains.html don't actually exist.
pagespeed RewriteLevel PassThrough;
pagespeed DomainRewriteHyperlinks off;
pagespeed MapRewriteDomain http://dst.example.com http://src.example.com;
pagespeed EnableFilters rewrite_domains;
}
server {
# Test host for shared memory cache.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name shmcache.example.com;
pagespeed FileCachePath "@@SHM_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_images;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name xfp.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RespectXForwardedProto on;
location /redirecting_origin {
pagespeed off;
# Hack: we clear the response headers using headers_more.
# If we don't, nginx will add an extra empty Location: headers here.
# It is kind of hard to get nginx to generate a relative location header
# that starts with "/".
more_clear_headers 'Location';
add_header Location /mod_pagespeed_example;
return 301;
}
location /redirect {
proxy_method GET;
proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/redirecting_origin;
proxy_set_header "Host" "xfp.example.com";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name xheader.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed XHeaderValue "UNSPECIFIED VERSION";
}
# Support for embedded configurations, where image flags in the
# VirtualHost serving HTML is not the same as the one serving resources,
# and thus we must embed the image flags in the rewritten image URLs.
#
# Note that we test with two distinct caches.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name embed-config-html.example.org;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/mod_pagespeed_test";
pagespeed AddOptionsToUrls on;
pagespeed JpegRecompressionQuality 73;
pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
pagespeed Domain embed-config-resources.example.com;
# Share a cache keyspace with embed-config-resources.example.com.
pagespeed CacheFragment "embed-config";
pagespeed LoadFromFile "http://embed-config-resources.example.com/"
"@@SERVER_ROOT@@/mod_pagespeed_example/";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name embed-config-resources.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/mod_pagespeed_example";
pagespeed AddOptionsToUrls on;
# Note that we do not set the jpeg quality here, but take
# it from image URL query parameters that we synthesize in
# from embed-config-html.example.org.
# Share a cache keyspace with embed-config-html.example.org.
pagespeed CacheFragment "embed-config";
pagespeed LoadFromFile "http://embed-config-resources.example.com/"
"@@SERVER_ROOT@@/mod_pagespeed_example/";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name ipro-for-browser.example.com;
root "@@SERVER_ROOT@@/mod_pagespeed_example";
pagespeed EnableFilters rewrite_images,rewrite_css;
pagespeed EnableFilters convert_to_webp_lossless;
pagespeed EnableFilters in_place_optimize_for_browser;
pagespeed JpegRecompressionQuality 75;
pagespeed WebpRecompressionQuality 70;
pagespeed InPlaceResourceOptimization on;
pagespeed AllowVaryOn "Accept";
pagespeed FileCachePath "@@IPRO_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name ipro-for-browser-vary-on-auto.example.com;
root "@@SERVER_ROOT@@/mod_pagespeed_example";
pagespeed EnableFilters rewrite_images,rewrite_css;
pagespeed EnableFilters convert_to_webp_animated;
pagespeed EnableFilters convert_to_webp_lossless;
pagespeed EnableFilters in_place_optimize_for_browser;
pagespeed InPlaceResourceOptimization on;
# pagespeed AllowVaryOn "Accept"; # Default is "Auto".
pagespeed ImageRecompressionQuality 90;
pagespeed JpegRecompressionQuality 75;
pagespeed JpegRecompressionQualityForSmallScreens 55;
pagespeed JpegQualityForSaveData 35;
pagespeed WebpRecompressionQuality 70;
pagespeed WebpRecompressionQualityForSmallScreens 50;
pagespeed WebpQualityForSaveData 30;
pagespeed WebpAnimatedRecompressionQuality 60;
pagespeed FileCachePath "@@IPRO_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name ipro-for-browser-vary-on-none.example.com;
root "@@SERVER_ROOT@@/mod_pagespeed_example";
pagespeed EnableFilters rewrite_images,in_place_optimize_for_browser;
pagespeed InPlaceResourceOptimization on;
pagespeed AllowVaryOn "None";
pagespeed ImageRecompressionQuality 75;
pagespeed FileCachePath "@@IPRO_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name respectvary.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RespectVary on;
add_header Vary User-Agent;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cache_a.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed CacheFlushPollIntervalSec 1;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters inline_css;
# Make a non-empty subdirectory config to make sure that
# cache.flush updates get transmitted to nested configurations.
#
# TODO(jmarantz): This test currently fails on ngx_pagespeed, meaning we
# don't handle cache flushing correctly here. Uncomment the config below to
# expose the error.
#location /mod_pagespeed_test/cache_flush/ {
# pagespeed RewriteLevel PassThrough;
# pagespeed EnableFilters inline_css;
# pagespeed DisableFilters add_instrumentation;
#}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cache_b.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed CacheFlushPollIntervalSec 1;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters inline_css;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cache_c.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed CacheFlushFilename "othercache.flush";
pagespeed CacheFlushPollIntervalSec 1;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters inline_css;
}
# The next three server blocks are for a cdn test.
server {
# Sets up a logical home-page server on www.example.com
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name www.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
pagespeed MapRewriteDomain cdn.example.com origin.example.com;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css,rewrite_images;
pagespeed CacheFragment "example";
}
server {
# Sets up a logical origin for CDNs to fetch content from, on
# origin.example.com.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name origin.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
pagespeed MapRewriteDomain cdn.example.com origin.example.com;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css,rewrite_images;
pagespeed CacheFragment "example";
}
server {
# Sets up a logical cdn, which is where we tell browsers to fetch resources
# from.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cdn.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
pagespeed MapRewriteDomain cdn.example.com origin.example.com;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css,rewrite_images;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name lff-ipro.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed LoadFromFile
"http://lff-ipro.example.com/mod_pagespeed_example/lff_ipro"
"@@SERVER_ROOT@@/mod_pagespeed_example/lff_ipro";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name tryfiles.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { }
location ~ "^/ngx_pagespeed_static/" { }
location ~ "^/ngx_pagespeed_beacon$" { }
try_files $uri $uri/ /mod_pagespeed_example/index.html;
}
server {
# Proxy modpagespeed.com for testing Issue 582.
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name modpagespeed.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed off;
location / {
proxy_pass http://modpagespeed.com;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name custom-paths.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed StatisticsPath /custom_pagespeed_statistics;
pagespeed ConsolePath /custom_pagespeed_console;
pagespeed MessagesPath /custom_pagespeed_message;
pagespeed AdminPath /custom_pagespeed_admin;
pagespeed StatisticsDomains Allow *;
pagespeed GlobalStatisticsDomains Allow *;
pagespeed MessagesDomains Allow *;
pagespeed ConsoleDomains Allow *;
pagespeed AdminDomains Allow *;
pagespeed GlobalAdminDomains Allow *;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name inherit-paths.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed StatisticsDomains Allow *;
pagespeed GlobalStatisticsDomains Allow *;
pagespeed MessagesDomains Allow *;
pagespeed ConsoleDomains Allow *;
pagespeed AdminDomains Allow *;
pagespeed GlobalAdminDomains Allow *;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name blocking.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_images;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name keepalive-html.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel CoreFilters;
error_log "@@TEST_TMP@@/keepalive-html.example.com.error.log" warn;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name keepalive-resource.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
error_log "@@TEST_TMP@@/keepalive-resource.example.com.error.log" warn;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name keepalive-beacon-get.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
error_log "@@TEST_TMP@@/keepalive-beacon-get.example.com.error.log" warn;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name keepalive-beacon-post.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
error_log "@@TEST_TMP@@/keepalive-beacon-post.example.com.error.log" warn;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name keepalive-static.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
error_log "@@TEST_TMP@@/keepalive-static.example.com.error.log" warn;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name response-header-filters.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed on;
location / {
proxy_pass http://localhost:@@SECONDARY_PORT@@;
proxy_set_header Host "enable-filter-header-origin.example.com";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name response-header-disable.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed EnableFilters add_instrumentation;
pagespeed on;
location / {
proxy_pass http://localhost:@@SECONDARY_PORT@@;
proxy_set_header Host "pagespeed-off-header-origin.example.com";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name headers.example.com;
pagespeed InPlaceRewriteDeadlineMs -1;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed LoadFromFile "http://headers.example.com/"
"@@SERVER_ROOT@@/";
location /mod_pagespeed_test/ {
more_set_headers "Server: override";
more_set_headers -s '404' 'Cache-Control: override';
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name custom404.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
error_page 404 /404.html;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name ipro.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed on;
pagespeed InPlaceResourceOptimization on;
location /mod_pagespeed_test/ipro/nocache/test_image_dont_reuse.png {
add_header Cache-Control no-cache;
}
location /mod_pagespeed_test/ipro/no-cache-control-header {
add_header Cache-Control "";
pagespeed ImplicitCacheTtlMs 333000;
}
}
# Test hosts to cover all possible cache configurations. L1 will be filecache
# or memcache depending on the setting of MEMCACHED_TEST. These four hosts
# are for the four settings for the L2 cache.
# 1. L2_d=LRU, L2_m=LRU
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name lrud-lrum.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_lrum";
pagespeed LRUCacheKbPerProcess 1024;
pagespeed LRUCacheByteLimit 2000;
pagespeed EnableFilters rewrite_images;
pagespeed CriticalImagesBeaconEnabled false;
}
# 2. L2_d=LRU, L2_m=SHM
pagespeed CreateSharedMemoryMetadataCache
"@@SECONDARY_CACHE@@_lrud_shmm" 8192;
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name lrud-shmm.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_shmm";
pagespeed LRUCacheKbPerProcess 1024;
pagespeed LRUCacheByteLimit 2000;
pagespeed EnableFilters rewrite_images;
pagespeed CriticalImagesBeaconEnabled false;
}
# 3. L2_d=none, L2_m=SHM
pagespeed CreateSharedMemoryMetadataCache
"@@SECONDARY_CACHE@@_noned_shmm" 8192;
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name noned-shmm.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_shmm";
pagespeed EnableFilters rewrite_images;
pagespeed CriticalImagesBeaconEnabled false;
}
# 4. L2_d=none, L2_m=none
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name noned-nonem.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_nonem";
pagespeed EnableFilters rewrite_images;
pagespeed CriticalImagesBeaconEnabled false;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name date.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
add_header "Date" "Fri, 16 Oct 2009 23:05:07 GMT";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name options-by-cookies-enabled.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_optionsbycookieson";
pagespeed AllowOptionsToBeSetByCookies true;
pagespeed StickyQueryParameters sticky_secret;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters collapse_whitespace;
pagespeed DisableFilters remove_comments,add_instrumentation;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name options-by-cookies-disabled.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_optionsbycookiesoff";
pagespeed AllowOptionsToBeSetByCookies false;
pagespeed DisableFilters add_instrumentation;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name redirect.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@_redirected";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters add_instrumentation,collapse_whitespace;
location /redirect {
rewrite ^/redirect/(.*) http://$server_name/$1 permanent;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name request-option-override.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RequestOptionOverride abc;
pagespeed RewriteLevel Passthrough;
pagespeed EnableFilters collapse_whitespace;
pagespeed DisableFilters remove_comments,add_instrumentation;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name signed-urls.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed UrlSigningKey helloworld;
pagespeed RewriteLevel Passthrough;
pagespeed EnableFilters collapse_whitespace,rewrite_images;
pagespeed DisableFilters remove_comments,add_instrumentation;
}
# For testing signed URLs, ignoring signature validity.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name signed-urls-transition.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed UrlSigningKey helloworld;
pagespeed AcceptInvalidSignatures true;
pagespeed RewriteLevel Passthrough;
pagespeed EnableFilters collapse_whitespace,rewrite_images;
pagespeed DisableFilters remove_comments,add_instrumentation;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name unsigned-urls-transition.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
# This server will not sign URLs, but AcceptInvalidSignature is on.
pagespeed AcceptInvalidSignatures true;
pagespeed RewriteLevel Passthrough;
pagespeed EnableFilters collapse_whitespace,rewrite_images;
pagespeed DisableFilters remove_comments,add_instrumentation;
}
# Start remote config servers.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/standard";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-partially-invalid.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl
"http://127.0.0.1:@@RCPORT@@/partly-invalid";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-invalid.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/invalid";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-failed-fetch.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/fail-future";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-slow-fetch.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/timeout";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-experiment.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/experiment";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-slightly-slow-fetch.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl
"http://127.0.0.1:@@RCPORT@@/slightly-slow";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-slightly-slow-expired-fetch.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/slow-expired";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-forbidden.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT@@/forbidden";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name remote-config-initially-forbidden.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed RemoteConfigurationUrl
"http://127.0.0.1:@@RCPORT@@/forbidden-once";
pagespeed RemoteConfigurationTimeoutMs 1500;
}
# Test that pagespeed is disabled when sendfile headers are present.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name uses-sendfile.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed EnableFilters inline_javascript,rewrite_javascript;
add_header 'X-Sendfile' 'blablabla';
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name doesnt-sendfile.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed EnableFilters inline_javascript,rewrite_javascript;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name uses-xaccelredirect.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed EnableFilters inline_javascript,rewrite_javascript;
add_header 'X-Accel-Redirect' 'blablabla';
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name map-static-domain.example.com;
pagespeed FileCachePath "@@SECONDARY_CACHE@@";
pagespeed MapRewriteDomain http://static-cdn.example.com
http://map-static-domain.example.com;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters defer_javascript,rewrite_domains;
}
# Proxy + IPRO a gzip'd file for testing Issue 896.
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name ipro-proxy.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
pagespeed on;
pagespeed InPlaceResourceOptimization on;
pagespeed EnableFilters rewrite_domains;
location / {
proxy_pass
"http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/mod_deflate/";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name compressedcache.example.com;
pagespeed on;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed EnableFilters rewrite_css;
pagespeed HttpCacheCompressionLevel 9;
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
fastcgi_buffering off;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name uncompressedcache.example.com;
pagespeed on;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed EnableFilters rewrite_css;
pagespeed HttpCacheCompressionLevel 0;
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
fastcgi_buffering off;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name compressed-css.example.com;
pagespeed on;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed InPlaceResourceOptimization on;
}
# nested gzip config: pagespeed gzip on/off
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name gzip-test1.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
location /mod_pagespeed_example/ {
pagespeed gzip off;
}
location /mod_pagespeed_example/styles/ {
pagespeed gzip on;
}
}
# nested gzip config: pagespeed on/off
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name gzip-test2.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
location /mod_pagespeed_example/ {
pagespeed off;
}
location /mod_pagespeed_example/styles/ {
pagespeed on;
}
}
server {
pagespeed on;
pagespeed EnableFilters add_instrumentation;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name proxy-post.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
location / {
# Transform the POST to a GET request for the origin,
# as nginx's static handler doesn't allow POST requests.
proxy_method GET;
proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/;
proxy_set_header "Host" "proxy-post-origin.example.com";
}
}
server {
pagespeed off;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name proxy-post-origin.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/";
}
server {
# Write all post data to temp files
client_body_in_file_only clean;
pagespeed CriticalImagesBeaconEnabled true;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name beacon-post-temp-file.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name script-filters.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/";
set $filters "";
set $domain_shards "cdn1.example.com,cdn2.example.com";
if ($http_X_Script) {
set $filters "add_instrumentation";
set $domain_shards "";
}
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_domains;
pagespeed EnableFilters $filters;
pagespeed ShardDomain script-filters.example.com "$domain_shards";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name purge.example.com;
# Test purging individual URLs without flushing the entire metadata cache.
pagespeed EnableCachePurge on;
pagespeed PurgeMethod PURGE;
root "@@SERVER_ROOT@@/purge";
pagespeed FileCachePath "@@FILE_CACHE@@_purge";
pagespeed DisableFilters add_instrumentation;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name psoff-dir-on.example.com;
# Test purging individual URLs without flushing the entire metadata cache.
pagespeed EnableCachePurge on;
pagespeed PurgeMethod PURGE;
root "@@SERVER_ROOT@@/purge";
pagespeed FileCachePath "@@FILE_CACHE@@_dir_on";
pagespeed DisableFilters add_instrumentation;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_css;
pagespeed off;
location / {
pagespeed on;
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cc-header-origin.example.com;
pagespeed InPlaceResourceOptimization off;
pagespeed off;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/";
location /mod_pagespeed_test/nostore {
add_header "Cache-Control" "max-age=12345";
add_header "Cache-Control" "public, no-store";
add_header "Cache-Control" "max-age=14";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name enable-filter-header-origin.example.com;
pagespeed InPlaceResourceOptimization off;
pagespeed off;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/";
location / {
add_header "PageSpeedFilters" "add_instrumentation";
}
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name pagespeed-off-header-origin.example.com;
pagespeed InPlaceResourceOptimization off;
pagespeed off;
pagespeed FileCachePath "@@FILE_CACHE@@";
root "@@SERVER_ROOT@@/";
location / {
add_header "PageSpeed" "off";
}
}
pagespeed MessagesDomains Allow messages-allowed.example.com;
pagespeed MessagesDomains Allow cleared-inherited.example.com;
pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
pagespeed MessagesDomains Allow more-messages-allowed.example.com;
pagespeed MessagesDomains Allow anything-*-wildcard.example.com;
pagespeed MessagesDomains Allow localhost;
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name messages-allowed.example.com
messages-not-allowed.example.com
more-messages-allowed.example.com
anything-a-wildcard.example.com
anything-b-wildcard.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name messages-still-not-allowed.example.com
but-this-message-allowed.example.com
and-this-one.example.com;
pagespeed MessagesDomains Allow but-this-message-allowed.example.com;
pagespeed MessagesDomains Allow and-this-one.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cleared-inherited.example.com
cleared-inherited-reallowed.example.com
messages-allowed-at-vhost.example.com
messages-not-allowed-at-vhost.example.com
anything-c-wildcard.example.com;
pagespeed MessagesDomains Disallow *;
pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
pagespeed MessagesDomains Allow messages-allowed-at-vhost.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name cleared-inherited-unlisted.example.com;
pagespeed MessagesDomains Allow *;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
server_name nothing-allowed.example.com;
pagespeed MessagesDomains Disallow *;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
server_name nothing-explicitly-allowed.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name everything-explicitly-allowed.example.com
everything-explicitly-allowed-but-aliased.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed StatisticsDomains Allow everything-explicitly-allowed.example.com;
pagespeed GlobalStatisticsDomains
Allow everything-explicitly-allowed.example.com;
pagespeed MessagesDomains Allow everything-explicitly-allowed.example.com;
pagespeed ConsoleDomains Allow everything-explicitly-allowed.example.com;
pagespeed AdminDomains Allow everything-explicitly-allowed.example.com;
pagespeed GlobalAdminDomains
Allow everything-explicitly-allowed.example.com;
}
server {
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name psol-vary.example.com;
pagespeed on;
pagespeed InPlaceResourceOptimization on;
pagespeed FileCachePath "@@FILE_CACHE@@";
}
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name flush.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed RewriteDeadlinePerFlushMs 1;
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
fastcgi_buffering off;
}
}
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name noflush.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed RewriteDeadlinePerFlushMs 1;
pagespeed FollowFlushes off;
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
fastcgi_buffering off;
}
}
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name special-response.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
error_page 404 /mod_pagespeed_test/php_withflush.php;
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
}
}
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name debug-filters.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters debug;
}
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name broken-fetch.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@/broken-fetch";
# Set up a fetch proxy that will 404 every request.
pagespeed FetchProxy "brokenfetch.example.com:1111";
# Prevent loopback fetch by explicitly authorizing the domain. Loopback
# fetches would work, and for this test, we're trying to have fetches fail,
# so they are instead picked up by the ipro recorder.
pagespeed Domain http://broken-fetch.example.com;
pagespeed InPlaceResourceOptimization on;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters rewrite_javascript;
pagespeed DisableFilters add_instrumentation;
pagespeed CriticalImagesBeaconEnabled false;
}
server {
pagespeed on;
listen @@SECONDARY_PORT@@;
listen [::]:@@SECONDARY_PORT@@;
server_name image-rewrite-with-flush.example.com;
pagespeed FileCachePath "@@FILE_CACHE@@";
pagespeed RewriteLevel PassThrough;
pagespeed UrlValuedAttribute li data-thumb image;
pagespeed EnableFilters core;
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
fastcgi_buffering off;
}
}
server {
listen @@PRIMARY_PORT@@;
listen [::]:@@PRIMARY_PORT@@;
server_name localhost;
pagespeed FileCachePath "@@FILE_CACHE@@";
location ~ ^/pagespeed_admin {
allow 127.0.0.1;
allow ::1;
deny all;
}
location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
# TODO(oschaaf): The following breaks nginx starting up, as the bad header
# value gets rejected during configuration parsing. However, we can't easily
# build a test for that right now.
# pagespeed AddResourceHeader "X-Fo\n\no" "Bar";
# pagespeed AddResourceHeader "X-Foo" "Ba\nr";
# For testing that AddResourceHeader gets passed through
pagespeed AddResourceHeader "X-Foo" "Bar";
add_header "" "";
}
# Backend for ipro-proxy.example.com
location /mod_pagespeed_test/ipro/mod_deflate/ {
pagespeed off;
}
location /mod_pagespeed_test/cachable_rewritten_html/ {
# This location has the html files that will be configured to be stored
# in the proxy_cache layer.
pagespeed DownstreamCachePurgeMethod "GET";
pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
pagespeed DownstreamCacheRebeaconingKey "random_rebeaconing_key";
# We use a very small deadline here to force the rewriting to not complete
# in the very first attempt.
pagespeed RewriteDeadlinePerFlushMs 1;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images;
pagespeed CriticalImagesBeaconEnabled true;
add_header Cache-Control "public, max-age=100";
}
location /mod_pagespeed_test/disable_no_transform/index.html {
pagespeed DisableRewriteOnNoTransform off;
}
location /mod_pagespeed_test/disable_no_transform/disable_no_transform.css {
add_header 'Cache-Control' 'no-transform';
}
location /mod_pagespeed_test/no_transform {
add_header 'Cache-Control' 'no-transform';
}
# uncomment the following two lines if you're testing memcached
#pagespeed MemcachedServers "localhost:11211";
#pagespeed MemcachedThreads 1;
pagespeed on;
#pagespeed CacheFlushPollIntervalSec 1;
#pagespeed RunExperiment on;
#pagespeed AnalyticsID "UA-XXXXXXXX-Y";
#pagespeed ExperimentSpec "id=1;percent=50;default";
#pagespeed ExperimentSpec "id=2;percent=50";
pagespeed Library 43 1o978_K0_LNE5_ystNklf
http://www.modpagespeed.com/rewrite_javascript.js;
pagespeed RetainComment " google_ad_section*";
pagespeed PermitIdsForCssCombining "allowed-to-combine-1-*";
pagespeed PermitIdsForCssCombining "allowed-to-combine-2-*";
# Test proxying of non-.pagespeed. resources.
pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/modpagespeed_http
http://@@PAGESPEED_TEST_HOST@@/do_not_modify;
pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_present
http://@@PAGESPEED_TEST_HOST@@:8091;
pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_absent
http://@@PAGESPEED_TEST_HOST@@:8092;
add_header X-Extra-Header 1;
# Establish a proxy mapping where the current server proxies an image
# stored on ref.pssdemos.com.
pagespeed MapProxyDomain localhost:@@PRIMARY_PORT@@/gstatic_images
http://www.gstatic.com/psa/static;
location /ngx_pagespeed_statistics {
allow 127.0.0.1;
allow ::1;
deny all;
}
location /ngx_pagespeed_message {
allow 127.0.0.1;
allow ::1;
deny all;
}
pagespeed Domain modpagespeed.com:1023;
# As opposed to mod_pagespeed, our default for aris is 'on'
pagespeed AvoidRenamingIntrospectiveJavascript off;
location /mod_pagespeed_test/nostore {
proxy_pass http://localhost:@@SECONDARY_PORT@@;
proxy_set_header Host "cc-header-origin.example.com";
}
location /mod_pagespeed_test/forbid_all_disabled/disabled {
# Prevent the enabling of these filters for files in this directory
# -and- all subdirectories.
#
# Apache checks here that they can't be renabled with .htaccess, but
# that's not how Nginx location blocks work.
pagespeed ForbidAllDisabledFilters true;
pagespeed DisableFilters remove_quotes,remove_comments;
pagespeed DisableFilters collapse_whitespace;
}
location /mod_pagespeed_test/max_html_parse_size {
pagespeed MaxHtmlParseBytes 5000;
}
location ~ \.php$ {
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
fastcgi_pass 127.0.0.1:9000;
}
location /mod_pagespeed_example/core_filters/ {
pagespeed RewriteLevel CoreFilters;
}
location /mod_pagespeed_example/pass_through/ {
pagespeed RewriteLevel PassThrough;
}
#location / {
# proxy_pass http://www.google.com;
#}
location /mod_pagespeed_test/ssi/ {
ssi on;
pagespeed EnableFilters collapse_whitespace,remove_comments,combine_css;
pagespeed DisableFilters add_instrumentation;
pagespeed DisableFilters inline_css;
}
location /mod_pagespeed_test/no_cache/ {
add_header Cache-Control no-cache;
}
location /mod_pagespeed_test/compressed/ {
add_header Content-Encoding gzip;
# Even though this is also in the server block it must be repeated here
# because add_header directives are only inherited if you don't define
# more of them. See: http://serverfault.com/questions/400197
add_header X-Extra-Header 1;
types {
text/javascript custom_ext;
}
}
location /mod_pagespeed_test/shard/ {
pagespeed ShardDomain "localhost:@@PRIMARY_PORT@@" shard1,shard2;
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters extend_cache;
}
location /mod_pagespeed_test/retain_cache_control/ {
pagespeed ModifyCachingHeaders off;
add_header Cache-Control "private, max-age=3000";
}
location /mod_pagespeed_test/retain_cache_control_with_downstream_caching/ {
pagespeed ModifyCachingHeaders on;
pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:8020/";
add_header Cache-Control "private, max-age=3000";
}
location /mod_pagespeed_test/avoid_renaming_introspective_javascript__on/ {
pagespeed AvoidRenamingIntrospectiveJavascript on;
}
location /mod_pagespeed_test/custom_options/ {
pagespeed DisableFilters convert_jpeg_to_progressive;
}
location /mod_pagespeed_test/ipro/test_image_dont_reuse.png {
expires 5m;
}
location /mod_pagespeed_test/example.json {
expires 10m;
}
location /mod_pagespeed_test/ipro/cc200/ {
add_header "Cache-Control" "max-age=200";
}
location /mod_pagespeed_test/ipro/cc200p/ {
add_header "Cache-Control" "private, max-age=200";
}
location /mod_pagespeed_test/ipro/cc200nc/ {
add_header "Cache-Control" "no-cache, max-age=200";
}
location /mod_pagespeed_test/ipro/cc200ns/ {
add_header "Cache-Control" "no-store, max-age=200";
}
location /mod_pagespeed_test/ipro/cc200nt/ {
add_header "Cache-Control" "no-transform, max-age=200";
}
location /mod_pagespeed_test/ipro/cc200sma5/ {
add_header "Cache-Control" "s-maxage=5, max-age=200";
}
location /mod_pagespeed_test/ipro/cc200sma50/ {
add_header "Cache-Control" "s-maxage=50, max-age=200";
}
location /mod_pagespeed_test/ipro/cc200sma50nsp/ {
add_header "Cache-Control" "s-maxage=50,max-age=200";
}
location /mod_pagespeed_test/ipro/cc9/ {
add_header "Cache-Control" "max-age=9";
}
location /mod_pagespeed_test/ipro/cc200cc9/ {
add_header "Cache-Control" "max-age=200, max-age=9";
}
location /mod_pagespeed_test/ipro/cc200sma50sma5/ {
add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=5";
}
location /mod_pagespeed_test/ipro/cc200sma50cc9/ {
add_header "Cache-Control" "max-age=200, s-maxage=50, max-age=9";
}
location /mod_pagespeed_test/ipro/cc200sma50cc9nsp/ {
add_header "Cache-Control" "max-age=200,s-maxage=50,max-age=9";
}
location /mod_pagespeed_test/ipro/cc200sma50sma51/ {
add_header "Cache-Control" "max-age=200, s-maxage=50, s-maxage=51";
}
location /mod_pagespeed_test/ipro/wait/ {
# TODO(jmarantz): ModPagespeedInPlaceWaitForOptimized should be superfluous,
# or made equivalent to ModPagespeedInPlaceRewriteDeadlineMs -1, which waits
# forever. Otherwise ModPagespeedInPlaceRewriteDeadlineMs should just have
# the specified deadline.
# # See https://github.com/pagespeed/mod_pagespeed/issues/1171 for more
# detailed discussion.
pagespeed InPlaceWaitForOptimized on;
}
location /mod_pagespeed_test/ipro/wait/long/ {
# Make the deadline long here for valgrind tests. We could
# conditionalize this.
pagespeed InPlaceRewriteDeadlineMs 10000;
}
location /mod_pagespeed_test/ipro/wait/short/ {
pagespeed EnableFilters in_place_optimize_for_browser;
# Make the deadline short here as we expect to always miss it
# in tests.
pagespeed InPlaceRewriteDeadlineMs 1;
}
location /mod_pagespeed_test/ipro/instant/wait/ {
pagespeed InPlaceWaitForOptimized on;
# TODO(jefftk): When running with valgrind we need to raise the rewrite
# deadline or else we get "Deadline exceeded for rewrite of resource".
# I had thought InPlaceWaitForOptimized was supposed to disable the
# rewrite deadline, but it seems not to.
#
# InPlaceWaitForOptimized is definitely doing something, though: when I
# remove the InPlaceRewriteDeadlineMs line below and run without valgrind,
# purple.css is optimized via ipro on the first request but if I set
# InPlaceWaitForOptimized to off then it is not.
pagespeed InPlaceRewriteDeadlineMs 1000;
}
location /mod_pagespeed_test/ipro/instant/deadline/ {
set $ps_deadline -1;
pagespeed ImageRecompressionQuality $ps_deadline;
pagespeed InPlaceRewriteDeadlineMs $ps_deadline;
}
# Test to make sure that user-authenticated resources do not get cached and
# optimized.
location /mod_pagespeed_test/auth/ {
auth_basic "Restricted";
auth_basic_user_file "@@SERVER_ROOT@@/mod_pagespeed_test/auth/passwd.conf";
}
location /mod_pagespeed_test/ipro/cookie/ {
# Add Vary:Cookie. This should prevent us from optimizing the
# vary_cookie.css even though PagespeedRespectVary is off.
# test/nginx_system_test.sh does the fetches test with and w/o cookies.
add_header Vary Cookie;
pagespeed RespectVary off;
pagespeed InPlaceWaitForOptimized on;
}
location /mod_pagespeed_test/ipro/cookie2/ {
# Add Vary:Cookie2. This should prevent us from optimizing the
# vary_cookie2.css even though PagespeedRespectVary is off.
# test/nginx_system_test.sh does the fetches test with and w/o cookie2.
add_header Vary Cookie2;
pagespeed RespectVary off;
pagespeed InPlaceWaitForOptimized on;
}
location /mod_pagespeed_test/vary/ {
pagespeed RespectVary on;
}
location ~ /mod_pagespeed_test/vary/\.(js|css)$ {
add_header Vary User-Agent;
}
location /mod_pagespeed_test/vary/no_respect/ {
pagespeed DisableFilters add_instrumentation,inline_css;
pagespeed RespectVary off;
}
location /mod_pagespeed_test/experimental_js_minifier/ {
pagespeed UseExperimentalJsMinifier on;
}
pagespeed FetchHttps enable;
location /mod_pagespeed_test/https_fetch/ {
pagespeed DisableFilters inline_images;
set $ps_gstatic "https://www.gstatic.com/psa/static";
pagespeed MapProxyDomain
http://localhost:@@PRIMARY_PORT@@/https_gstatic_dot_com
$ps_gstatic;
}
location /mod_pagespeed_test/strip_subresource_hints/default/ {
pagespeed DisableFilters add_instrumentation;
pagespeed RewriteLevel CoreFilters;
pagespeed DisAllow *dontrewriteme*;
}
location /mod_pagespeed_test/strip_subresource_hints/default_passthrough/ {
pagespeed DisableFilters add_instrumentation;
pagespeed RewriteLevel PassThrough;
}
location /mod_pagespeed_test/strip_subresource_hints/preserve_on/ {
pagespeed DisableFilters add_instrumentation;
pagespeed PreserveSubresourceHints on;
pagespeed RewriteLevel CoreFilters;
}
location /mod_pagespeed_test/strip_subresource_hints/preserve_off/ {
pagespeed DisableFilters add_instrumentation;
pagespeed PreserveSubresourceHints off;
pagespeed RewriteLevel CoreFilters;
}
# $host implicitly tests script variable support. I'd love to test it more
# directly, but so far this is the best I've come up with and duplicating
# the test doesn't seem to make sense.
pagespeed LoadFromFile
"http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
"@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
location /mod_pagespeed_test/public/ {
add_header "Cache-Control" "public, max-age=600";
pagespeed PreserveUrlRelativity off;
}
pagespeed EnableFilters remove_comments;
# Test LoadFromFile mapping by mapping one dir to another.
pagespeed LoadFromFile
"http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/load_from_file/web_dir/"
"@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/";
pagespeed LoadFromFileMatch
"^http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/load_from_file_match/web_([^/]*)/"
"@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_\1/";
pagespeed LoadFromFileRule Disallow
"@@SERVER_ROOT@@/mod_pagespeed_test/load_from_file/file_dir/httponly/";
pagespeed LoadFromFileRuleMatch Disallow \.ssp.css$ps_dollar;
pagespeed LoadFromFileRuleMatch Allow exception\.ssp\.css$ps_dollar;
#charset koi8-r;
#access_log logs/host.access.log main;
index index.html;
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
sendfile on;
keepalive_timeout 65;
types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
application/rss+xml rss;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
image/webp webp;
application/json json;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
audio/midi mid midi kar;
audio/mpeg mp3;
audio/ogg ogg;
audio/x-m4a m4a;
audio/x-realaudio ra;
video/3gpp 3gpp 3gp;
video/mp4 mp4;
video/mpeg mpeg mpg;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-m4v m4v;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
default_type application/octet-stream;
}
lib/ngx_pagespeed/test/run_tests.sh
-110
View File
@@ -1,110 +0,0 @@
#!/bin/bash
#
# Copyright 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: jefftk@google.com (Jeff Kaufman)
#
# Runs ngx_pagespeed system tests.
#
# Exits with status 0 if all tests pass.
# Exits with status 1 immediately if any test fails.
# Exits with status 2 if command line args are wrong.
#
# Usage:
# ./run_tests.sh
# Or:
# ./run_tests.sh /path/to/mod_pagespeed /path/to/nginx/binary
#
# If you built ngx_pagespeed with "scripts/build_ngx_pagespeed.sh --devel" then
# you don't need to pass any arguments to run_tests.sh. Otherwise, you'll need
# to tell it where to find a mod_pagespeed checkout (for example html files etc)
# and the nginx binary to test.
#
# By default the test script uses several ports. If you have a port conflict
# and need to override one you can do that by setting the relevant environment
# variable. For example:
# PRIMARY_PORT=1234 ./run_tests.sh /.../mod_pagespeed /.../nginx/binary
# Normally we test only with the native fetcher off. Set
# TEST_NATIVE_FETCHER=true to also test the native fetcher, set
# TEST_SERF_FETCHER=false to skip the serf fetcher.
TEST_NATIVE_FETCHER=${TEST_NATIVE_FETCHER:-false}
TEST_SERF_FETCHER=${TEST_SERF_FETCHER:-true}
# Normally we actually run the tests, but you might only want us to set up nginx
# for you so you can do manual testing. If so, set RUN_TESTS=false and this
# will exit after configuring and starting nginx.
RUN_TESTS=${RUN_TESTS:-true}
# To run tests with valgrind, set the environment variable USE_VALGRIND to
# true.
USE_VALGRIND=${USE_VALGRIND:-false}
if [ "$#" -eq 0 ]; then
MOD_PAGESPEED_DIR="testing-dependencies/mod_pagespeed/"
NGINX_EXECUTABLE="nginx/sbin/nginx"
elif [ "$#" -eq 2 ]; then
MOD_PAGESPEED_DIR="$1"
NGINX_EXECUTABLE="$2"
else
echo "Usage: $0 [mod_pagespeed_dir nginx_executable]"
exit 2
fi
: ${PRIMARY_PORT:=8050}
: ${SECONDARY_PORT:=8051}
: ${CONTROLLER_PORT:=8053}
: ${RCPORT:=9991}
: ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com}
: ${PHP_PORT:=9000}
this_dir="$( cd $(dirname "$0") && pwd)"
function run_test_checking_failure() {
"$MOD_PAGESPEED_DIR/install/start_php.sh" "$PHP_PORT"
USE_VALGRIND="$USE_VALGRIND" \
PRIMARY_PORT="$PRIMARY_PORT" \
SECONDARY_PORT="$SECONDARY_PORT" \
MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
RUN_TESTS="$RUN_TESTS" \
CONTROLLER_PORT="$CONTROLLER_PORT" \
RCPORT="$RCPORT" \
bash "$this_dir/nginx_system_test.sh"
STATUS=$?
echo "With $@ setup."
case $STATUS in
0)
return # No failure.
;;
3)
return # Only expected failures.
;;
4)
return # Return passing error code when running manually.
;;
*)
exit 1 # Real failure.
esac
}
if $TEST_SERF_FETCHER; then
NATIVE_FETCHER=off run_test_checking_failure "serf fetcher"
fi
if $TEST_NATIVE_FETCHER; then
NATIVE_FETCHER=on run_test_checking_failure "native fetcher"
fi
lib/ngx_pagespeed/test/valgrind.sup
-193
View File
@@ -1,193 +0,0 @@
# Copyright 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: oschaaf@we-amp.com (Otto van der Schaaf)
# The first few suppressions can be found in other modules
# and easily found when searched for, and seem false positives.
{
<nginx false positive>
Memcheck:Param
socketcall.sendmsg(msg.msg_iov[i])
fun:__sendmsg_nocancel
fun:ngx_write_channel
fun:ngx_signal_worker_processes
fun:ngx_master_process_cycle
fun:main
}
{
<nginx false positive>
Memcheck:Param
socketcall.sendmsg(msg.msg_iov[i])
fun:__sendmsg_nocancel
fun:ngx_write_channel
fun:ngx_master_process_cycle
fun:main
}
{
<nginx false positive>
Memcheck:Param
socketcall.sendmsg(msg.msg_iov[i])
fun:__sendmsg_nocancel
fun:ngx_write_channel
fun:ngx_pass_open_channel
fun:ngx_start_cache_manager_processes
fun:ngx_master_process_cycle
fun:main
}
{
<nginx false positive>
Memcheck:Param
socketcall.sendmsg(msg.msg_iov[i])
fun:__sendmsg_nocancel
fun:ngx_write_channel
fun:ngx_pass_open_channel
fun:ngx_start_cache_manager_processes
fun:ngx_master_process_cycle
fun:main
}
{
<nginx false positive>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_event_process_init
fun:ngx_worker_process_init
fun:ngx_worker_process_cycle
fun:ngx_spawn_process
fun:ngx_start_worker_processes
fun:ngx_master_process_cycle
fun:main
}
{
<nginx false positive>
Memcheck:Param
socketcall.sendmsg(msg.msg_iov[i])
fun:__sendmsg_nocancel
fun:ngx_write_channel
fun:ngx_pass_open_channel
fun:ngx_start_worker_processes
fun:ngx_master_process_cycle
fun:main
}
# similar to http://trac.nginx.org/nginx/ticket/369
{
<nginx false positive>
Memcheck:Param
pwrite64(buf)
obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
fun:ngx_write_file
fun:ngx_write_chain_to_file
fun:ngx_write_chain_to_temp_file
fun:ngx_event_pipe_write_chain_to_temp_file
fun:ngx_event_pipe
fun:ngx_http_upstream_process_upstream
fun:ngx_http_upstream_process_header
fun:ngx_http_upstream_handler
fun:ngx_epoll_process_events
fun:ngx_process_events_and_timers
fun:ngx_worker_process_cycle
}
# Mentioned in https://github.com/pagespeed/ngx_pagespeed/issues/103
# Assuming a false postives as the issue is closed.
{
<nginx false positive>
Memcheck:Param
write(buf)
obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
fun:ngx_log_error_core
fun:ngx_http_parse_complex_uri
fun:ngx_http_process_request_uri
fun:ngx_http_process_request_line
fun:ngx_http_wait_request_handler
fun:ngx_epoll_process_events
fun:ngx_process_events_and_timers
fun:ngx_worker_process_cycle
fun:ngx_spawn_process
fun:ngx_start_worker_processes
fun:ngx_master_process_cycle
}
# Extra suppresions for testing in release mode:
{
<re2 uninitialised value in optimized code>
Memcheck:Cond
fun:_ZN3re24Prog8OptimizeEv
...
}
{
<re2 uninitialised value in optimized code>
Memcheck:Value8
fun:_ZN3re24Prog8OptimizeEv
...
}
{
<re2 uninitialised value in optimized code>
Memcheck:Cond
fun:_ZN3re2L4AddQEPNS_9SparseSetEi
...
}
{
<re2 uninitialised value in optimized code>
Memcheck:Value8
fun:_ZN3re2L4AddQEPNS_9SparseSetEi
...
}
{
<re2 uninitialized value in optimized code>
Memcheck:Value8
fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
...
}
{
<re2 uninitialized value in optimized code>
Memcheck:Cond
fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
...
}
{
<re2 uninitialized value in optimized code>
Memcheck:Value8
fun:_ZNK3re210SparseSetTIvE8containsEi
...
}
{
<re2 uninitialized value in optimized code>
Memcheck:Cond
fun:_ZNK3re210SparseSetTIvE8containsEi
...
}
{
<re2 uninitialized value in optimized code>
Memcheck:Value8
fun:_ZNK3re211SparseArrayIiE9has_indexEi
...
}
{
<re2 uninitialized value in optimized code>
Memcheck:Cond
fun:_ZNK3re211SparseArrayIiE9has_indexEi
...
}